99 lines
3.3 KiB
Bash
99 lines
3.3 KiB
Bash
|
#!/bin/bash
|
|||
|
|
|
|||
|
|
# ==============================================================
|
|||
|
|
# txt.sh
|
|||
|
|
#
|
|||
|
|
# 把 Docusaurus build 出來的 HTML，轉換成純文字檔（.txt）
|
|||
|
|
#
|
|||
|
|
# 運作方式：
|
|||
|
|
# 1. 掃描 build 底下所有的 index.html
|
|||
|
|
# 2. 排除非文章頁面：tags、archive、page（分頁）、以及網站首頁
|
|||
|
|
# 3. 用 BeautifulSoup 抓出 <article> 內容，過濾掉側邊欄
|
|||
|
|
# 4. 移除 Docusaurus 自己塞在 <article> 裡的介面元素
|
|||
|
|
# （麵包屑、本頁導覽、編輯此頁⋯⋯只有 /docs 會有這些）
|
|||
|
|
# 5. 幫 h1~h4 標題加上裝飾符號，保留標題層級感
|
|||
|
|
# 6. 用 w3m -dump 轉成純文字，正確處理中英文寬度
|
|||
|
|
# 7. 輸出檔名採用文章 slug（資料夾名稱）
|
|||
|
|
#
|
|||
|
|
# 使用 xargs -P 平行處理，加快多篇文章的轉換速度
|
|||
|
|
#
|
|||
|
|
# 前置需求：python3、beautifulsoup4（pip install beautifulsoup4）、w3m
|
|||
|
|
# ==============================================================
|
|||
|
|
|
|||
|
|
cd "$(dirname "$0")"
|
|||
|
|
|
|||
|
|
|
|||
|
|
# 換成你自己的網頁 build 資料夾
|
|||
|
|
SRC="/path/to/build"
|
|||
|
|
DST="/path/to/build"
|
|||
|
|
mkdir -p "$DST"
|
|||
|
|
|
|||
|
|
export SRC DST
|
|||
|
|
|
|||
|
|
process_one() {
|
|||
|
|
file="$1"
|
|||
|
|
rel="${file#$SRC/}"
|
|||
|
|
slug=$(dirname "$rel")
|
|||
|
|
out="$DST/$slug.txt"
|
|||
|
|
|
|||
|
|
tmp=$(mktemp)
|
|||
|
|
|
|||
|
|
# 注意：python 程式碼用單引號包住，檔名透過 sys.argv 傳進去，
|
|||
|
|
# 這樣路徑裡有引號或空白也不會出事
|
|||
|
|
python3 -c '
|
|||
|
|
import sys
|
|||
|
|
from bs4 import BeautifulSoup
|
|||
|
|
|
|||
|
|
soup = BeautifulSoup(open(sys.argv[1], encoding="utf-8"), "html.parser")
|
|||
|
|
article = soup.find("article")
|
|||
|
|
if not article:
|
|||
|
|
article = soup
|
|||
|
|
|
|||
|
|
# Docusaurus 會把這些介面元素塞在 <article> 裡面，轉成純文字後很礙眼。
|
|||
|
|
# 主要是 /docs 頁面才有；不想拿掉哪一個，把該行前面加 # 註解掉即可。
|
|||
|
|
NOISE = [
|
|||
|
|
"nav.theme-doc-breadcrumbs", # 麵包屑（首頁 › 科學 › 文章名）
|
|||
|
|
".breadcrumbs", # 麵包屑的備用選擇器
|
|||
|
|
".theme-doc-toc-mobile", # 手機版的「本頁導覽」
|
|||
|
|
".theme-doc-toc-desktop", # 桌面版目錄（通常在 article 外，保險起見）
|
|||
|
|
".tableOfContents", # 目錄的備用選擇器
|
|||
|
|
".theme-doc-version-badge", # 版本標籤
|
|||
|
|
".theme-doc-version-banner", # 版本提示橫幅
|
|||
|
|
"footer.theme-doc-footer", # 頁尾的「編輯此頁」與最後更新時間
|
|||
|
|
".pagination-nav", # 上一頁／下一頁
|
|||
|
|
".theme-edit-this-page", # 編輯此頁的備用選擇器
|
|||
|
|
]
|
|||
|
|
|
|||
|
|
for selector in NOISE:
|
|||
|
|
for el in article.select(selector):
|
|||
|
|
el.decompose()
|
|||
|
|
|
|||
|
|
# 幫不同層級的標題加上裝飾，這樣轉成純文字之後還是看得出層級
|
|||
|
|
for tag in article.find_all("h1"):
|
|||
|
|
tag.string = "\n═══ " + tag.get_text() + " ═══\n"
|
|||
|
|
for tag in article.find_all("h2"):
|
|||
|
|
tag.string = "\n── " + tag.get_text() + " ──\n"
|
|||
|
|
for tag in article.find_all("h3"):
|
|||
|
|
tag.string = "\n◆ " + tag.get_text() + "\n"
|
|||
|
|
for tag in article.find_all("h4"):
|
|||
|
|
tag.string = "\n・" + tag.get_text() + "\n"
|
|||
|
|
|
|||
|
|
print(str(article))
|
|||
|
|
' "$file" > "$tmp"
|
|||
|
|
|
|||
|
|
w3m -dump -cols 80 -O UTF-8 -T text/html "$tmp" > "$out"
|
|||
|
|
rm -f "$tmp"
|
|||
|
|
|
|||
|
|
echo "完成:$out"
|
|||
|
|
}
|
|||
|
|
export -f process_one
|
|||
|
|
|
|||
|
|
find "$SRC" -name "*.html" \
|
|||
|
|
| grep -vE "/tags/|/archive/|/page/" \
|
|||
|
|
| grep -v "^${SRC}/index.html$" \
|
|||
|
|
| xargs -P 23 -I {} bash -c 'process_one "$@"' _ {}
|
|||
|
|
|
|||
|
|
echo ""
|
|||
|
|
echo "全部轉換完成!輸出在 $DST/"
|
|||
|
|
|
|||
|
|
python3 txt_index.py
|