feat: wiki 讀取與對白抽取腳本 (#4)
fetch_wiki.py 用 tea 讀取 wiki;extract_dialogues.py 抽出「」對白與前後文, 並用規則判定明確的說話者。原文與輸出放在 repo 之外。 Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Sonnet 5.5
parent
b0ed5efedf
commit
9cf8196a3e
@@ -0,0 +1,50 @@
|
||||
#!/usr/bin/env python3
|
||||
"""用 tea 的驗證讀取 Gitea wiki 全部頁面,存成 markdown。
|
||||
|
||||
用法:fetch_wiki.py <owner/repo> <輸出目錄>
|
||||
輸出目錄放在 repo 之外;原文不入版本控制。
|
||||
"""
|
||||
import base64
|
||||
import json
|
||||
import re
|
||||
import subprocess
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
def tea_api(endpoint):
|
||||
out = subprocess.run(["tea", "api", endpoint], capture_output=True, text=True, check=True).stdout
|
||||
return json.loads(out)
|
||||
|
||||
|
||||
def list_pages(repo):
|
||||
pages, page = [], 1
|
||||
while True:
|
||||
batch = tea_api(f"repos/{repo}/wiki/pages?page={page}&limit=50")
|
||||
if not batch:
|
||||
return pages
|
||||
pages.extend(batch)
|
||||
page += 1
|
||||
|
||||
|
||||
def safe_name(title):
|
||||
return re.sub(r"[\\/:*?\"<>|]", "_", title).strip()
|
||||
|
||||
|
||||
def main():
|
||||
repo, outdir = sys.argv[1], Path(sys.argv[2])
|
||||
outdir.mkdir(parents=True, exist_ok=True)
|
||||
pages = list_pages(repo)
|
||||
seen = set()
|
||||
for p in pages:
|
||||
if p["title"] in seen:
|
||||
continue
|
||||
seen.add(p["title"])
|
||||
page = tea_api(f"repos/{repo}/wiki/page/{p['sub_url']}")
|
||||
text = base64.b64decode(page["content_base64"]).decode("utf-8")
|
||||
(outdir / f"{safe_name(p['title'])}.md").write_text(text, encoding="utf-8")
|
||||
print(f"{len(seen)} pages -> {outdir}")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Reference in New Issue
Block a user