From 9cf8196a3ec0e6342df91b24920bc7c5827bedd1 Mon Sep 17 00:00:00 2001 From: "Jeffery (jiantw83)" Date: Wed, 14 Oct 2026 17:12:20 +0000 Subject: [PATCH] =?UTF-8?q?feat:=20wiki=20=E8=AE=80=E5=8F=96=E8=88=87?= =?UTF-8?q?=E5=B0=8D=E7=99=BD=E6=8A=BD=E5=8F=96=E8=85=B3=E6=9C=AC=20(#4)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit fetch_wiki.py 用 tea 讀取 wiki；extract_dialogues.py 抽出「」對白與前後文， 並用規則判定明確的說話者。原文與輸出放在 repo 之外。 Co-Authored-By: Claude Sonnet 5.5 --- .gitignore | 5 +- tools/corpus/README.md | 48 +++++++ tools/corpus/characters.json | 21 +++ tools/corpus/extract_dialogues.py | 171 +++++++++++++++++++++++++ tools/corpus/fetch_wiki.py | 50 ++++++++ tools/corpus/test_extract_dialogues.py | 64 +++++++++ 6 files changed, 357 insertions(+), 2 deletions(-) create mode 100644 tools/corpus/README.md create mode 100644 tools/corpus/characters.json create mode 100644 tools/corpus/extract_dialogues.py create mode 100644 tools/corpus/fetch_wiki.py create mode 100644 tools/corpus/test_extract_dialogues.py diff --git a/.gitignore b/.gitignore index 9796fd8..4cbdad3 100644 --- a/.gitignore +++ b/.gitignore @@ -1,5 +1,6 @@ node_modules/ .env # 輕小說原文與建置中間檔放在 repo 之外，不入版本控制 -corpus/ -build/ +/corpus/ +/build/ +__pycache__/ diff --git a/tools/corpus/README.md b/tools/corpus/README.md new file mode 100644 index 0000000..70a04c1 --- /dev/null +++ b/tools/corpus/README.md @@ -0,0 +1,48 @@ +# 文本建置：wiki → 對白 + +建置流水線的第一段。這些腳本不屬於插件執行期；原文與輸出放在 repo 之外，不入版本控制。 + +## 1. 讀取 wiki + +用 tea 的驗證讀取 Gitea wiki 全部頁面，存成 markdown： + +```bash +python3 -I tools/corpus/fetch_wiki.py jiantw83/sao ~/anime-companion-data/sao/wiki +``` + +## 2. 抽出對白 + +抽出所有「」對白與前後文，並用規則判定明確的說話者： + +```bash +python3 -I tools/corpus/extract_dialogues.py ~/anime-companion-data/sao/wiki ~/anime-companion-data/sao/dialogues.jsonl +``` + +輸出每個對白一行 JSON： + +| 欄位 | 說明 | +| --- | --- | +| `id` | `v{冊}c{章}-{章內序號}` | +| `volume`、`chapter`、`page`、`para` | 位置；`page` 是分頁，`para` 是章內段落序號 | +| `text` | 對白內容（不含「」） | +| `before`、`after` | 同段落中對白前後的敘述 | +| `prev_par`、`next_par` | 前後段落 | +| `speaker` | 規則判定的角色代號，無法判定為 `null` | +| `alias` | 判定依據的名字（例如「莉法」或「直葉」） | +| `mode` | 由別名推得的模式：`ingame`、`real`、`leafa` | +| `confidence` | `high`：句中只有一個候選且有說話動詞；`medium`：多個候選取句首第一個，需 W4 覆核 | +| `rule` | `lead_in`（同段落「…說：」）、`lead_in_prev`（前一段以「：」結尾）、`trail`（對白後接「某某說」） | +| `candidates` | 句中出現的所有候選名字 | +| `suspect_duplicate` | 同一章 6 段內出現相同的長對白，疑為 wiki 頁面銜接的重複 | + +## 規則的邊界 + +這個譯本很少直接寫「某某說」，多數對白靠上下文（「他這麼說」「她點了點頭」）或輪流發言判斷，所以規則只涵蓋約一成的對白。其餘由 W4 的 sub-agent 標註。 + +別名表在 `characters.json`，新增角色或別名時改它。 + +## 測試 + +```bash +cd tools/corpus && python3 -m unittest -q test_extract_dialogues +``` diff --git a/tools/corpus/characters.json b/tools/corpus/characters.json new file mode 100644 index 0000000..07c9f8f --- /dev/null +++ b/tools/corpus/characters.json @@ -0,0 +1,21 @@ +{ + "亞絲娜": { "id": "asuna", "mode": "ingame" }, + "結城明日奈": { "id": "asuna", "mode": "real" }, + "明日奈": { "id": "asuna", "mode": "real" }, + "桐人": { "id": "kirito", "mode": "ingame" }, + "桐谷和人": { "id": "kirito", "mode": "real" }, + "和人": { "id": "kirito", "mode": "real" }, + "莉法": { "id": "suguha", "mode": "leafa" }, + "桐谷直葉": { "id": "suguha", "mode": "real" }, + "直葉": { "id": "suguha", "mode": "real" }, + "克萊因": { "id": "klein", "mode": "ingame" }, + "艾基爾": { "id": "agil", "mode": "ingame" }, + "莉茲貝特": { "id": "lisbeth", "mode": "ingame" }, + "莉茲": { "id": "lisbeth", "mode": "ingame" }, + "西莉卡": { "id": "silica", "mode": "ingame" }, + "結衣": { "id": "yui", "mode": "ingame" }, + "茅場晶彥": { "id": "kayaba", "mode": "ingame" }, + "茅場": { "id": "kayaba", "mode": "ingame" }, + "克拉帝爾": { "id": "kuradeel", "mode": "ingame" }, + "須鄉": { "id": "sugou", "mode": "real" } +} diff --git a/tools/corpus/extract_dialogues.py b/tools/corpus/extract_dialogues.py new file mode 100644 index 0000000..9dae438 --- /dev/null +++ b/tools/corpus/extract_dialogues.py @@ -0,0 +1,171 @@ +#!/usr/bin/env python3 +"""從 wiki 的小說頁抽出所有「」對白，並用規則判定明確的說話者。 + +用法：extract_dialogues.py <輸出 jsonl> + +每個對白一筆：冊、章、前後文、說話者（規則無法判定則為 null）與規則依據。 +規則只處理明確的句式；其餘交給 W4 的標註。輸出與原文放在 repo 之外。 +""" +import json +import re +import sys +from collections import Counter +from pathlib import Path + +ALIASES = json.loads((Path(__file__).with_name("characters.json")).read_text(encoding="utf-8")) +NAME_RE = "|".join(sorted(map(re.escape, ALIASES), key=len, reverse=True)) +TOKEN_RE = re.compile(f"({NAME_RE})|我(?!們)") +QUOTE_RE = re.compile(r"「([^」]*)」") +SENT_END = "。！？!?" +CUE_RE = re.compile(r"說|問|叫|喊|答|道|聲音|低語|呢喃|嘀咕|開口|回|笑|哭|吼|嘟噥|喃喃") +# 「對我」「在我背後的」中的「我」是受詞，不是說話者。 +OBJECT_RE = re.compile( + r"(?:在我(?:的)?(?:背後|身後|旁邊|面前|耳邊|身邊)的?|(?:對著?|向|朝|跟|和|問|告訴|看著|瞪著|望著|回|給|讓|把|被)我(?:說|問)?)" +) +CJK_DIGITS = {"零": 0, "一": 1, "二": 2, "三": 3, "四": 4, "五": 5, "六": 6, "七": 7, "八": 8, "九": 9} +FILE_RE = re.compile(r"Sword Art Online (\d) .*? - 第(.+?)章(?: \((\d+)_(\d+)\))?\.md$") + + +def cn_number(s): + if "十" in s: + head, _, tail = s.partition("十") + return (CJK_DIGITS[head] if head else 1) * 10 + (CJK_DIGITS[tail] if tail else 0) + return CJK_DIGITS[s] + + +def strip_quotes(text): + return QUOTE_RE.sub("", text) + + +def last_sentence(text): + text = text.strip() + parts = re.split(f"[{SENT_END}]", text) + parts = [p for p in parts if p.strip()] + return parts[-1].strip() if parts else "" + + +def first_sentence(text): + return re.split(f"[{SENT_END}]", text.strip(), maxsplit=1)[0] + + +FRAME_RE = re.compile(r"^(?:面對|聽見|聽到|看見|看到|見到|對於|當|由於|因為|雖然|在|到了|一邊|就在)") + + +def pick_subject(clause): + """回傳 (說話者別名, 全部候選, 是否高信心)。 + + 動詞前的主詞通常是句首第一個名字（跳過「面對…」「聽見…」之類的開頭子句）； + 只有一個候選且句中有說話動詞才算高信心，多個候選取第一個並降為中信心。 + """ + clause = OBJECT_RE.sub("", strip_quotes(clause)) + parts = [p for p in re.split("[，、,]", clause) if p.strip()] + while len(parts) > 1 and FRAME_RE.match(parts[0]): + parts = parts[1:] + tokens = [] + for m in TOKEN_RE.finditer("，".join(parts)): + alias = m.group(1) or "我" + if alias not in tokens: + tokens.append(alias) + if not tokens: + return None, [], False + return tokens[0], tokens, len(tokens) == 1 and bool(CUE_RE.search(clause)) + + +def resolve(alias): + if alias == "我": + return "kirito", "ingame" + info = ALIASES[alias] + return info["id"], info["mode"] + + +def attribute(before, after, prev_par): + """回傳 (speaker, alias, confidence, rule, candidates)。""" + b = before.strip() + if b.endswith(("：", ":")): + who, cands, cue = pick_subject(last_sentence(b)) + if who: + return (*resolve(who)[:1], who, "high" if cue else "medium", "lead_in", cands) + return None, None, None, None, cands + if not b and prev_par and prev_par.rstrip().endswith(("：", ":")): + who, cands, cue = pick_subject(last_sentence(prev_par)) + if who: + return (*resolve(who)[:1], who, "high" if cue else "medium", "lead_in_prev", cands) + return None, None, None, None, cands + a = after.lstrip("，、 ") + m = re.match(f"({NAME_RE}|我)", a) + if m and not a.startswith("我們"): + clause = first_sentence(a) + if CUE_RE.search(clause): + who, cands, _ = pick_subject(clause) + if who == m.group(1): + return (*resolve(who)[:1], who, "high", "trail", cands) + return None, None, None, None, [] + + +def trim(s, n=120): + s = s.strip() + return s if len(s) <= n else s[:n] + "…" + + +def load_chapters(wiki): + pages = {} + for f in sorted(Path(wiki).glob("*.md")): + m = FILE_RE.search(f.name) + if not m: + print(f"略過：{f.name}", file=sys.stderr) + continue + vol, ch, part = int(m.group(1)), cn_number(m.group(2)), int(m.group(3) or 1) + pages.setdefault((vol, ch), []).append((part, f)) + for key in sorted(pages): + paras = [] + for part, f in sorted(pages[key]): + paras += [(part, ln.strip()) for ln in f.read_text(encoding="utf-8").split("\n") if ln.strip()] + yield key, paras + + +def extract(wiki): + for (vol, ch), paras in load_chapters(wiki): + seen = {} + n = 0 + for pi, (part, par) in enumerate(paras): + quotes = list(QUOTE_RE.finditer(par)) + prev_par = paras[pi - 1][1] if pi else "" + next_par = paras[pi + 1][1] if pi + 1 < len(paras) else "" + for qi, q in enumerate(quotes): + lo = quotes[qi - 1].end() if qi else 0 + hi = quotes[qi + 1].start() if qi + 1 < len(quotes) else len(par) + before, after = par[lo:q.start()], par[q.end():hi] + speaker, alias, conf, rule, cands = attribute(before, after, prev_par if qi == 0 else "") + text = q.group(1) + n += 1 + dup = len(text) >= 8 and pi - seen.get(text, -99) <= 6 + seen[text] = pi + rec = { + "id": f"v{vol}c{ch:02d}-{n:04d}", "volume": vol, "chapter": ch, "page": part, "para": pi, + "text": text, "before": trim(before), "after": trim(after), + "prev_par": trim(prev_par), "next_par": trim(next_par), + "speaker": speaker, "alias": alias, "confidence": conf, "rule": rule, + "mode": resolve(alias)[1] if alias else None, + "candidates": cands, "suspect_duplicate": dup, + } + yield rec + + +def main(): + wiki, out = sys.argv[1], Path(sys.argv[2]) + out.parent.mkdir(parents=True, exist_ok=True) + stats, rules = Counter(), Counter() + total = 0 + with out.open("w", encoding="utf-8") as fh: + for rec in extract(wiki): + fh.write(json.dumps(rec, ensure_ascii=False) + "\n") + total += 1 + stats[rec["speaker"] or "(未判定)"] += 1 + rules[rec["rule"] or "(無)"] += 1 + print(f"{total} 句 -> {out}") + print("說話者：", dict(stats.most_common())) + print("規則：", dict(rules.most_common())) + + +if __name__ == "__main__": + main() diff --git a/tools/corpus/fetch_wiki.py b/tools/corpus/fetch_wiki.py new file mode 100644 index 0000000..e534a5e --- /dev/null +++ b/tools/corpus/fetch_wiki.py @@ -0,0 +1,50 @@ +#!/usr/bin/env python3 +"""用 tea 的驗證讀取 Gitea wiki 全部頁面，存成 markdown。 + +用法：fetch_wiki.py <輸出目錄> +輸出目錄放在 repo 之外；原文不入版本控制。 +""" +import base64 +import json +import re +import subprocess +import sys +from pathlib import Path + + +def tea_api(endpoint): + out = subprocess.run(["tea", "api", endpoint], capture_output=True, text=True, check=True).stdout + return json.loads(out) + + +def list_pages(repo): + pages, page = [], 1 + while True: + batch = tea_api(f"repos/{repo}/wiki/pages?page={page}&limit=50") + if not batch: + return pages + pages.extend(batch) + page += 1 + + +def safe_name(title): + return re.sub(r"[\\/:*?\"<>|]", "_", title).strip() + + +def main(): + repo, outdir = sys.argv[1], Path(sys.argv[2]) + outdir.mkdir(parents=True, exist_ok=True) + pages = list_pages(repo) + seen = set() + for p in pages: + if p["title"] in seen: + continue + seen.add(p["title"]) + page = tea_api(f"repos/{repo}/wiki/page/{p['sub_url']}") + text = base64.b64decode(page["content_base64"]).decode("utf-8") + (outdir / f"{safe_name(p['title'])}.md").write_text(text, encoding="utf-8") + print(f"{len(seen)} pages -> {outdir}") + + +if __name__ == "__main__": + main() diff --git a/tools/corpus/test_extract_dialogues.py b/tools/corpus/test_extract_dialogues.py new file mode 100644 index 0000000..bf6f54a --- /dev/null +++ b/tools/corpus/test_extract_dialogues.py @@ -0,0 +1,64 @@ +import unittest + +from extract_dialogues import attribute, cn_number, pick_subject + + +class PickSubject(unittest.TestCase): + def test_single_candidate_with_cue_is_high(self): + self.assertEqual(pick_subject("亞絲娜一邊流淚一邊對我叫道"), ("亞絲娜", ["亞絲娜"], True)) + + def test_object_wo_is_not_speaker(self): + self.assertEqual(pick_subject("在我背後的亞絲娜同樣相當氣憤地反問")[0], "亞絲娜") + + def test_narrator_is_wo(self): + self.assertEqual(pick_subject("我低聲說道")[0], "我") + + def test_first_subject_wins_and_is_not_high(self): + who, cands, high = pick_subject("我關掉地圖，瞄了亞絲娜一眼，說道") + self.assertEqual((who, cands, high), ("我", ["我", "亞絲娜"], False)) + + def test_frame_clause_is_skipped(self): + self.assertEqual(pick_subject("面對桐人這種問題，莉法先是瞪了他一下後才繼續說道")[0], "莉法") + + def test_wo_men_is_not_speaker(self): + self.assertIsNone(pick_subject("我們一起說道")[0]) + + def test_no_candidate(self): + self.assertEqual(pick_subject("她聳了聳肩後繼續說道"), (None, [], False)) + + +class Attribute(unittest.TestCase): + def test_lead_in_same_paragraph(self): + r = attribute("亞絲娜小聲說：", "", "") + self.assertEqual((r[0], r[2], r[3]), ("asuna", "high", "lead_in")) + + def test_lead_in_previous_paragraph_even_with_inner_quote(self): + r = attribute("", "", "他喊著「回復！」。亞絲娜小聲對我說：") + self.assertEqual((r[0], r[3]), ("asuna", "lead_in_prev")) + + def test_narrator_maps_to_kirito(self): + self.assertEqual(attribute("我低聲說道：", "", "")[0], "kirito") + + def test_trail(self): + r = attribute("", "，亞絲娜低聲說。", "") + self.assertEqual((r[0], r[3]), ("asuna", "trail")) + + def test_trail_without_cue_is_not_attributed(self): + self.assertIsNone(attribute("", "亞絲娜點了點頭。", "")[0]) + + def test_leafa_mode_from_alias(self): + from extract_dialogues import resolve + self.assertEqual(resolve("莉法"), ("suguha", "leafa")) + self.assertEqual(resolve("直葉"), ("suguha", "real")) + + def test_unknown_when_nothing_matches(self): + self.assertEqual(attribute("", "", "")[0], None) + + +class Numerals(unittest.TestCase): + def test_chapters(self): + self.assertEqual([cn_number(x) for x in ("一", "十", "十一", "二十", "二十五")], [1, 10, 11, 20, 25]) + + +if __name__ == "__main__": + unittest.main()