Merge pull request 'feat: wiki 讀取與對白抽取腳本' (#13) from feat/w3-corpus-extract into main

Reviewed-on: https://gitea.jsc.idv.tw/jiantw83/jsc-plugins/pulls/13

.gitignore

@@ -1,5 +1,6 @@
 node_modules/
 .env
 # 輕小說原文與建置中間檔放在 repo 之外，不入版本控制
-corpus/
+/corpus/
-build/
+/build/
+__pycache__/

tools/corpus/README.md

@@ -0,0 +1,48 @@
+# 文本建置：wiki → 對白
+
+建置流水線的第一段。這些腳本不屬於插件執行期；原文與輸出放在 repo 之外，不入版本控制。
+
+## 1. 讀取 wiki
+
+用 tea 的驗證讀取 Gitea wiki 全部頁面，存成 markdown：
+
+```bash
+python3 -I tools/corpus/fetch_wiki.py jiantw83/sao ~/anime-companion-data/sao/wiki
+```
+
+## 2. 抽出對白
+
+抽出所有「」對白與前後文，並用規則判定明確的說話者：
+
+```bash
+python3 -I tools/corpus/extract_dialogues.py ~/anime-companion-data/sao/wiki ~/anime-companion-data/sao/dialogues.jsonl
+```
+
+輸出每個對白一行 JSON：
+
+| 欄位 | 說明 |
+| --- | --- |
+| `id` | `v{冊}c{章}-{章內序號}` |
+| `volume`、`chapter`、`page`、`para` | 位置；`page` 是分頁，`para` 是章內段落序號 |
+| `text` | 對白內容（不含「」） |
+| `before`、`after` | 同段落中對白前後的敘述 |
+| `prev_par`、`next_par` | 前後段落 |
+| `speaker` | 規則判定的角色代號，無法判定為 `null` |
+| `alias` | 判定依據的名字（例如「莉法」或「直葉」） |
+| `mode` | 由別名推得的模式：`ingame`、`real`、`leafa` |
+| `confidence` | `high`：句中只有一個候選且有說話動詞；`medium`：多個候選取句首第一個，需 W4 覆核 |
+| `rule` | `lead_in`（同段落「…說：」）、`lead_in_prev`（前一段以「：」結尾）、`trail`（對白後接「某某說」） |
+| `candidates` | 句中出現的所有候選名字 |
+| `suspect_duplicate` | 同一章 6 段內出現相同的長對白，疑為 wiki 頁面銜接的重複 |
+
+## 規則的邊界
+
+這個譯本很少直接寫「某某說」，多數對白靠上下文（「他這麼說」「她點了點頭」）或輪流發言判斷，所以規則只涵蓋約一成的對白。其餘由 W4 的 sub-agent 標註。
+
+別名表在 `characters.json`，新增角色或別名時改它。
+
+## 測試
+
+```bash
+cd tools/corpus && python3 -m unittest -q test_extract_dialogues
+```

tools/corpus/characters.json

@@ -0,0 +1,21 @@
+{
+  "亞絲娜": { "id": "asuna", "mode": "ingame" },
+  "結城明日奈": { "id": "asuna", "mode": "real" },
+  "明日奈": { "id": "asuna", "mode": "real" },
+  "桐人": { "id": "kirito", "mode": "ingame" },
+  "桐谷和人": { "id": "kirito", "mode": "real" },
+  "和人": { "id": "kirito", "mode": "real" },
+  "莉法": { "id": "suguha", "mode": "leafa" },
+  "桐谷直葉": { "id": "suguha", "mode": "real" },
+  "直葉": { "id": "suguha", "mode": "real" },
+  "克萊因": { "id": "klein", "mode": "ingame" },
+  "艾基爾": { "id": "agil", "mode": "ingame" },
+  "莉茲貝特": { "id": "lisbeth", "mode": "ingame" },
+  "莉茲": { "id": "lisbeth", "mode": "ingame" },
+  "西莉卡": { "id": "silica", "mode": "ingame" },
+  "結衣": { "id": "yui", "mode": "ingame" },
+  "茅場晶彥": { "id": "kayaba", "mode": "ingame" },
+  "茅場": { "id": "kayaba", "mode": "ingame" },
+  "克拉帝爾": { "id": "kuradeel", "mode": "ingame" },
+  "須鄉": { "id": "sugou", "mode": "real" }
+}

tools/corpus/extract_dialogues.py

@@ -0,0 +1,171 @@
+#!/usr/bin/env python3
+"""從 wiki 的小說頁抽出所有「」對白，並用規則判定明確的說話者。
+
+用法：extract_dialogues.py <wiki 目錄> <輸出 jsonl>
+
+每個對白一筆：冊、章、前後文、說話者（規則無法判定則為 null）與規則依據。
+規則只處理明確的句式；其餘交給 W4 的標註。輸出與原文放在 repo 之外。
+"""
+import json
+import re
+import sys
+from collections import Counter
+from pathlib import Path
+
+ALIASES = json.loads((Path(__file__).with_name("characters.json")).read_text(encoding="utf-8"))
+NAME_RE = "|".join(sorted(map(re.escape, ALIASES), key=len, reverse=True))
+TOKEN_RE = re.compile(f"({NAME_RE})|我(?!們)")
+QUOTE_RE = re.compile(r"「([^」]*)」")
+SENT_END = "。！？!?"
+CUE_RE = re.compile(r"說|問|叫|喊|答|道|聲音|低語|呢喃|嘀咕|開口|回|笑|哭|吼|嘟噥|喃喃")
+# 「對我」「在我背後的」中的「我」是受詞，不是說話者。
+OBJECT_RE = re.compile(
+    r"(?:在我(?:的)?(?:背後|身後|旁邊|面前|耳邊|身邊)的?|(?:對著?|向|朝|跟|和|問|告訴|看著|瞪著|望著|回|給|讓|把|被)我(?:說|問)?)"
+)
+CJK_DIGITS = {"零": 0, "一": 1, "二": 2, "三": 3, "四": 4, "五": 5, "六": 6, "七": 7, "八": 8, "九": 9}
+FILE_RE = re.compile(r"Sword Art Online (\d) .*? - 第(.+?)章(?: \((\d+)_(\d+)\))?\.md$")
+
+
+def cn_number(s):
+    if "十" in s:
+        head, _, tail = s.partition("十")
+        return (CJK_DIGITS[head] if head else 1) * 10 + (CJK_DIGITS[tail] if tail else 0)
+    return CJK_DIGITS[s]
+
+
+def strip_quotes(text):
+    return QUOTE_RE.sub("", text)
+
+
+def last_sentence(text):
+    text = text.strip()
+    parts = re.split(f"[{SENT_END}]", text)
+    parts = [p for p in parts if p.strip()]
+    return parts[-1].strip() if parts else ""
+
+
+def first_sentence(text):
+    return re.split(f"[{SENT_END}]", text.strip(), maxsplit=1)[0]
+
+
+FRAME_RE = re.compile(r"^(?:面對|聽見|聽到|看見|看到|見到|對於|當|由於|因為|雖然|在|到了|一邊|就在)")
+
+
+def pick_subject(clause):
+    """回傳 (說話者別名, 全部候選, 是否高信心)。
+
+    動詞前的主詞通常是句首第一個名字（跳過「面對…」「聽見…」之類的開頭子句）；
+    只有一個候選且句中有說話動詞才算高信心，多個候選取第一個並降為中信心。
+    """
+    clause = OBJECT_RE.sub("", strip_quotes(clause))
+    parts = [p for p in re.split("[，、,]", clause) if p.strip()]
+    while len(parts) > 1 and FRAME_RE.match(parts[0]):
+        parts = parts[1:]
+    tokens = []
+    for m in TOKEN_RE.finditer("，".join(parts)):
+        alias = m.group(1) or "我"
+        if alias not in tokens:
+            tokens.append(alias)
+    if not tokens:
+        return None, [], False
+    return tokens[0], tokens, len(tokens) == 1 and bool(CUE_RE.search(clause))
+
+
+def resolve(alias):
+    if alias == "我":
+        return "kirito", "ingame"
+    info = ALIASES[alias]
+    return info["id"], info["mode"]
+
+
+def attribute(before, after, prev_par):
+    """回傳 (speaker, alias, confidence, rule, candidates)。"""
+    b = before.strip()
+    if b.endswith(("：", ":")):
+        who, cands, cue = pick_subject(last_sentence(b))
+        if who:
+            return (*resolve(who)[:1], who, "high" if cue else "medium", "lead_in", cands)
+        return None, None, None, None, cands
+    if not b and prev_par and prev_par.rstrip().endswith(("：", ":")):
+        who, cands, cue = pick_subject(last_sentence(prev_par))
+        if who:
+            return (*resolve(who)[:1], who, "high" if cue else "medium", "lead_in_prev", cands)
+        return None, None, None, None, cands
+    a = after.lstrip("，、 ")
+    m = re.match(f"({NAME_RE}|我)", a)
+    if m and not a.startswith("我們"):
+        clause = first_sentence(a)
+        if CUE_RE.search(clause):
+            who, cands, _ = pick_subject(clause)
+            if who == m.group(1):
+                return (*resolve(who)[:1], who, "high", "trail", cands)
+    return None, None, None, None, []
+
+
+def trim(s, n=120):
+    s = s.strip()
+    return s if len(s) <= n else s[:n] + "…"
+
+
+def load_chapters(wiki):
+    pages = {}
+    for f in sorted(Path(wiki).glob("*.md")):
+        m = FILE_RE.search(f.name)
+        if not m:
+            print(f"略過：{f.name}", file=sys.stderr)
+            continue
+        vol, ch, part = int(m.group(1)), cn_number(m.group(2)), int(m.group(3) or 1)
+        pages.setdefault((vol, ch), []).append((part, f))
+    for key in sorted(pages):
+        paras = []
+        for part, f in sorted(pages[key]):
+            paras += [(part, ln.strip()) for ln in f.read_text(encoding="utf-8").split("\n") if ln.strip()]
+        yield key, paras
+
+
+def extract(wiki):
+    for (vol, ch), paras in load_chapters(wiki):
+        seen = {}
+        n = 0
+        for pi, (part, par) in enumerate(paras):
+            quotes = list(QUOTE_RE.finditer(par))
+            prev_par = paras[pi - 1][1] if pi else ""
+            next_par = paras[pi + 1][1] if pi + 1 < len(paras) else ""
+            for qi, q in enumerate(quotes):
+                lo = quotes[qi - 1].end() if qi else 0
+                hi = quotes[qi + 1].start() if qi + 1 < len(quotes) else len(par)
+                before, after = par[lo:q.start()], par[q.end():hi]
+                speaker, alias, conf, rule, cands = attribute(before, after, prev_par if qi == 0 else "")
+                text = q.group(1)
+                n += 1
+                dup = len(text) >= 8 and pi - seen.get(text, -99) <= 6
+                seen[text] = pi
+                rec = {
+                    "id": f"v{vol}c{ch:02d}-{n:04d}", "volume": vol, "chapter": ch, "page": part, "para": pi,
+                    "text": text, "before": trim(before), "after": trim(after),
+                    "prev_par": trim(prev_par), "next_par": trim(next_par),
+                    "speaker": speaker, "alias": alias, "confidence": conf, "rule": rule,
+                    "mode": resolve(alias)[1] if alias else None,
+                    "candidates": cands, "suspect_duplicate": dup,
+                }
+                yield rec
+
+
+def main():
+    wiki, out = sys.argv[1], Path(sys.argv[2])
+    out.parent.mkdir(parents=True, exist_ok=True)
+    stats, rules = Counter(), Counter()
+    total = 0
+    with out.open("w", encoding="utf-8") as fh:
+        for rec in extract(wiki):
+            fh.write(json.dumps(rec, ensure_ascii=False) + "\n")
+            total += 1
+            stats[rec["speaker"] or "(未判定)"] += 1
+            rules[rec["rule"] or "(無)"] += 1
+    print(f"{total} 句 -> {out}")
+    print("說話者：", dict(stats.most_common()))
+    print("規則：", dict(rules.most_common()))
+
+
+if __name__ == "__main__":
+    main()

tools/corpus/fetch_wiki.py

@@ -0,0 +1,50 @@
+#!/usr/bin/env python3
+"""用 tea 的驗證讀取 Gitea wiki 全部頁面，存成 markdown。
+
+用法：fetch_wiki.py <owner/repo> <輸出目錄>
+輸出目錄放在 repo 之外；原文不入版本控制。
+"""
+import base64
+import json
+import re
+import subprocess
+import sys
+from pathlib import Path
+
+
+def tea_api(endpoint):
+    out = subprocess.run(["tea", "api", endpoint], capture_output=True, text=True, check=True).stdout
+    return json.loads(out)
+
+
+def list_pages(repo):
+    pages, page = [], 1
+    while True:
+        batch = tea_api(f"repos/{repo}/wiki/pages?page={page}&limit=50")
+        if not batch:
+            return pages
+        pages.extend(batch)
+        page += 1
+
+
+def safe_name(title):
+    return re.sub(r"[\\/:*?\"<>|]", "_", title).strip()
+
+
+def main():
+    repo, outdir = sys.argv[1], Path(sys.argv[2])
+    outdir.mkdir(parents=True, exist_ok=True)
+    pages = list_pages(repo)
+    seen = set()
+    for p in pages:
+        if p["title"] in seen:
+            continue
+        seen.add(p["title"])
+        page = tea_api(f"repos/{repo}/wiki/page/{p['sub_url']}")
+        text = base64.b64decode(page["content_base64"]).decode("utf-8")
+        (outdir / f"{safe_name(p['title'])}.md").write_text(text, encoding="utf-8")
+    print(f"{len(seen)} pages -> {outdir}")
+
+
+if __name__ == "__main__":
+    main()

tools/corpus/test_extract_dialogues.py

@@ -0,0 +1,64 @@
+import unittest
+
+from extract_dialogues import attribute, cn_number, pick_subject
+
+
+class PickSubject(unittest.TestCase):
+    def test_single_candidate_with_cue_is_high(self):
+        self.assertEqual(pick_subject("亞絲娜一邊流淚一邊對我叫道"), ("亞絲娜", ["亞絲娜"], True))
+
+    def test_object_wo_is_not_speaker(self):
+        self.assertEqual(pick_subject("在我背後的亞絲娜同樣相當氣憤地反問")[0], "亞絲娜")
+
+    def test_narrator_is_wo(self):
+        self.assertEqual(pick_subject("我低聲說道")[0], "我")
+
+    def test_first_subject_wins_and_is_not_high(self):
+        who, cands, high = pick_subject("我關掉地圖，瞄了亞絲娜一眼，說道")
+        self.assertEqual((who, cands, high), ("我", ["我", "亞絲娜"], False))
+
+    def test_frame_clause_is_skipped(self):
+        self.assertEqual(pick_subject("面對桐人這種問題，莉法先是瞪了他一下後才繼續說道")[0], "莉法")
+
+    def test_wo_men_is_not_speaker(self):
+        self.assertIsNone(pick_subject("我們一起說道")[0])
+
+    def test_no_candidate(self):
+        self.assertEqual(pick_subject("她聳了聳肩後繼續說道"), (None, [], False))
+
+
+class Attribute(unittest.TestCase):
+    def test_lead_in_same_paragraph(self):
+        r = attribute("亞絲娜小聲說：", "", "")
+        self.assertEqual((r[0], r[2], r[3]), ("asuna", "high", "lead_in"))
+
+    def test_lead_in_previous_paragraph_even_with_inner_quote(self):
+        r = attribute("", "", "他喊著「回復！」。亞絲娜小聲對我說：")
+        self.assertEqual((r[0], r[3]), ("asuna", "lead_in_prev"))
+
+    def test_narrator_maps_to_kirito(self):
+        self.assertEqual(attribute("我低聲說道：", "", "")[0], "kirito")
+
+    def test_trail(self):
+        r = attribute("", "，亞絲娜低聲說。", "")
+        self.assertEqual((r[0], r[3]), ("asuna", "trail"))
+
+    def test_trail_without_cue_is_not_attributed(self):
+        self.assertIsNone(attribute("", "亞絲娜點了點頭。", "")[0])
+
+    def test_leafa_mode_from_alias(self):
+        from extract_dialogues import resolve
+        self.assertEqual(resolve("莉法"), ("suguha", "leafa"))
+        self.assertEqual(resolve("直葉"), ("suguha", "real"))
+
+    def test_unknown_when_nothing_matches(self):
+        self.assertEqual(attribute("", "", "")[0], None)
+
+
+class Numerals(unittest.TestCase):
+    def test_chapters(self):
+        self.assertEqual([cn_number(x) for x in ("一", "十", "十一", "二十", "二十五")], [1, 10, 11, 20, 25])
+
+
+if __name__ == "__main__":
+    unittest.main()