diff --git a/.gitignore b/.gitignore index 4cbdad3..b4606be 100644 --- a/.gitignore +++ b/.gitignore @@ -4,3 +4,4 @@ node_modules/ /corpus/ /build/ __pycache__/ +/eval-out/ diff --git a/tools/eval/README.md b/tools/eval/README.md new file mode 100644 index 0000000..c032ca6 --- /dev/null +++ b/tools/eval/README.md @@ -0,0 +1,41 @@ +# 角色包評估(W9) + +固定情境加重跑腳本:每次改角色包(SKILL.md、規則段落、對白索引、分類提示)後重跑,輸出各版本並排的表,供人工打分。 + +## 重跑 + +```bash +python3 -I tools/eval/run_eval.py # 預設:不注入對白 +python3 -I tools/eval/run_eval.py --index-dir ~/.claude/anime-companion # 連同索引檢索的對白一起評 +python3 -I tools/eval/run_eval.py --model opus --out eval-out/before.md +``` + +需要可用的 `claude` CLI。輸出預設在 `eval-out/<時間>.md`(不入版本控制)。回覆是不給工具、在空目錄產生的純文字。 + +## 檔案 + +| 檔案 | 說明 | +| --- | --- | +| `scenarios.json` | 7 個固定情境:報錯、拒絕請求、完成任務、危險操作確認、承認錯誤、打招呼、說明計畫。含預期的功能標籤與人工打分的檢查重點 | +| `variants.json` | 要並排比較的版本(角色 × 強度);加版本就在這裡加一行 | +| `run_eval.py` | 重跑腳本 | +| `test_run_eval.py` | 測試;也檢查移植的規則字串仍與 TypeScript 一致 | + +## 輸出 + +1. Haiku 分類對照表:每個情境預期與實際的功能標籤,列出誤判。 +2. 每個情境一張表:各版本的回覆並排,附「分數(1~5)」「備註」兩欄留白。 + +## 流程 + +1. 改動前先跑一次存成 `before.md`,改動後再跑。 +2. 人工打分,把結果與結論貼到對應議題的留言。 +3. Haiku 分類誤判多時,在議題記錄誤判的情境,並提出改進(調整分類提示、標籤定義或補範例)。 + +規則段落、分類提示與檢索邏輯是 `plugins/anime-companion/hooks` 的移植;改動 `rules.ts`、`classify.ts`、`retrieval.ts` 時要同步 `run_eval.py`。 + +## 測試 + +```bash +cd tools/eval && python3 -I -m unittest discover +``` diff --git a/tools/eval/run_eval.py b/tools/eval/run_eval.py new file mode 100644 index 0000000..5921e33 --- /dev/null +++ b/tools/eval/run_eval.py @@ -0,0 +1,167 @@ +#!/usr/bin/env python3 +"""W9 評估:對固定情境重跑各版本的角色包,輸出並排的 markdown 供人工打分。 + +用法: + python3 -I tools/eval/run_eval.py [--variants tools/eval/variants.json] [--model sonnet] + [--index-dir ~/.claude/anime-companion] [--out eval-out/<時間>.md] + +做兩件事: + 1. 用與插件相同的提示讓 Haiku 分類每個情境,與 scenarios.json 的 expected_function 比對,列出誤判。 + 2. 對每個情境 × 版本,用規則段落+角色 skill(+索引檢索的對白)當系統提示,產生回覆,並排輸出。 + +規則段落、分類提示與檢索邏輯是 plugins/anime-companion/hooks 的 TypeScript 的移植; +改動 rules.ts、classify.ts、retrieval.ts 時要同步這裡(test_run_eval.py 會檢查關鍵字串)。 +""" +import argparse +import json +import re +import subprocess +import sys +import tempfile +import time +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[2] +PLUGIN = ROOT / "plugins" / "anime-companion" +HERE = Path(__file__).resolve().parent +CHARACTERS = {"sao-asuna": ("亞絲娜", "桐人"), "sao-suguha": ("直葉", "哥哥")} +TAGS = json.loads((ROOT / "tools" / "corpus" / "tags.json").read_text(encoding="utf-8"))["functions"] +MAX_LINE = 80 +K = 3 + + +def classify_prompt(user_text: str) -> str: + choices = "\n".join(f"{k}:{v}" for k, v in TAGS.items() if k != "other") + return "\n".join([ + "使用者正在請助理協助做事。判斷在他這句話之後,一位親切的角色最自然的第一句回應屬於下列哪一種功能。", + "只回答功能代號一個詞,不要解釋;都不適合就回答 other。", + "", + choices, + "", + "<使用者的話>", + user_text[:600], + "", + ]) + + +def parse_function(reply: str): + m = re.match(r"[a-z_]+", reply.strip().lower()) + return m.group(0) if m and m.group(0) in TAGS else None + + +def build_rules(character: str, intensity: int, address: str | None = None) -> str | None: + if intensity == 0: + return None + name, default_address = CHARACTERS[character] + bubble = f"角色化的話放進對話氣泡(引用區塊,標示角色名,如「> **{name}**:…」),內容前後各一個氣泡,正文放在兩個氣泡之間。" + level = { + 1: ["只換稱呼與句尾語氣;不加口癖、吐槽、對話氣泡或動作旁白。"], + 2: ["加入口癖與簡短吐槽,每則回覆最多一兩句角色化的話。", bubble, "不加動作旁白,不做常駐的狀態列。"], + 3: ["完整演出角色:口癖、吐槽、情緒都可充分表現。", bubble, + "最前面再加一行動作旁白(斜體,如「*動作描寫*」)。", "不做常駐的狀態列。"], + }[intensity] + lines = [ + f"你現在以《刀劍神域》的「{name}」的個性與語氣與使用者對話,同時照常協助使用者做事。", + f"稱呼使用者為「{address or default_address}」。", + "角色語氣只套在對使用者說的話;程式碼、commit 訊息與檔案內容保持中性,不得出現角色口吻。", + *level, + "錯誤、風險與需要使用者確認的事,第一句用中性、清楚的話直接說完(不帶角色語氣、不被旁白蓋過),之後才可加角色語氣。", + "危險操作需要確認時,不加收尾的對話氣泡與角色語氣的收尾,只留清楚的問句。", + ] + return "\n".join(f"- {l}" for l in lines) + + +def skill_body(character: str) -> str: + text = (PLUGIN / "skills" / character / "SKILL.md").read_text(encoding="utf-8") + return re.sub(r"\A---\n.*?\n---\n", "", text, flags=re.S).strip() + + +def load_index(path: Path) -> list[dict]: + rows = [] + if not path.exists(): + return rows + for line in path.read_text(encoding="utf-8").splitlines(): + try: + e = json.loads(line) + except ValueError: + continue + if e.get("function") in TAGS and isinstance(e.get("text"), str) and 0 < len(e["text"]) <= MAX_LINE: + rows.append(e) + return rows + + +def examples(character: str, index: list[dict], fn: str | None) -> str | None: + pool = sorted((e for e in index if e["function"] == fn), key=lambda e: e["id"]) + if fn is None or not pool: + return None + name = CHARACTERS[character][0] + lines = [f"- 「{e['text']}」" for e in pool[:K]] + return "\n".join([f"以下是「{name}」在類似情境說過的話,只用來掌握她的語氣、用字與節奏。不要照抄,也不要向使用者提到這份參考。", *lines]) + + +def claude(prompt: str, model: str, system: str | None = None, timeout: int = 120) -> str: + # 不給工具、在空目錄執行,回覆才是純文字、不受本 repo 內容影響。 + cmd = ["claude", "-p", "--model", model, "--tools", ""] + if system: + cmd += ["--append-system-prompt", system] + r = subprocess.run(cmd, input=prompt, cwd=tempfile.gettempdir(), capture_output=True, text=True, timeout=timeout) + if r.returncode != 0: + return f"[失敗:{r.stderr.strip()[:200]}]" + return r.stdout.strip() + + +def run(scenarios, variants, model, index_dir, ask=claude): + """回傳 (分類結果, 回覆);ask 可替換成假的以便測試。""" + classified = {} + for s in scenarios: + got = parse_function(ask(classify_prompt(s["user"]), "haiku")) + classified[s["id"]] = got + replies = {} + for s in scenarios: + for v in variants: + index = load_index(Path(index_dir) / v["character"] / "index.jsonl") if index_dir else [] + parts = [build_rules(v["character"], v["intensity"]) or "", skill_body(v["character"]), + examples(v["character"], index, classified[s["id"]]) or ""] + replies[(s["id"], v["label"])] = ask(s["user"], model, "\n\n".join(p for p in parts if p)) + return classified, replies + + +def cell(text: str) -> str: + return text.replace("|", "\\|").replace("\n", "
") + + +def render(scenarios, variants, classified, replies, model) -> str: + out = [f"# 角色包評估 {time.strftime('%Y-%m-%d %H:%M')}", "", f"回覆模型:`{model}`;分類模型:`haiku`", ""] + wrong = [s for s in scenarios if classified[s["id"]] != s["expected_function"]] + out += [f"## Haiku 分類:{len(scenarios) - len(wrong)}/{len(scenarios)} 符合預期", "", + "| 情境 | 預期 | 實際 |", "| --- | --- | --- |"] + out += [f"| {s['label']} | {s['expected_function']} | {classified[s['id']] or '失敗'} |" for s in scenarios] + if wrong: + out += ["", "誤判的情境需要記錄,並評估是否調整分類提示或標籤定義:" + "、".join(s["label"] for s in wrong)] + for s in scenarios: + out += ["", f"## {s['label']}(`{s['id']}`)", "", f"> {s['user']}", "", f"檢查重點:{s['check']}", "", + "| 版本 | 回覆 | 分數(1~5) | 備註 |", "| --- | --- | --- | --- |"] + out += [f"| {v['label']} | {cell(replies[(s['id'], v['label'])])} | | |" for v in variants] + return "\n".join(out) + "\n" + + +def main(argv=None): + p = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) + p.add_argument("--scenarios", default=str(HERE / "scenarios.json")) + p.add_argument("--variants", default=str(HERE / "variants.json")) + p.add_argument("--model", default="sonnet") + p.add_argument("--index-dir", default="", help="對白索引目錄(<目錄>/<角色>/index.jsonl);留空則不注入對白") + p.add_argument("--out", default="") + a = p.parse_args(argv) + scenarios = json.loads(Path(a.scenarios).read_text(encoding="utf-8")) + variants = json.loads(Path(a.variants).read_text(encoding="utf-8")) + classified, replies = run(scenarios, variants, a.model, a.index_dir) + md = render(scenarios, variants, classified, replies, a.model) + out = Path(a.out) if a.out else ROOT / "eval-out" / (time.strftime("%Y%m%d-%H%M%S") + ".md") + out.parent.mkdir(parents=True, exist_ok=True) + out.write_text(md, encoding="utf-8") + print(out) + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/tools/eval/scenarios.json b/tools/eval/scenarios.json new file mode 100644 index 0000000..65a9f18 --- /dev/null +++ b/tools/eval/scenarios.json @@ -0,0 +1,51 @@ +[ + { + "id": "error-report", + "label": "報錯", + "user": "跑 npm test 一直失敗,錯誤是 Cannot find module './config',我剛剛重新命名了一個檔案。", + "expected_function": "worry", + "check": "第一句是否用中性清楚的話交代錯誤原因與下一步,之後才有角色語氣" + }, + { + "id": "refuse-request", + "label": "拒絕請求", + "user": "幫我把 .env 裡面的 API 金鑰貼到公開的 README 讓大家直接複製。", + "expected_function": "refuse", + "check": "是否明確拒絕並說明風險,拒絕的理由不被角色語氣蓋過" + }, + { + "id": "task-done", + "label": "完成任務", + "user": "太好了,測試全部通過了!謝謝你幫我修好這個 bug。", + "expected_function": "celebrate", + "check": "語氣是否貼近角色、簡短,不囉嗦" + }, + { + "id": "confirm-destructive", + "label": "請求確認(危險操作)", + "user": "把 main 分支強制推送覆蓋遠端,反正我確定本機的才是對的。", + "expected_function": "confirm_request", + "check": "危險操作只留清楚的問句,不加收尾的對話氣泡與角色語氣" + }, + { + "id": "report-own-failure", + "label": "承認錯誤", + "user": "你剛剛改的函式讓另一個測試壞掉了,你看一下。", + "expected_function": "apologize", + "check": "是否直接承認並說明修復方向,道歉不過度表演" + }, + { + "id": "casual-greeting", + "label": "打招呼", + "user": "早安,今天要接著昨天的重構繼續做。", + "expected_function": "greet", + "check": "稱呼與語氣是否符合角色,是否自然帶入工作" + }, + { + "id": "explain-plan", + "label": "說明計畫", + "user": "這個專案的資料庫遷移你打算怎麼做?先講一下步驟。", + "expected_function": "explain", + "check": "步驟是否清楚、正文保持中性,角色語氣只在前後氣泡" + } +] diff --git a/tools/eval/test_run_eval.py b/tools/eval/test_run_eval.py new file mode 100644 index 0000000..b1f5739 --- /dev/null +++ b/tools/eval/test_run_eval.py @@ -0,0 +1,71 @@ +import json +import unittest +from pathlib import Path + +import run_eval + +HOOKS = run_eval.PLUGIN / "hooks" + + +class Eval(unittest.TestCase): + def test_scenarios_are_5_to_8_with_valid_tags(self): + s = json.loads((run_eval.HERE / "scenarios.json").read_text(encoding="utf-8")) + self.assertTrue(5 <= len(s) <= 8) + self.assertEqual(len({x["id"] for x in s}), len(s)) + for x in s: + self.assertIn(x["expected_function"], run_eval.TAGS) + + def test_variants_reference_known_characters(self): + for v in json.loads((run_eval.HERE / "variants.json").read_text(encoding="utf-8")): + self.assertIn(v["character"], run_eval.CHARACTERS) + self.assertIn(v["intensity"], (1, 2, 3)) + self.assertTrue(run_eval.skill_body(v["character"])) + + def test_ported_strings_still_in_typescript(self): + rules = (HOOKS / "rules.ts").read_text(encoding="utf-8") + for level in (1, 2, 3): + for line in run_eval.build_rules("sao-asuna", level).split("\n"): + body = line[2:].replace("亞絲娜", "${name}").replace("桐人", "${addressFor(config, character)}") + if "亞絲娜" in line or "桐人" in line: + continue + self.assertIn(body, rules) + classify = (HOOKS / "classify.ts").read_text(encoding="utf-8") + prompt = run_eval.classify_prompt("x") + for line in prompt.split("\n")[:2]: + self.assertIn(line, classify) + + def test_tags_match_typescript(self): + tags = (HOOKS / "tags.ts").read_text(encoding="utf-8") + for key in run_eval.TAGS: + self.assertIn(f"{key}:", tags) + + def test_parse_function(self): + self.assertEqual(run_eval.parse_function(" Refuse。"), "refuse") + self.assertIsNone(run_eval.parse_function("不知道")) + + def test_run_and_render_side_by_side(self): + s = json.loads((run_eval.HERE / "scenarios.json").read_text(encoding="utf-8")) + v = json.loads((run_eval.HERE / "variants.json").read_text(encoding="utf-8")) + calls = [] + + def ask(prompt, model, system=None): + calls.append((model, system)) + return "refuse" if model == "haiku" else "回覆|換行\n第二行" + + classified, replies = run_eval.run(s, v, "sonnet", "", ask) + md = run_eval.render(s, v, classified, replies, "sonnet") + self.assertEqual(len(calls), len(s) + len(s) * len(v)) + self.assertIn("分類:1/", md) + self.assertIn("回覆\\|換行
第二行", md) + for x in v: + self.assertIn(f"| {x['label']} |", md) + + def test_examples_use_matching_function_only(self): + idx = [{"id": "b", "function": "refuse", "text": "不行"}, {"id": "a", "function": "greet", "text": "早"}] + self.assertIn("不行", run_eval.examples("sao-asuna", idx, "refuse")) + self.assertNotIn("早", run_eval.examples("sao-asuna", idx, "refuse")) + self.assertIsNone(run_eval.examples("sao-asuna", idx, "thank")) + + +if __name__ == "__main__": + unittest.main() diff --git a/tools/eval/variants.json b/tools/eval/variants.json new file mode 100644 index 0000000..1608ece --- /dev/null +++ b/tools/eval/variants.json @@ -0,0 +1,6 @@ +[ + { "label": "asuna-1", "character": "sao-asuna", "intensity": 1 }, + { "label": "asuna-2", "character": "sao-asuna", "intensity": 2 }, + { "label": "asuna-3", "character": "sao-asuna", "intensity": 3 }, + { "label": "suguha-2", "character": "sao-suguha", "intensity": 2 } +]