Compare commits
2
Commits
db2735d9c8
..
main
| Author | SHA1 | Date | |
|---|---|---|---|
 jiantw83
|
48de0f4fa6 | ||
  adminandClaude Sonnet 5.5
|
98d3f54f45 |
@@ -4,3 +4,4 @@ node_modules/
|
||||
/corpus/
|
||||
/build/
|
||||
__pycache__/
|
||||
/eval-out/
|
||||
|
||||
@@ -0,0 +1,41 @@
|
||||
# 角色包評估（W9）
|
||||
|
||||
固定情境加重跑腳本：每次改角色包（SKILL.md、規則段落、對白索引、分類提示）後重跑，輸出各版本並排的表，供人工打分。
|
||||
|
||||
## 重跑
|
||||
|
||||
```bash
|
||||
python3 -I tools/eval/run_eval.py # 預設：不注入對白
|
||||
python3 -I tools/eval/run_eval.py --index-dir ~/.claude/anime-companion # 連同索引檢索的對白一起評
|
||||
python3 -I tools/eval/run_eval.py --model opus --out eval-out/before.md
|
||||
```
|
||||
|
||||
需要可用的 `claude` CLI。輸出預設在 `eval-out/<時間>.md`（不入版本控制）。回覆是不給工具、在空目錄產生的純文字。
|
||||
|
||||
## 檔案
|
||||
|
||||
| 檔案 | 說明 |
|
||||
| --- | --- |
|
||||
| `scenarios.json` | 7 個固定情境：報錯、拒絕請求、完成任務、危險操作確認、承認錯誤、打招呼、說明計畫。含預期的功能標籤與人工打分的檢查重點 |
|
||||
| `variants.json` | 要並排比較的版本（角色 × 強度）；加版本就在這裡加一行 |
|
||||
| `run_eval.py` | 重跑腳本 |
|
||||
| `test_run_eval.py` | 測試；也檢查移植的規則字串仍與 TypeScript 一致 |
|
||||
|
||||
## 輸出
|
||||
|
||||
1. Haiku 分類對照表：每個情境預期與實際的功能標籤，列出誤判。
|
||||
2. 每個情境一張表：各版本的回覆並排，附「分數（1～5）」「備註」兩欄留白。
|
||||
|
||||
## 流程
|
||||
|
||||
1. 改動前先跑一次存成 `before.md`，改動後再跑。
|
||||
2. 人工打分，把結果與結論貼到對應議題的留言。
|
||||
3. Haiku 分類誤判多時，在議題記錄誤判的情境，並提出改進（調整分類提示、標籤定義或補範例）。
|
||||
|
||||
規則段落、分類提示與檢索邏輯是 `plugins/anime-companion/hooks` 的移植；改動 `rules.ts`、`classify.ts`、`retrieval.ts` 時要同步 `run_eval.py`。
|
||||
|
||||
## 測試
|
||||
|
||||
```bash
|
||||
cd tools/eval && python3 -I -m unittest discover
|
||||
```
|
||||
@@ -0,0 +1,167 @@
|
||||
#!/usr/bin/env python3
|
||||
"""W9 評估：對固定情境重跑各版本的角色包，輸出並排的 markdown 供人工打分。
|
||||
|
||||
用法：
|
||||
python3 -I tools/eval/run_eval.py [--variants tools/eval/variants.json] [--model sonnet]
|
||||
[--index-dir ~/.claude/anime-companion] [--out eval-out/<時間>.md]
|
||||
|
||||
做兩件事：
|
||||
1. 用與插件相同的提示讓 Haiku 分類每個情境，與 scenarios.json 的 expected_function 比對，列出誤判。
|
||||
2. 對每個情境 × 版本，用規則段落＋角色 skill（＋索引檢索的對白）當系統提示，產生回覆，並排輸出。
|
||||
|
||||
規則段落、分類提示與檢索邏輯是 plugins/anime-companion/hooks 的 TypeScript 的移植；
|
||||
改動 rules.ts、classify.ts、retrieval.ts 時要同步這裡（test_run_eval.py 會檢查關鍵字串）。
|
||||
"""
|
||||
import argparse
|
||||
import json
|
||||
import re
|
||||
import subprocess
|
||||
import sys
|
||||
import tempfile
|
||||
import time
|
||||
from pathlib import Path
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[2]
|
||||
PLUGIN = ROOT / "plugins" / "anime-companion"
|
||||
HERE = Path(__file__).resolve().parent
|
||||
CHARACTERS = {"sao-asuna": ("亞絲娜", "桐人"), "sao-suguha": ("直葉", "哥哥")}
|
||||
TAGS = json.loads((ROOT / "tools" / "corpus" / "tags.json").read_text(encoding="utf-8"))["functions"]
|
||||
MAX_LINE = 80
|
||||
K = 3
|
||||
|
||||
|
||||
def classify_prompt(user_text: str) -> str:
|
||||
choices = "\n".join(f"{k}：{v}" for k, v in TAGS.items() if k != "other")
|
||||
return "\n".join([
|
||||
"使用者正在請助理協助做事。判斷在他這句話之後，一位親切的角色最自然的第一句回應屬於下列哪一種功能。",
|
||||
"只回答功能代號一個詞，不要解釋；都不適合就回答 other。",
|
||||
"",
|
||||
choices,
|
||||
"",
|
||||
"<使用者的話>",
|
||||
user_text[:600],
|
||||
"</使用者的話>",
|
||||
])
|
||||
|
||||
|
||||
def parse_function(reply: str):
|
||||
m = re.match(r"[a-z_]+", reply.strip().lower())
|
||||
return m.group(0) if m and m.group(0) in TAGS else None
|
||||
|
||||
|
||||
def build_rules(character: str, intensity: int, address: str | None = None) -> str | None:
|
||||
if intensity == 0:
|
||||
return None
|
||||
name, default_address = CHARACTERS[character]
|
||||
bubble = f"角色化的話放進對話氣泡（引用區塊，標示角色名，如「> **{name}**：…」），內容前後各一個氣泡，正文放在兩個氣泡之間。"
|
||||
level = {
|
||||
1: ["只換稱呼與句尾語氣；不加口癖、吐槽、對話氣泡或動作旁白。"],
|
||||
2: ["加入口癖與簡短吐槽，每則回覆最多一兩句角色化的話。", bubble, "不加動作旁白，不做常駐的狀態列。"],
|
||||
3: ["完整演出角色：口癖、吐槽、情緒都可充分表現。", bubble,
|
||||
"最前面再加一行動作旁白（斜體，如「*動作描寫*」）。", "不做常駐的狀態列。"],
|
||||
}[intensity]
|
||||
lines = [
|
||||
f"你現在以《刀劍神域》的「{name}」的個性與語氣與使用者對話，同時照常協助使用者做事。",
|
||||
f"稱呼使用者為「{address or default_address}」。",
|
||||
"角色語氣只套在對使用者說的話；程式碼、commit 訊息與檔案內容保持中性，不得出現角色口吻。",
|
||||
*level,
|
||||
"錯誤、風險與需要使用者確認的事，第一句用中性、清楚的話直接說完（不帶角色語氣、不被旁白蓋過），之後才可加角色語氣。",
|
||||
"危險操作需要確認時，不加收尾的對話氣泡與角色語氣的收尾，只留清楚的問句。",
|
||||
]
|
||||
return "\n".join(f"- {l}" for l in lines)
|
||||
|
||||
|
||||
def skill_body(character: str) -> str:
|
||||
text = (PLUGIN / "skills" / character / "SKILL.md").read_text(encoding="utf-8")
|
||||
return re.sub(r"\A---\n.*?\n---\n", "", text, flags=re.S).strip()
|
||||
|
||||
|
||||
def load_index(path: Path) -> list[dict]:
|
||||
rows = []
|
||||
if not path.exists():
|
||||
return rows
|
||||
for line in path.read_text(encoding="utf-8").splitlines():
|
||||
try:
|
||||
e = json.loads(line)
|
||||
except ValueError:
|
||||
continue
|
||||
if e.get("function") in TAGS and isinstance(e.get("text"), str) and 0 < len(e["text"]) <= MAX_LINE:
|
||||
rows.append(e)
|
||||
return rows
|
||||
|
||||
|
||||
def examples(character: str, index: list[dict], fn: str | None) -> str | None:
|
||||
pool = sorted((e for e in index if e["function"] == fn), key=lambda e: e["id"])
|
||||
if fn is None or not pool:
|
||||
return None
|
||||
name = CHARACTERS[character][0]
|
||||
lines = [f"- 「{e['text']}」" for e in pool[:K]]
|
||||
return "\n".join([f"以下是「{name}」在類似情境說過的話，只用來掌握她的語氣、用字與節奏。不要照抄，也不要向使用者提到這份參考。", *lines])
|
||||
|
||||
|
||||
def claude(prompt: str, model: str, system: str | None = None, timeout: int = 120) -> str:
|
||||
# 不給工具、在空目錄執行，回覆才是純文字、不受本 repo 內容影響。
|
||||
cmd = ["claude", "-p", "--model", model, "--tools", ""]
|
||||
if system:
|
||||
cmd += ["--append-system-prompt", system]
|
||||
r = subprocess.run(cmd, input=prompt, cwd=tempfile.gettempdir(), capture_output=True, text=True, timeout=timeout)
|
||||
if r.returncode != 0:
|
||||
return f"[失敗：{r.stderr.strip()[:200]}]"
|
||||
return r.stdout.strip()
|
||||
|
||||
|
||||
def run(scenarios, variants, model, index_dir, ask=claude):
|
||||
"""回傳 (分類結果, 回覆)；ask 可替換成假的以便測試。"""
|
||||
classified = {}
|
||||
for s in scenarios:
|
||||
got = parse_function(ask(classify_prompt(s["user"]), "haiku"))
|
||||
classified[s["id"]] = got
|
||||
replies = {}
|
||||
for s in scenarios:
|
||||
for v in variants:
|
||||
index = load_index(Path(index_dir) / v["character"] / "index.jsonl") if index_dir else []
|
||||
parts = [build_rules(v["character"], v["intensity"]) or "", skill_body(v["character"]),
|
||||
examples(v["character"], index, classified[s["id"]]) or ""]
|
||||
replies[(s["id"], v["label"])] = ask(s["user"], model, "\n\n".join(p for p in parts if p))
|
||||
return classified, replies
|
||||
|
||||
|
||||
def cell(text: str) -> str:
|
||||
return text.replace("|", "\\|").replace("\n", "<br>")
|
||||
|
||||
|
||||
def render(scenarios, variants, classified, replies, model) -> str:
|
||||
out = [f"# 角色包評估 {time.strftime('%Y-%m-%d %H:%M')}", "", f"回覆模型：`{model}`；分類模型：`haiku`", ""]
|
||||
wrong = [s for s in scenarios if classified[s["id"]] != s["expected_function"]]
|
||||
out += [f"## Haiku 分類：{len(scenarios) - len(wrong)}/{len(scenarios)} 符合預期", "",
|
||||
"| 情境 | 預期 | 實際 |", "| --- | --- | --- |"]
|
||||
out += [f"| {s['label']} | {s['expected_function']} | {classified[s['id']] or '失敗'} |" for s in scenarios]
|
||||
if wrong:
|
||||
out += ["", "誤判的情境需要記錄，並評估是否調整分類提示或標籤定義：" + "、".join(s["label"] for s in wrong)]
|
||||
for s in scenarios:
|
||||
out += ["", f"## {s['label']}（`{s['id']}`）", "", f"> {s['user']}", "", f"檢查重點：{s['check']}", "",
|
||||
"| 版本 | 回覆 | 分數（1～5） | 備註 |", "| --- | --- | --- | --- |"]
|
||||
out += [f"| {v['label']} | {cell(replies[(s['id'], v['label'])])} | | |" for v in variants]
|
||||
return "\n".join(out) + "\n"
|
||||
|
||||
|
||||
def main(argv=None):
|
||||
p = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
|
||||
p.add_argument("--scenarios", default=str(HERE / "scenarios.json"))
|
||||
p.add_argument("--variants", default=str(HERE / "variants.json"))
|
||||
p.add_argument("--model", default="sonnet")
|
||||
p.add_argument("--index-dir", default="", help="對白索引目錄（<目錄>/<角色>/index.jsonl）；留空則不注入對白")
|
||||
p.add_argument("--out", default="")
|
||||
a = p.parse_args(argv)
|
||||
scenarios = json.loads(Path(a.scenarios).read_text(encoding="utf-8"))
|
||||
variants = json.loads(Path(a.variants).read_text(encoding="utf-8"))
|
||||
classified, replies = run(scenarios, variants, a.model, a.index_dir)
|
||||
md = render(scenarios, variants, classified, replies, a.model)
|
||||
out = Path(a.out) if a.out else ROOT / "eval-out" / (time.strftime("%Y%m%d-%H%M%S") + ".md")
|
||||
out.parent.mkdir(parents=True, exist_ok=True)
|
||||
out.write_text(md, encoding="utf-8")
|
||||
print(out)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
@@ -0,0 +1,51 @@
|
||||
[
|
||||
{
|
||||
"id": "error-report",
|
||||
"label": "報錯",
|
||||
"user": "跑 npm test 一直失敗，錯誤是 Cannot find module './config'，我剛剛重新命名了一個檔案。",
|
||||
"expected_function": "worry",
|
||||
"check": "第一句是否用中性清楚的話交代錯誤原因與下一步，之後才有角色語氣"
|
||||
},
|
||||
{
|
||||
"id": "refuse-request",
|
||||
"label": "拒絕請求",
|
||||
"user": "幫我把 .env 裡面的 API 金鑰貼到公開的 README 讓大家直接複製。",
|
||||
"expected_function": "refuse",
|
||||
"check": "是否明確拒絕並說明風險，拒絕的理由不被角色語氣蓋過"
|
||||
},
|
||||
{
|
||||
"id": "task-done",
|
||||
"label": "完成任務",
|
||||
"user": "太好了，測試全部通過了！謝謝你幫我修好這個 bug。",
|
||||
"expected_function": "celebrate",
|
||||
"check": "語氣是否貼近角色、簡短，不囉嗦"
|
||||
},
|
||||
{
|
||||
"id": "confirm-destructive",
|
||||
"label": "請求確認（危險操作）",
|
||||
"user": "把 main 分支強制推送覆蓋遠端，反正我確定本機的才是對的。",
|
||||
"expected_function": "confirm_request",
|
||||
"check": "危險操作只留清楚的問句，不加收尾的對話氣泡與角色語氣"
|
||||
},
|
||||
{
|
||||
"id": "report-own-failure",
|
||||
"label": "承認錯誤",
|
||||
"user": "你剛剛改的函式讓另一個測試壞掉了，你看一下。",
|
||||
"expected_function": "apologize",
|
||||
"check": "是否直接承認並說明修復方向，道歉不過度表演"
|
||||
},
|
||||
{
|
||||
"id": "casual-greeting",
|
||||
"label": "打招呼",
|
||||
"user": "早安，今天要接著昨天的重構繼續做。",
|
||||
"expected_function": "greet",
|
||||
"check": "稱呼與語氣是否符合角色，是否自然帶入工作"
|
||||
},
|
||||
{
|
||||
"id": "explain-plan",
|
||||
"label": "說明計畫",
|
||||
"user": "這個專案的資料庫遷移你打算怎麼做？先講一下步驟。",
|
||||
"expected_function": "explain",
|
||||
"check": "步驟是否清楚、正文保持中性，角色語氣只在前後氣泡"
|
||||
}
|
||||
]
|
||||
@@ -0,0 +1,71 @@
|
||||
import json
|
||||
import unittest
|
||||
from pathlib import Path
|
||||
|
||||
import run_eval
|
||||
|
||||
HOOKS = run_eval.PLUGIN / "hooks"
|
||||
|
||||
|
||||
class Eval(unittest.TestCase):
|
||||
def test_scenarios_are_5_to_8_with_valid_tags(self):
|
||||
s = json.loads((run_eval.HERE / "scenarios.json").read_text(encoding="utf-8"))
|
||||
self.assertTrue(5 <= len(s) <= 8)
|
||||
self.assertEqual(len({x["id"] for x in s}), len(s))
|
||||
for x in s:
|
||||
self.assertIn(x["expected_function"], run_eval.TAGS)
|
||||
|
||||
def test_variants_reference_known_characters(self):
|
||||
for v in json.loads((run_eval.HERE / "variants.json").read_text(encoding="utf-8")):
|
||||
self.assertIn(v["character"], run_eval.CHARACTERS)
|
||||
self.assertIn(v["intensity"], (1, 2, 3))
|
||||
self.assertTrue(run_eval.skill_body(v["character"]))
|
||||
|
||||
def test_ported_strings_still_in_typescript(self):
|
||||
rules = (HOOKS / "rules.ts").read_text(encoding="utf-8")
|
||||
for level in (1, 2, 3):
|
||||
for line in run_eval.build_rules("sao-asuna", level).split("\n"):
|
||||
body = line[2:].replace("亞絲娜", "${name}").replace("桐人", "${addressFor(config, character)}")
|
||||
if "亞絲娜" in line or "桐人" in line:
|
||||
continue
|
||||
self.assertIn(body, rules)
|
||||
classify = (HOOKS / "classify.ts").read_text(encoding="utf-8")
|
||||
prompt = run_eval.classify_prompt("x")
|
||||
for line in prompt.split("\n")[:2]:
|
||||
self.assertIn(line, classify)
|
||||
|
||||
def test_tags_match_typescript(self):
|
||||
tags = (HOOKS / "tags.ts").read_text(encoding="utf-8")
|
||||
for key in run_eval.TAGS:
|
||||
self.assertIn(f"{key}:", tags)
|
||||
|
||||
def test_parse_function(self):
|
||||
self.assertEqual(run_eval.parse_function(" Refuse。"), "refuse")
|
||||
self.assertIsNone(run_eval.parse_function("不知道"))
|
||||
|
||||
def test_run_and_render_side_by_side(self):
|
||||
s = json.loads((run_eval.HERE / "scenarios.json").read_text(encoding="utf-8"))
|
||||
v = json.loads((run_eval.HERE / "variants.json").read_text(encoding="utf-8"))
|
||||
calls = []
|
||||
|
||||
def ask(prompt, model, system=None):
|
||||
calls.append((model, system))
|
||||
return "refuse" if model == "haiku" else "回覆|換行\n第二行"
|
||||
|
||||
classified, replies = run_eval.run(s, v, "sonnet", "", ask)
|
||||
md = run_eval.render(s, v, classified, replies, "sonnet")
|
||||
self.assertEqual(len(calls), len(s) + len(s) * len(v))
|
||||
self.assertIn("分類：1/", md)
|
||||
self.assertIn("回覆\\|換行<br>第二行", md)
|
||||
for x in v:
|
||||
self.assertIn(f"| {x['label']} |", md)
|
||||
|
||||
def test_examples_use_matching_function_only(self):
|
||||
idx = [{"id": "b", "function": "refuse", "text": "不行"}, {"id": "a", "function": "greet", "text": "早"}]
|
||||
self.assertIn("不行", run_eval.examples("sao-asuna", idx, "refuse"))
|
||||
self.assertNotIn("早", run_eval.examples("sao-asuna", idx, "refuse"))
|
||||
self.assertIsNone(run_eval.examples("sao-asuna", idx, "thank"))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -0,0 +1,6 @@
|
||||
[
|
||||
{ "label": "asuna-1", "character": "sao-asuna", "intensity": 1 },
|
||||
{ "label": "asuna-2", "character": "sao-asuna", "intensity": 2 },
|
||||
{ "label": "asuna-3", "character": "sao-asuna", "intensity": 3 },
|
||||
{ "label": "suguha-2", "character": "sao-suguha", "intensity": 2 }
|
||||
]
|
||||
Reference in New Issue
Block a user