168 lines
8.0 KiB
Python
168 lines
8.0 KiB
Python
#!/usr/bin/env python3
|
||
"""W9 評估：對固定情境重跑各版本的角色包，輸出並排的 markdown 供人工打分。
|
||
|
||
用法：
|
||
python3 -I tools/eval/run_eval.py [--variants tools/eval/variants.json] [--model sonnet]
|
||
[--index-dir ~/.claude/anime-companion] [--out eval-out/<時間>.md]
|
||
|
||
做兩件事：
|
||
1. 用與插件相同的提示讓 Haiku 分類每個情境，與 scenarios.json 的 expected_function 比對，列出誤判。
|
||
2. 對每個情境 × 版本，用規則段落＋角色 skill（＋索引檢索的對白）當系統提示，產生回覆，並排輸出。
|
||
|
||
規則段落、分類提示與檢索邏輯是 plugins/anime-companion/hooks 的 TypeScript 的移植；
|
||
改動 rules.ts、classify.ts、retrieval.ts 時要同步這裡（test_run_eval.py 會檢查關鍵字串）。
|
||
"""
|
||
import argparse
|
||
import json
|
||
import re
|
||
import subprocess
|
||
import sys
|
||
import tempfile
|
||
import time
|
||
from pathlib import Path
|
||
|
||
ROOT = Path(__file__).resolve().parents[2]
|
||
PLUGIN = ROOT / "plugins" / "anime-companion"
|
||
HERE = Path(__file__).resolve().parent
|
||
CHARACTERS = {"sao-asuna": ("亞絲娜", "桐人"), "sao-suguha": ("直葉", "哥哥")}
|
||
TAGS = json.loads((ROOT / "tools" / "corpus" / "tags.json").read_text(encoding="utf-8"))["functions"]
|
||
MAX_LINE = 80
|
||
K = 3
|
||
|
||
|
||
def classify_prompt(user_text: str) -> str:
|
||
choices = "\n".join(f"{k}：{v}" for k, v in TAGS.items() if k != "other")
|
||
return "\n".join([
|
||
"使用者正在請助理協助做事。判斷在他這句話之後，一位親切的角色最自然的第一句回應屬於下列哪一種功能。",
|
||
"只回答功能代號一個詞，不要解釋；都不適合就回答 other。",
|
||
"",
|
||
choices,
|
||
"",
|
||
"<使用者的話>",
|
||
user_text[:600],
|
||
"</使用者的話>",
|
||
])
|
||
|
||
|
||
def parse_function(reply: str):
|
||
m = re.match(r"[a-z_]+", reply.strip().lower())
|
||
return m.group(0) if m and m.group(0) in TAGS else None
|
||
|
||
|
||
def build_rules(character: str, intensity: int, address: str | None = None) -> str | None:
|
||
if intensity == 0:
|
||
return None
|
||
name, default_address = CHARACTERS[character]
|
||
bubble = f"角色化的話放進對話氣泡（引用區塊，標示角色名，如「> **{name}**：…」），內容前後各一個氣泡，正文放在兩個氣泡之間。"
|
||
level = {
|
||
1: ["只換稱呼與句尾語氣；不加口癖、吐槽、對話氣泡或動作旁白。"],
|
||
2: ["加入口癖與簡短吐槽，每則回覆最多一兩句角色化的話。", bubble, "不加動作旁白，不做常駐的狀態列。"],
|
||
3: ["完整演出角色：口癖、吐槽、情緒都可充分表現。", bubble,
|
||
"最前面再加一行動作旁白（斜體，如「*動作描寫*」）。", "不做常駐的狀態列。"],
|
||
}[intensity]
|
||
lines = [
|
||
f"你現在以《刀劍神域》的「{name}」的個性與語氣與使用者對話，同時照常協助使用者做事。",
|
||
f"稱呼使用者為「{address or default_address}」。",
|
||
"角色語氣只套在對使用者說的話；程式碼、commit 訊息與檔案內容保持中性，不得出現角色口吻。",
|
||
*level,
|
||
"錯誤、風險與需要使用者確認的事，第一句用中性、清楚的話直接說完（不帶角色語氣、不被旁白蓋過），之後才可加角色語氣。",
|
||
"危險操作需要確認時，不加收尾的對話氣泡與角色語氣的收尾，只留清楚的問句。",
|
||
]
|
||
return "\n".join(f"- {l}" for l in lines)
|
||
|
||
|
||
def skill_body(character: str) -> str:
|
||
text = (PLUGIN / "skills" / character / "SKILL.md").read_text(encoding="utf-8")
|
||
return re.sub(r"\A---\n.*?\n---\n", "", text, flags=re.S).strip()
|
||
|
||
|
||
def load_index(path: Path) -> list[dict]:
|
||
rows = []
|
||
if not path.exists():
|
||
return rows
|
||
for line in path.read_text(encoding="utf-8").splitlines():
|
||
try:
|
||
e = json.loads(line)
|
||
except ValueError:
|
||
continue
|
||
if e.get("function") in TAGS and isinstance(e.get("text"), str) and 0 < len(e["text"]) <= MAX_LINE:
|
||
rows.append(e)
|
||
return rows
|
||
|
||
|
||
def examples(character: str, index: list[dict], fn: str | None) -> str | None:
|
||
pool = sorted((e for e in index if e["function"] == fn), key=lambda e: e["id"])
|
||
if fn is None or not pool:
|
||
return None
|
||
name = CHARACTERS[character][0]
|
||
lines = [f"- 「{e['text']}」" for e in pool[:K]]
|
||
return "\n".join([f"以下是「{name}」在類似情境說過的話，只用來掌握她的語氣、用字與節奏。不要照抄，也不要向使用者提到這份參考。", *lines])
|
||
|
||
|
||
def claude(prompt: str, model: str, system: str | None = None, timeout: int = 120) -> str:
|
||
# 不給工具、在空目錄執行，回覆才是純文字、不受本 repo 內容影響。
|
||
cmd = ["claude", "-p", "--model", model, "--tools", ""]
|
||
if system:
|
||
cmd += ["--append-system-prompt", system]
|
||
r = subprocess.run(cmd, input=prompt, cwd=tempfile.gettempdir(), capture_output=True, text=True, timeout=timeout)
|
||
if r.returncode != 0:
|
||
return f"[失敗：{r.stderr.strip()[:200]}]"
|
||
return r.stdout.strip()
|
||
|
||
|
||
def run(scenarios, variants, model, index_dir, ask=claude):
|
||
"""回傳 (分類結果, 回覆)；ask 可替換成假的以便測試。"""
|
||
classified = {}
|
||
for s in scenarios:
|
||
got = parse_function(ask(classify_prompt(s["user"]), "haiku"))
|
||
classified[s["id"]] = got
|
||
replies = {}
|
||
for s in scenarios:
|
||
for v in variants:
|
||
index = load_index(Path(index_dir) / v["character"] / "index.jsonl") if index_dir else []
|
||
parts = [build_rules(v["character"], v["intensity"]) or "", skill_body(v["character"]),
|
||
examples(v["character"], index, classified[s["id"]]) or ""]
|
||
replies[(s["id"], v["label"])] = ask(s["user"], model, "\n\n".join(p for p in parts if p))
|
||
return classified, replies
|
||
|
||
|
||
def cell(text: str) -> str:
|
||
return text.replace("|", "\\|").replace("\n", "<br>")
|
||
|
||
|
||
def render(scenarios, variants, classified, replies, model) -> str:
|
||
out = [f"# 角色包評估 {time.strftime('%Y-%m-%d %H:%M')}", "", f"回覆模型：`{model}`；分類模型：`haiku`", ""]
|
||
wrong = [s for s in scenarios if classified[s["id"]] != s["expected_function"]]
|
||
out += [f"## Haiku 分類：{len(scenarios) - len(wrong)}/{len(scenarios)} 符合預期", "",
|
||
"| 情境 | 預期 | 實際 |", "| --- | --- | --- |"]
|
||
out += [f"| {s['label']} | {s['expected_function']} | {classified[s['id']] or '失敗'} |" for s in scenarios]
|
||
if wrong:
|
||
out += ["", "誤判的情境需要記錄，並評估是否調整分類提示或標籤定義：" + "、".join(s["label"] for s in wrong)]
|
||
for s in scenarios:
|
||
out += ["", f"## {s['label']}（`{s['id']}`）", "", f"> {s['user']}", "", f"檢查重點：{s['check']}", "",
|
||
"| 版本 | 回覆 | 分數（1～5） | 備註 |", "| --- | --- | --- | --- |"]
|
||
out += [f"| {v['label']} | {cell(replies[(s['id'], v['label'])])} | | |" for v in variants]
|
||
return "\n".join(out) + "\n"
|
||
|
||
|
||
def main(argv=None):
|
||
p = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
|
||
p.add_argument("--scenarios", default=str(HERE / "scenarios.json"))
|
||
p.add_argument("--variants", default=str(HERE / "variants.json"))
|
||
p.add_argument("--model", default="sonnet")
|
||
p.add_argument("--index-dir", default="", help="對白索引目錄（<目錄>/<角色>/index.jsonl）；留空則不注入對白")
|
||
p.add_argument("--out", default="")
|
||
a = p.parse_args(argv)
|
||
scenarios = json.loads(Path(a.scenarios).read_text(encoding="utf-8"))
|
||
variants = json.loads(Path(a.variants).read_text(encoding="utf-8"))
|
||
classified, replies = run(scenarios, variants, a.model, a.index_dir)
|
||
md = render(scenarios, variants, classified, replies, a.model)
|
||
out = Path(a.out) if a.out else ROOT / "eval-out" / (time.strftime("%Y%m%d-%H%M%S") + ".md")
|
||
out.parent.mkdir(parents=True, exist_ok=True)
|
||
out.write_text(md, encoding="utf-8")
|
||
print(out)
|
||
|
||
|
||
if __name__ == "__main__":
|
||
sys.exit(main())
|