feat: 角色包評估的固定情境與重跑腳本 (#10)

Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
2026-10-15 02:23:56 +00:00
parent db2735d9c8
commit 98d3f54f45
6 changed files with 337 additions and 0 deletions
@@ -0,0 +1,71 @@
+import json
+import unittest
+from pathlib import Path
+
+import run_eval
+
+HOOKS = run_eval.PLUGIN / "hooks"
+
+
+class Eval(unittest.TestCase):
+    def test_scenarios_are_5_to_8_with_valid_tags(self):
+        s = json.loads((run_eval.HERE / "scenarios.json").read_text(encoding="utf-8"))
+        self.assertTrue(5 <= len(s) <= 8)
+        self.assertEqual(len({x["id"] for x in s}), len(s))
+        for x in s:
+            self.assertIn(x["expected_function"], run_eval.TAGS)
+
+    def test_variants_reference_known_characters(self):
+        for v in json.loads((run_eval.HERE / "variants.json").read_text(encoding="utf-8")):
+            self.assertIn(v["character"], run_eval.CHARACTERS)
+            self.assertIn(v["intensity"], (1, 2, 3))
+            self.assertTrue(run_eval.skill_body(v["character"]))
+
+    def test_ported_strings_still_in_typescript(self):
+        rules = (HOOKS / "rules.ts").read_text(encoding="utf-8")
+        for level in (1, 2, 3):
+            for line in run_eval.build_rules("sao-asuna", level).split("\n"):
+                body = line[2:].replace("亞絲娜", "${name}").replace("桐人", "${addressFor(config, character)}")
+                if "亞絲娜" in line or "桐人" in line:
+                    continue
+                self.assertIn(body, rules)
+        classify = (HOOKS / "classify.ts").read_text(encoding="utf-8")
+        prompt = run_eval.classify_prompt("x")
+        for line in prompt.split("\n")[:2]:
+            self.assertIn(line, classify)
+
+    def test_tags_match_typescript(self):
+        tags = (HOOKS / "tags.ts").read_text(encoding="utf-8")
+        for key in run_eval.TAGS:
+            self.assertIn(f"{key}:", tags)
+
+    def test_parse_function(self):
+        self.assertEqual(run_eval.parse_function(" Refuse。"), "refuse")
+        self.assertIsNone(run_eval.parse_function("不知道"))
+
+    def test_run_and_render_side_by_side(self):
+        s = json.loads((run_eval.HERE / "scenarios.json").read_text(encoding="utf-8"))
+        v = json.loads((run_eval.HERE / "variants.json").read_text(encoding="utf-8"))
+        calls = []
+
+        def ask(prompt, model, system=None):
+            calls.append((model, system))
+            return "refuse" if model == "haiku" else "回覆|換行\n第二行"
+
+        classified, replies = run_eval.run(s, v, "sonnet", "", ask)
+        md = run_eval.render(s, v, classified, replies, "sonnet")
+        self.assertEqual(len(calls), len(s) + len(s) * len(v))
+        self.assertIn("分類：1/", md)
+        self.assertIn("回覆\\|換行<br>第二行", md)
+        for x in v:
+            self.assertIn(f"| {x['label']} |", md)
+
+    def test_examples_use_matching_function_only(self):
+        idx = [{"id": "b", "function": "refuse", "text": "不行"}, {"id": "a", "function": "greet", "text": "早"}]
+        self.assertIn("不行", run_eval.examples("sao-asuna", idx, "refuse"))
+        self.assertNotIn("早", run_eval.examples("sao-asuna", idx, "refuse"))
+        self.assertIsNone(run_eval.examples("sao-asuna", idx, "thank"))
+
+
+if __name__ == "__main__":
+    unittest.main()