72 lines
3.0 KiB
Python
72 lines
3.0 KiB
Python
import json
|
|
import unittest
|
|
from pathlib import Path
|
|
|
|
import run_eval
|
|
|
|
HOOKS = run_eval.PLUGIN / "hooks"
|
|
|
|
|
|
class Eval(unittest.TestCase):
|
|
def test_scenarios_are_5_to_8_with_valid_tags(self):
|
|
s = json.loads((run_eval.HERE / "scenarios.json").read_text(encoding="utf-8"))
|
|
self.assertTrue(5 <= len(s) <= 8)
|
|
self.assertEqual(len({x["id"] for x in s}), len(s))
|
|
for x in s:
|
|
self.assertIn(x["expected_function"], run_eval.TAGS)
|
|
|
|
def test_variants_reference_known_characters(self):
|
|
for v in json.loads((run_eval.HERE / "variants.json").read_text(encoding="utf-8")):
|
|
self.assertIn(v["character"], run_eval.CHARACTERS)
|
|
self.assertIn(v["intensity"], (1, 2, 3))
|
|
self.assertTrue(run_eval.skill_body(v["character"]))
|
|
|
|
def test_ported_strings_still_in_typescript(self):
|
|
rules = (HOOKS / "rules.ts").read_text(encoding="utf-8")
|
|
for level in (1, 2, 3):
|
|
for line in run_eval.build_rules("sao-asuna", level).split("\n"):
|
|
body = line[2:].replace("亞絲娜", "${name}").replace("桐人", "${addressFor(config, character)}")
|
|
if "亞絲娜" in line or "桐人" in line:
|
|
continue
|
|
self.assertIn(body, rules)
|
|
classify = (HOOKS / "classify.ts").read_text(encoding="utf-8")
|
|
prompt = run_eval.classify_prompt("x")
|
|
for line in prompt.split("\n")[:2]:
|
|
self.assertIn(line, classify)
|
|
|
|
def test_tags_match_typescript(self):
|
|
tags = (HOOKS / "tags.ts").read_text(encoding="utf-8")
|
|
for key in run_eval.TAGS:
|
|
self.assertIn(f"{key}:", tags)
|
|
|
|
def test_parse_function(self):
|
|
self.assertEqual(run_eval.parse_function(" Refuse。"), "refuse")
|
|
self.assertIsNone(run_eval.parse_function("不知道"))
|
|
|
|
def test_run_and_render_side_by_side(self):
|
|
s = json.loads((run_eval.HERE / "scenarios.json").read_text(encoding="utf-8"))
|
|
v = json.loads((run_eval.HERE / "variants.json").read_text(encoding="utf-8"))
|
|
calls = []
|
|
|
|
def ask(prompt, model, system=None):
|
|
calls.append((model, system))
|
|
return "refuse" if model == "haiku" else "回覆|換行\n第二行"
|
|
|
|
classified, replies = run_eval.run(s, v, "sonnet", "", ask)
|
|
md = run_eval.render(s, v, classified, replies, "sonnet")
|
|
self.assertEqual(len(calls), len(s) + len(s) * len(v))
|
|
self.assertIn("分類：1/", md)
|
|
self.assertIn("回覆\\|換行<br>第二行", md)
|
|
for x in v:
|
|
self.assertIn(f"| {x['label']} |", md)
|
|
|
|
def test_examples_use_matching_function_only(self):
|
|
idx = [{"id": "b", "function": "refuse", "text": "不行"}, {"id": "a", "function": "greet", "text": "早"}]
|
|
self.assertIn("不行", run_eval.examples("sao-asuna", idx, "refuse"))
|
|
self.assertNotIn("早", run_eval.examples("sao-asuna", idx, "refuse"))
|
|
self.assertIsNone(run_eval.examples("sao-asuna", idx, "thank"))
|
|
|
|
|
|
if __name__ == "__main__":
|
|
unittest.main()
|