hex/scripts/test-vocabulary-eval.py

66 lines
3.4 KiB
Python

#!/usr/bin/env python3
import runpy
from pathlib import Path
import unittest
from unittest.mock import patch
import contextlib
import io
import json
import tempfile
score = runpy.run_path(str(Path(__file__).with_name("vocabulary-eval.py")))
class Scoring(unittest.TestCase):
def test_regenerated_wav_replaces_old_receipt_and_limit_preserves_other_clips(self):
generator = runpy.run_path(str(Path(__file__).with_name("vocabulary-corpus.py")))
with tempfile.TemporaryDirectory() as directory:
root = Path(directory)
fixture = Path(__file__).parent.parent / "tests/fixtures/vocabulary/manifest.json"
prior = json.loads(fixture.read_text())
prior["clips"] = prior["clips"][:2]
prior["clips"][0]["sha256"] = "f" * 64
(root / "manifest.json").write_text(json.dumps(prior))
# No WAV exists: the first is regenerated; the second receipt must
# survive this bounded generation pass. No provider call is made.
with patch("sys.argv", ["generate", directory, "--generate", "--limit", "1"]), \
patch.dict("os.environ", {"ELEVENLABS_API_KEY": "test-only"}), \
patch("urllib.request.urlopen", return_value=io.BytesIO(b"\0\0" * 16000)), \
contextlib.redirect_stdout(io.StringIO()):
generator["main"]()
result = json.loads((root / "manifest.json").read_text())
self.assertEqual(len(result["clips"]), 2)
self.assertNotEqual(result["clips"][0]["sha256"], "f" * 64)
self.assertEqual(result["clips"][1], prior["clips"][1])
def test_boundaries_spacing_and_repetition(self):
count = score["occurrences"]
self.assertEqual(count("OPEN CODE and OpenCode", "OpenCode"), 2)
self.assertEqual(count("hexadecimal toolkit effects runes", "Hex"), 0)
self.assertEqual(count("hexadecimal toolkit effects runes", "Kit"), 0)
self.assertEqual(count("A rune carved into stone", "Rune"), 1)
def test_guard_and_target_denominators_and_all_hex_timings(self):
manifest = dict(vocabulary=["Hex", "Rune"], clips=[
dict(id="term", expected="Use Hex", expected_terms=["Hex"], group="target"),
dict(id="guard", expected="Hanks found a rune", group="distractor")])
results = [dict(type="clip", id="term", transcript="Use Hex", audio_ms=1000, inference_ms=10),
dict(type="clip", id="guard", transcript="Hex found a rune", audio_ms=1000, inference_ms=10)]
for run, time in [(1, 10), (2, 20), (3, 60)]:
results.extend(dict(type="measurement", id=c["id"], run=run, inference_ms=time) for c in manifest["clips"])
report = score["score"](manifest, results)
self.assertEqual(report["target"]["term_recall"], 1)
self.assertIsNone(report["guard"]["term_recall"])
self.assertEqual(report["guard"]["false_replacements"], 1)
self.assertEqual(report["guard"]["false_replacement_rate"], 1)
self.assertEqual(report["guard"]["wer"], .25)
self.assertEqual(report["all"]["median_clip_ms"], 20)
self.assertEqual(report["all"]["p95_clip_ms"], 60)
with self.assertRaises(ValueError):
score["score"](manifest, results[:-1])
with self.assertRaises(ValueError):
score["score"](manifest, results + [results[0]])
if __name__ == "__main__":
unittest.main()