hex/scripts/vocabulary-eval.py

156 lines
6.8 KiB
Python

#!/usr/bin/env python3
"""Merge benchmark manifests or score Hex/decoder-spike JSONL without audio I/O."""
import argparse
import collections
import json
import hashlib
import math
from pathlib import Path
import re
import statistics
def words(text):
return [w.lower() for w in re.findall(r"[^\W_]+", text)]
def occurrences(text, term):
target = "".join(words(term))
if not target:
raise ValueError("empty vocabulary term")
tokens = words(text)
start, count = 0, 0
while start < len(tokens):
candidate = ""
next_start = start + 1
for end in range(start, len(tokens)):
candidate += tokens[end]
if candidate == target:
count += 1
next_start = end + 1
break
if not target.startswith(candidate):
break
start = next_start
return count
def edits(reference, hypothesis):
reference, hypothesis = words(reference), words(hypothesis)
row = list(range(len(hypothesis) + 1))
for i, left in enumerate(reference):
current = [i + 1]
for j, right in enumerate(hypothesis):
current.append(min(row[j] + (left != right), row[j + 1] + 1, current[j] + 1))
row = current
return row[-1], len(reference)
def merge(output, inputs):
clips, vocabulary, ids = [], {}, set()
for source in inputs:
manifest = json.loads(source.read_text())
for term in manifest.get("vocabulary", []):
vocabulary.setdefault("".join(words(term)), term)
for clip in manifest["clips"]:
if clip["id"] in ids:
raise ValueError(f"duplicate clip: {clip['id']}")
ids.add(clip["id"])
audio = (source.parent / clip["audio"]).resolve(strict=True)
if "sha256" in clip and hashlib.sha256(audio.read_bytes()).hexdigest() != clip["sha256"]:
raise ValueError(f"WAV checksum mismatch: {clip['id']}")
if "group" not in clip:
raise ValueError(f"merge requires an explicit corpus group: {clip['id']}")
clip["audio"] = str(audio)
clips.append(clip)
output.parent.mkdir(parents=True, exist_ok=True)
# Merged manifests may contain private text and local absolute paths.
import os
with os.fdopen(os.open(output, os.O_WRONLY | os.O_CREAT | os.O_TRUNC, 0o600), "w") as file:
os.fchmod(file.fileno(), 0o600)
json.dump(dict(vocabulary=list(vocabulary.values()), clips=clips), file, indent=2)
file.write("\n")
def score(manifest, records):
definitions = {c["id"]: c for c in manifest["clips"]}
if len(definitions) != len(manifest["clips"]) or not definitions:
raise ValueError("empty or duplicate clip IDs")
runs = collections.defaultdict(dict)
measurements = collections.defaultdict(dict)
for row in records:
if row["type"] == "measurement":
if row["id"] not in definitions or row["id"] in measurements[row["run"]]:
raise ValueError("unexpected/duplicate timing")
measurements[row["run"]][row["id"]] = row
continue
if row["type"] != "clip":
continue
run = row.get("run", 1)
if row["id"] not in definitions or row["id"] in runs[run]:
raise ValueError(f"unexpected/duplicate result: {row['id']}")
if "expected" in row and row["expected"] != definitions[row["id"]]["expected"]:
raise ValueError(f"reference changed: {row['id']}")
runs[run][row["id"]] = row
if not runs or any(set(rows) != set(definitions) for rows in runs.values()):
raise ValueError("incomplete run; refusing to score a partial corpus")
first = runs[min(runs)]
timings = runs
if measurements:
if any(set(rows) != set(definitions) for rows in measurements.values()):
raise ValueError("incomplete timing pass")
timings = {run: {ident: {**first[ident], **row} for ident, row in rows.items()}
for run, rows in measurements.items()}
summaries = {}
groups = sorted({c.get("group", "ordinary") for c in definitions.values()})
for group in ["all", "guard"] + groups:
subset = [c for c in definitions.values() if group == "all" or
(group == "guard" and c.get("group", "ordinary") in ("ordinary", "distractor")) or
group == c.get("group", "ordinary")]
if not subset:
continue
errors = reference_words = expected = recalled = false = affected = 0
for c in subset:
text = first[c["id"]]["transcript"]
e, n = edits(c["expected"], text)
errors += e
reference_words += n
terms = c.get("expected_terms", [])
expected += len(terms)
recalled += sum(occurrences(text, t) > 0 for t in terms)
inserted = sum(max(0, occurrences(text, t) - occurrences(c["expected"], t)) for t in manifest["vocabulary"])
false += inserted
affected += inserted > 0
times = sorted(rows[c["id"]]["inference_ms"] for rows in timings.values() for c in subset)
finish = sorted(rows[c["id"]]["finish_ms"] for rows in timings.values() for c in subset if "finish_ms" in rows[c["id"]])
summaries[group] = dict(clips=len(subset), word_errors=errors, reference_words=reference_words,
wer=errors / reference_words if reference_words else None, expected_terms=expected,
recalled_terms=recalled, term_recall=recalled / expected if expected else None,
false_replacements=false, false_replacement_rate=affected / len(subset) if manifest["vocabulary"] else None,
median_clip_ms=statistics.median(times), p95_clip_ms=times[math.ceil(len(times)*.95)-1],
p95_finish_ms=finish[math.ceil(len(finish)*.95)-1] if finish else None,
median_rtf=statistics.median(sum(rows[c['id']]['inference_ms'] for c in subset) /
sum(rows[c['id']]['audio_ms'] for c in subset) for rows in timings.values()))
return summaries
def main():
parser = argparse.ArgumentParser(description=__doc__)
commands = parser.add_subparsers(dest="command", required=True)
merger = commands.add_parser("merge")
merger.add_argument("output", type=Path)
merger.add_argument("manifests", nargs="+", type=Path)
scorer = commands.add_parser("score")
scorer.add_argument("manifest", type=Path)
scorer.add_argument("results", type=Path)
args = parser.parse_args()
if args.command == "merge":
merge(args.output, args.manifests)
else:
records = [json.loads(line) for line in args.results.read_text().splitlines() if line.strip()]
print(json.dumps(score(json.loads(args.manifest.read_text()), records), indent=2))
if __name__ == "__main__":
main()