mirror of
https://github.com/anomalyco/hex.git
synced 2026-11-22 21:23:02 +01:00
242 lines
12 KiB
TypeScript
242 lines
12 KiB
TypeScript
// Dictation end to end (docs/perf.md). The copy hears `speech.wav`, one
|
|
// spoken sentence and then silence, instead of a microphone
|
|
// (HEX_AUDIO_FIXTURE, seeds/dictation.ts), so nobody is recorded. The bench
|
|
// drives it through its local API as `hex dictation` does: `start` as the
|
|
// sentence begins, then `stop --print` in the silence after it, which
|
|
// returns the processed text instead of pasting it. Nothing reaches another
|
|
// app. Only bench builds that have HEX_AUDIO_FIXTURE (#1230): an
|
|
// older one opens the real microphone, which setup notices too late.
|
|
// - `warm`: the microphone open while idle, the default (`hex-dictation`).
|
|
// - `released`: Release microphone while idle (`hex-dictation-released`).
|
|
import { bench, native } from "@hona/strobe"
|
|
import { closeSync, openSync, readFileSync, readSync, statSync, writeFileSync } from "node:fs"
|
|
import { join } from "node:path"
|
|
import { pathToFileURL } from "node:url"
|
|
import { dictationSeed, SILENCE_MS, speechMs } from "./seeds/dictation"
|
|
|
|
const home = dictationSeed()
|
|
if (!home) throw new Error("dictation.bench.ts needs Hex's English model installed and a text-to-speech voice")
|
|
|
|
const SPEECH_MS = speechMs(home)
|
|
/** The fixture's length: it loops. */
|
|
const PERIOD_MS = SPEECH_MS + SILENCE_MS
|
|
/** Each dictation stops this long into the silence after the sentence. */
|
|
const STOP_AFTER_MS = SPEECH_MS + 500
|
|
|
|
type Ctx = { home: string; target: string; page: { wait(ms: number): Promise<unknown> } }
|
|
type Reply = { status: string; text?: string; code?: string; message?: string }
|
|
|
|
/** One `POST /control` to the copy, as the `hex` CLI sends it (docs/cli.md). */
|
|
async function control(ctx: Ctx, command: Record<string, unknown>): Promise<Reply> {
|
|
const api = JSON.parse(readFileSync(join(ctx.home, "hex-data", "local-api.json"), "utf8"))
|
|
const response = await fetch(`http://127.0.0.1:${api.port}/control`, {
|
|
method: "POST",
|
|
headers: { authorization: `Bearer ${api.token}`, "content-type": "application/json" },
|
|
body: JSON.stringify({ version: 4, ...command }),
|
|
})
|
|
return (await response.json()) as Reply
|
|
}
|
|
|
|
const stop = (ctx: Ctx) => control(ctx, { command: "dictation", action: "stop", print: true })
|
|
|
|
/** When the fixture last opened (epoch ms), from the copy's log: it plays from its start at each open. */
|
|
function fixtureOpened(ctx: Ctx): number {
|
|
const log = readFileSync(join(ctx.home, "hex-data", "logs", "process.log"), "utf8")
|
|
const opened = [...log.matchAll(/^\S*?(\d{4}-\d\d-\d\dT[\d:.]+Z).*HEX_AUDIO_FIXTURE replaces the microphone/gm)]
|
|
if (!opened.length) throw new Error("the copy never opened its audio fixture")
|
|
return Date.parse(opened[opened.length - 1][1])
|
|
}
|
|
|
|
/** Starts a dictation as the sentence begins. A released microphone opens
|
|
* the fixture at its start; an open one waits for the loop to come round. */
|
|
async function start(ctx: Ctx): Promise<Reply> {
|
|
if (ctx.target === "warm") {
|
|
const into = (Date.now() - fixtureOpened(ctx)) % PERIOD_MS
|
|
await ctx.page.wait(PERIOD_MS - into)
|
|
}
|
|
return control(ctx, { command: "dictation", action: "start" })
|
|
}
|
|
|
|
/** The copy's event log (`live.ndjson`), whose states say when a recording began. */
|
|
const events = (ctx: Ctx) => join(ctx.home, "hex-data", "logs", "live.ndjson")
|
|
|
|
/** Resolves once the event log, read from byte `from` on, shows the
|
|
* recording began. A released microphone's start reports `dictating` twice:
|
|
* at the command, and at the opened microphone's first chunk, where the
|
|
* recording begins. The log is written as each event happens; this reads it
|
|
* every millisecond. */
|
|
async function recordingBegan(ctx: Ctx, from: number): Promise<void> {
|
|
const deadline = Date.now() + 5_000
|
|
const fd = openSync(events(ctx), "r")
|
|
try {
|
|
let text = ""
|
|
for (;;) {
|
|
const chunk = Buffer.alloc(64 * 1024)
|
|
const read = readSync(fd, chunk, 0, chunk.length, from + Buffer.byteLength(text))
|
|
text += chunk.toString("utf8", 0, read)
|
|
if ((text.match(/"state":"dictating"/g) ?? []).length >= 2) return
|
|
if (Date.now() > deadline) throw new Error("the recording never began")
|
|
await Bun.sleep(1)
|
|
}
|
|
} finally {
|
|
closeSync(fd)
|
|
}
|
|
}
|
|
|
|
/** The browser started by the `browser` case's setup. */
|
|
let browser: ReturnType<typeof Bun.spawn> | undefined
|
|
|
|
/** Puts a browser in front, as when someone dictates into a web page: a
|
|
* browser with its own profile in the copy's home, on a long local page,
|
|
* brought to the foreground. Windows: Microsoft Edge, on an unlocked
|
|
* session (a locked one keeps the lock screen in front). Linux: Chromium
|
|
* on X11 with its page in the accessibility tree (Chromium exposes pages
|
|
* to AT-SPI only with renderer accessibility on). */
|
|
async function browserInFront(ctx: Ctx): Promise<void> {
|
|
const page = join(ctx.home, "long-page.html")
|
|
const rows = Array.from({ length: 3_000 }, (_, i) => `<p>Paragraph ${i} with a <a href="#${i}">link</a> and <button>button ${i}</button></p>`)
|
|
writeFileSync(page, `<!doctype html><title>Long page</title><body>${rows.join("")}</body>`)
|
|
if (process.platform === "linux") {
|
|
const root = process.getuid?.() === 0 ? ["--no-sandbox"] : []
|
|
const profile = `--user-data-dir=${join(ctx.home, "chromium")}`
|
|
browser = Bun.spawn(["chromium", profile, ...root, "--no-first-run", "--no-default-browser-check", "--force-renderer-accessibility", "--new-window", pathToFileURL(page).href])
|
|
await ctx.page.wait(5_000)
|
|
Bun.spawnSync(["xdotool", "search", "--sync", "--name", "Long page", "windowactivate", "--sync"])
|
|
const pid = Bun.spawnSync(["xdotool", "getactivewindow", "getwindowpid"]).stdout.toString().trim()
|
|
const front = pid ? readFileSync(`/proc/${pid}/comm`, "utf8").trim() : "nothing"
|
|
if (!front.startsWith("chromium")) throw new Error(`the browser case needs Chromium in front on X11; in front: ${front}`)
|
|
return
|
|
}
|
|
const edge = "C:/Program Files (x86)/Microsoft/Edge/Application/msedge.exe"
|
|
browser = Bun.spawn([edge, `--user-data-dir=${join(ctx.home, "edge")}`, "--no-first-run", "--no-default-browser-check", "--new-window", pathToFileURL(page).href])
|
|
await ctx.page.wait(5_000)
|
|
// A background process may take the foreground once a key went to it.
|
|
const script = `
|
|
Add-Type -Name W -Namespace F -MemberDefinition '[DllImport("user32.dll")] public static extern bool SetForegroundWindow(System.IntPtr h); [DllImport("user32.dll")] public static extern void keybd_event(byte k, byte s, uint f, System.UIntPtr e); [DllImport("user32.dll")] public static extern System.IntPtr GetForegroundWindow(); [DllImport("user32.dll")] public static extern uint GetWindowThreadProcessId(System.IntPtr h, out uint p);'
|
|
$w = Get-Process msedge | Where-Object { $_.MainWindowTitle -like 'Long page*' } | Select-Object -First 1
|
|
if (-not $w) { 'no window'; exit 1 }
|
|
[F.W]::keybd_event(0x12, 0, 0, [System.UIntPtr]::Zero); [F.W]::keybd_event(0x12, 0, 2, [System.UIntPtr]::Zero)
|
|
[void][F.W]::SetForegroundWindow($w.MainWindowHandle)
|
|
Start-Sleep -Milliseconds 500
|
|
$p = 0; [void][F.W]::GetWindowThreadProcessId([F.W]::GetForegroundWindow(), [ref]$p)
|
|
(Get-Process -Id $p).ProcessName`
|
|
const front = Bun.spawnSync(["powershell", "-NoProfile", "-Command", script]).stdout.toString().trim()
|
|
if (front !== "msedge") throw new Error(`the browser case needs Edge in front on an unlocked session; in front: ${front}`)
|
|
}
|
|
|
|
async function printed(reply: Reply): Promise<string> {
|
|
if (reply.status !== "transcript" || !reply.text) throw new Error(`stop --print answered ${JSON.stringify(reply)}`)
|
|
return reply.text
|
|
}
|
|
|
|
// Windows and Linux open no window with `--hidden`; the macOS app opens none
|
|
// once set up, and `--device` makes a build without the fixture fail to open
|
|
// the microphone instead of recording it.
|
|
const args = process.platform === "darwin" ? ["--device", "Audio fixture"] : ["--hidden"]
|
|
|
|
export default bench({
|
|
targets: {
|
|
warm: native("hex-dictation", { args, home, window: false }),
|
|
released: native("hex-dictation-released", { args, home, window: false }),
|
|
// Released and idle between dictations. ggml keeps a Metal model's
|
|
// memory wired for 3 minutes after its last use; 2 s lets a run see what
|
|
// an idle Hex does once that has run out.
|
|
idle: native("hex-dictation-released", { args, home, window: false, env: { GGML_METAL_RESIDENCY_KEEP_ALIVE_S: "2" } }),
|
|
},
|
|
// The model loads and the listener starts; one dictation then warms the
|
|
// whole path (stream, model, output).
|
|
async setup(ctx) {
|
|
const deadline = Date.now() + 60_000
|
|
for (;;) {
|
|
const reply = await control(ctx, { command: "dictation", action: "start" }).catch((error) => ({ status: "error", message: String(error) }) as Reply)
|
|
if (reply.status === "done") break
|
|
if (Date.now() > deadline) throw new Error(`dictation start answered ${JSON.stringify(reply)}`)
|
|
await ctx.page.wait(500)
|
|
}
|
|
await ctx.page.wait(STOP_AFTER_MS)
|
|
await printed(await stop(ctx))
|
|
fixtureOpened(ctx)
|
|
if (ctx.case === "recording") await start(ctx)
|
|
if (ctx.case === "browser") await browserInFront(ctx)
|
|
},
|
|
teardown() {
|
|
browser?.kill()
|
|
browser = undefined
|
|
},
|
|
cases: {
|
|
// Listening between dictations, microphone released: what the loaded
|
|
// model and the listener cost while nothing records.
|
|
listening: {
|
|
targets: ["idle"],
|
|
primary: "cpu.wakes",
|
|
metrics: ["cpu.work", "memory.app.host"],
|
|
async run({ page }) {
|
|
await page.wait(8_000)
|
|
},
|
|
},
|
|
// Start to capture with the microphone released: the start command
|
|
// reaching the idle listener, the microphone opening, and its first
|
|
// chunk recorded. Not on macOS: its log reports `dictating` once per
|
|
// start, so the log doesn't say when the first chunk arrived.
|
|
...(process.platform !== "darwin" && {
|
|
start: {
|
|
targets: ["released"],
|
|
primary: "phase.start",
|
|
metrics: ["cpu.work", "cpu.wakes"],
|
|
async run(ctx) {
|
|
// The listener settles into its idle wait after the last dictation.
|
|
await ctx.page.wait(1_000)
|
|
const from = statSync(events(ctx)).size
|
|
await ctx.phase("start", async () => {
|
|
await start(ctx)
|
|
await recordingBegan(ctx, from)
|
|
})
|
|
await ctx.page.wait(STOP_AFTER_MS)
|
|
await printed(await stop(ctx))
|
|
},
|
|
},
|
|
}),
|
|
// Release to text: the stop's wait for the transcript (the 200 ms
|
|
// release tail, the live stream's finish, processing, and History).
|
|
dictation: {
|
|
targets: ["warm", "released"],
|
|
primary: "phase.finish",
|
|
metrics: ["cpu.work", "cpu.wakes", "memory.app.host"],
|
|
async run(ctx) {
|
|
await start(ctx)
|
|
await ctx.page.wait(STOP_AFTER_MS)
|
|
await printed(await ctx.phase("finish", () => stop(ctx)))
|
|
},
|
|
},
|
|
// Release to text with a browser in front (Windows unlocked, Linux on
|
|
// X11): the dictation case's stop, plus the page read the foreground
|
|
// context starts for a browser. Nothing set up here uses the page.
|
|
...(process.platform !== "darwin" && {
|
|
browser: {
|
|
targets: ["warm"],
|
|
primary: "phase.finish",
|
|
metrics: ["cpu.work", "cpu.wakes"],
|
|
async run(ctx) {
|
|
await start(ctx)
|
|
await ctx.page.wait(STOP_AFTER_MS)
|
|
await printed(await ctx.phase("finish", () => stop(ctx)))
|
|
},
|
|
},
|
|
}),
|
|
// While a dictation records: the audio path, the live stream's
|
|
// inference, and the HUD meter's plumbing. Each run is 6 s of one
|
|
// recording; reset finishes it and starts the next.
|
|
recording: {
|
|
targets: ["warm", "released"],
|
|
primary: "cpu.wakes",
|
|
metrics: ["cpu.work", "memory.app.host"],
|
|
async run({ page }) {
|
|
await page.wait(6_000)
|
|
},
|
|
async reset(ctx) {
|
|
await printed(await stop(ctx))
|
|
await start(ctx)
|
|
},
|
|
},
|
|
},
|
|
})
|