mirror of
https://github.com/anomalyco/hex.git
synced 2026-11-22 21:23:02 +01:00
* ios: clean up what I say with Hex's on-device dictation tagger (experiment)
What changed
- Experiments › Clean up what I say (off by default, App Group key
experiments.cleanupTagger): Hex's trained cleanup tagger removes ums,
repeats, and the words a spoken correction replaces ("3pm, actually 4pm"),
on the iPhone, in in-keyboard dictation and in app dictation. It runs after
transcription and learned fixes (in the app also after custom vocabulary),
before Apple Intelligence cleanup, translation, and formatting, and never
for Verbatim. When both cleanups are on, Apple Intelligence now receives the
tagged text instead of the raw transcript; nothing else changes while the
experiment is off.
- Model: the BERT L-4/H-512 token classifier (ckpt-1200k-l4h512) converted to
an int8 Core ML ML Program (coremltools 9, fp16 compute, enumerated lengths
16/32/64/128 plus a length input that masks the padding; the output is
logit(DELETE) - logit(KEEP)), 28.5 MB, with its WordPiece vocabulary. Both
ship once, in HEXKeyboard.appex (HEXKeyboard/CleanupTagger/), and the app
loads them from its embedded keyboard extension. Core ML codegen is off for
the keyboard target.
- HEXShared: CleanupTokenizer (Hugging Face BertTokenizer: special tokens,
BertNormalizer, BertPreTokenizer, WordPiece, byte-exact vocabulary keys,
HF's Unicode 8.0 character classes), CleanupReconstruction (a port of
js/reconstruct.js with JavaScript's UTF-16 casing), and CleanupTagger
(chunks of whole words within 128 tokens, P(DELETE) > 0.98, one serial
queue, 150 ms budget). A timeout, an error, or a result with no words left
keeps the transcript. The model loads when a dictation starts recording with
the experiment on, stays loaded, and is freed on a memory warning. It runs
on the Neural Engine or the CPU, never the GPU.
- Call sites: one each in KeyboardLocalDictation (after DictationCorrections,
before translation) and DictationController (after VocabularySpelling,
before the Apple Intelligence pass). prepareIfEnabled runs at recording
start, in KeyboardViewController and in DictationController.startRecording.
Why
- In the simulator the first load took 0.15 s on the CPU and 0.8 s on the GPU
path, both over the 150 ms budget. So loading starts when recording starts,
and the GPU isn't used (it also added 30 MB of footprint, against 3 MB on the
CPU, which keeps the int8 weights memory-mapped).
- Threshold 0.98, not the exported 0.95: 1/45 false deletes on hand-written
clean sentences against 3/45, with the same self-correction accuracy.
Verified
- xcodebuild test on the iPhone 18 Pro simulator: 168 tests, 0 failures,
including the new CleanupTaggerTests and DictationControllerTests
testCleanupTaggerRunsAfterLearnedFixesAndSkipsVerbatim. After the fixtures
were rewritten one case per line (same content), CleanupTaggerTests and
DictationControllerTests passed again (28 tests).
- Parity on the 137 hand-written sentences at 0.98: labels match the fp32
checkpoint on all 1,128 words, and 136/137 outputs match the ONNX int8
pipeline. #120 ("The cat keeps trying to scratch that chair.") differs on
two words that ONNX keeps (P 0.9725/0.9794) and Core ML deletes (0.984).
The fp32 checkpoint also deletes them (0.9879/0.9853), so the difference
comes from ONNX's int8 activations. 128/137 outputs match the hand-written
targets. The tokenizer matches HF on 1,891 words, and reconstruction matches
reconstruct.js on 670 cases.
- Simulator, CPU path: load 140-161 ms, first sentence after load about 8 ms,
then 3.3 ms median and 7.5 ms p95 per sentence; phys_footprint +2.9 MB.
- The Release generic iOS build with CODE_SIGNING_ALLOWED=NO succeeds.
HEXKeyboard.appex is 30.2 MB (model 27.8 MB, vocabulary 0.2 MB).
- Not verified on a physical iPhone: Neural Engine load time, latency, and the
keyboard extension's footprint next to Apple Speech.
Note: this adds a 28.4 MB binary (weight.bin) to git; the repo has no LFS.
* Fetch the cleanup tagger's Core ML model at build time instead of committing it
The 27 MB mlpackage would have been ~30x the largest file in the repo, kept in history forever. It is now the ios-model-cleanup-tagger-1 release asset; ios/scripts/fetch-cleanup-model.sh downloads it, checks its SHA-256, and skips when the pinned copy is already in place. Xcode Cloud runs it in ci_post_clone.sh before xcodegen; the folder is gitignored.
Verified the fetched package is byte-identical to the one the tests ran against (diff -r), and a second run is a no-op.
209 lines
8 KiB
Swift
209 lines
8 KiB
Swift
import Foundation
|
|
|
|
/// The cleanup tagger's BERT uncased WordPiece tokenizer, matching Hugging Face's
|
|
/// `BertTokenizer` as the tagger was trained and exported with it (`tokenizer.json`):
|
|
/// special tokens split out first, then `BertNormalizer` (clean text, space around CJK
|
|
/// ideographs, NFD accent stripping, lowercasing), `BertPreTokenizer` (whitespace and
|
|
/// punctuation), and greedy longest-match WordPiece with `##` continuations.
|
|
///
|
|
/// Hugging Face's character classes come from Unicode 8.0 tables, so punctuation,
|
|
/// nonspacing marks, and control or format characters assigned later are treated as
|
|
/// ordinary characters, exactly as the trained model saw them.
|
|
nonisolated struct CleanupTokenizer: Sendable {
|
|
static let padID: Int32 = 0
|
|
static let unknownID: Int32 = 100
|
|
static let clsID: Int32 = 101
|
|
static let sepID: Int32 = 102
|
|
static let maxCharactersPerWord = 100
|
|
|
|
enum LoadError: Error {
|
|
case wrongSize(Int)
|
|
}
|
|
|
|
private static let specialTokens: [(scalars: [Unicode.Scalar], id: Int32)] = [
|
|
("[PAD]", 0), ("[UNK]", 100), ("[CLS]", 101), ("[SEP]", 102), ("[MASK]", 103),
|
|
].map { (Array($0.0.unicodeScalars), $0.1) }
|
|
private static let tableVersion = Unicode.Version(major: 8, minor: 0)
|
|
private static let continuation: [UInt8] = Array("##".utf8)
|
|
|
|
/// Keyed by exact UTF-8 bytes: Swift's `String` equality is canonical equivalence,
|
|
/// which would match a precomposed vocabulary entry the Rust tokenizer never matches.
|
|
private let vocabulary: [[UInt8]: Int32]
|
|
|
|
/// `text` is `vocab.txt`: one token per line, the line number is its id.
|
|
init(vocabulary text: String, expectedSize: Int? = 30522) throws {
|
|
var vocabulary: [[UInt8]: Int32] = [:]
|
|
vocabulary.reserveCapacity(expectedSize ?? 32_000)
|
|
var lines = text.utf8.split(separator: UInt8(ascii: "\n"), omittingEmptySubsequences: false)
|
|
if lines.last?.isEmpty == true {
|
|
lines.removeLast()
|
|
}
|
|
for (id, line) in lines.enumerated() where vocabulary[Array(line)] == nil {
|
|
vocabulary[Array(line)] = Int32(id)
|
|
}
|
|
if let expectedSize, lines.count != expectedSize {
|
|
throw LoadError.wrongSize(lines.count)
|
|
}
|
|
self.vocabulary = vocabulary
|
|
}
|
|
|
|
/// WordPiece ids for one whitespace-separated word, without [CLS] or [SEP].
|
|
func encode(_ word: String) -> [Int32] {
|
|
var ids: [Int32] = []
|
|
let scalars = Array(word.unicodeScalars)
|
|
var text: [Unicode.Scalar] = []
|
|
var index = 0
|
|
while index < scalars.count {
|
|
if scalars[index] == "[", let special = Self.specialToken(in: scalars, at: index) {
|
|
appendText(text, to: &ids)
|
|
text.removeAll()
|
|
ids.append(special.id)
|
|
index += special.length
|
|
} else {
|
|
text.append(scalars[index])
|
|
index += 1
|
|
}
|
|
}
|
|
appendText(text, to: &ids)
|
|
return ids
|
|
}
|
|
|
|
private func appendText(_ text: [Unicode.Scalar], to ids: inout [Int32]) {
|
|
guard !text.isEmpty else { return }
|
|
for piece in Self.preTokenize(Self.normalize(text)) {
|
|
ids += wordPiece(piece)
|
|
}
|
|
}
|
|
|
|
private static func specialToken(in scalars: [Unicode.Scalar], at index: Int) -> (id: Int32, length: Int)? {
|
|
for token in specialTokens where index + token.scalars.count <= scalars.count {
|
|
if scalars[index..<(index + token.scalars.count)].elementsEqual(token.scalars) {
|
|
return (token.id, token.scalars.count)
|
|
}
|
|
}
|
|
return nil
|
|
}
|
|
|
|
// MARK: BertNormalizer
|
|
|
|
static func normalize(_ scalars: [Unicode.Scalar]) -> [Unicode.Scalar] {
|
|
var cleaned = String.UnicodeScalarView()
|
|
for scalar in scalars {
|
|
if scalar.value == 0 || scalar.value == 0xFFFD || isControl(scalar) {
|
|
continue
|
|
}
|
|
if isWhitespace(scalar) {
|
|
cleaned.append(" ")
|
|
} else if isChineseCharacter(scalar) {
|
|
cleaned.append(contentsOf: [" ", scalar, " "])
|
|
} else {
|
|
cleaned.append(scalar)
|
|
}
|
|
}
|
|
var normalized: [Unicode.Scalar] = []
|
|
normalized.reserveCapacity(cleaned.count)
|
|
for scalar in String(cleaned).decomposedStringWithCanonicalMapping.unicodeScalars
|
|
where !isNonspacingMark(scalar) {
|
|
normalized.append(contentsOf: scalar.properties.lowercaseMapping.unicodeScalars)
|
|
}
|
|
return normalized
|
|
}
|
|
|
|
private static func isWhitespace(_ scalar: Unicode.Scalar) -> Bool {
|
|
scalar == "\t" || scalar == "\n" || scalar == "\r" || scalar.properties.isWhitespace
|
|
}
|
|
|
|
private static func isControl(_ scalar: Unicode.Scalar) -> Bool {
|
|
if scalar == "\t" || scalar == "\n" || scalar == "\r" { return false }
|
|
switch scalar.properties.generalCategory {
|
|
case .control, .format, .privateUse, .surrogate:
|
|
return inTables(scalar)
|
|
default:
|
|
return false
|
|
}
|
|
}
|
|
|
|
private static func isNonspacingMark(_ scalar: Unicode.Scalar) -> Bool {
|
|
scalar.properties.generalCategory == .nonspacingMark && inTables(scalar)
|
|
}
|
|
|
|
private static func isPunctuation(_ scalar: Unicode.Scalar) -> Bool {
|
|
if scalar.isASCII {
|
|
let value = scalar.value
|
|
return (0x21...0x2F).contains(value) || (0x3A...0x40).contains(value)
|
|
|| (0x5B...0x60).contains(value) || (0x7B...0x7E).contains(value)
|
|
}
|
|
switch scalar.properties.generalCategory {
|
|
case .connectorPunctuation, .dashPunctuation, .openPunctuation, .closePunctuation,
|
|
.initialPunctuation, .finalPunctuation, .otherPunctuation:
|
|
return inTables(scalar)
|
|
default:
|
|
return false
|
|
}
|
|
}
|
|
|
|
/// Hugging Face's category tables stop at Unicode 8.0.
|
|
private static func inTables(_ scalar: Unicode.Scalar) -> Bool {
|
|
guard let age = scalar.properties.age else { return false }
|
|
return age.major < tableVersion.major || (age.major == tableVersion.major && age.minor <= tableVersion.minor)
|
|
}
|
|
|
|
private static func isChineseCharacter(_ scalar: Unicode.Scalar) -> Bool {
|
|
switch scalar.value {
|
|
case 0x4E00...0x9FFF, 0x3400...0x4DBF, 0x20000...0x2A6DF, 0x2A700...0x2B73F,
|
|
0x2B740...0x2B81F, 0x2B920...0x2CEAF, 0xF900...0xFAFF, 0x2F800...0x2FA1F:
|
|
return true
|
|
default:
|
|
return false
|
|
}
|
|
}
|
|
|
|
// MARK: BertPreTokenizer
|
|
|
|
static func preTokenize(_ scalars: [Unicode.Scalar]) -> [[Unicode.Scalar]] {
|
|
var pieces: [[Unicode.Scalar]] = []
|
|
var current: [Unicode.Scalar] = []
|
|
for scalar in scalars {
|
|
if scalar.properties.isWhitespace {
|
|
if !current.isEmpty { pieces.append(current) }
|
|
current.removeAll()
|
|
} else if isPunctuation(scalar) {
|
|
if !current.isEmpty { pieces.append(current) }
|
|
current.removeAll()
|
|
pieces.append([scalar])
|
|
} else {
|
|
current.append(scalar)
|
|
}
|
|
}
|
|
if !current.isEmpty { pieces.append(current) }
|
|
return pieces
|
|
}
|
|
|
|
// MARK: WordPiece
|
|
|
|
private func wordPiece(_ piece: [Unicode.Scalar]) -> [Int32] {
|
|
guard piece.count <= Self.maxCharactersPerWord else { return [Self.unknownID] }
|
|
let bytes = piece.map { Array(String($0).utf8) }
|
|
var ids: [Int32] = []
|
|
var start = 0
|
|
while start < piece.count {
|
|
var end = piece.count
|
|
var match: Int32?
|
|
while start < end {
|
|
var key = start > 0 ? Self.continuation : []
|
|
for index in start..<end {
|
|
key += bytes[index]
|
|
}
|
|
if let id = vocabulary[key] {
|
|
match = id
|
|
break
|
|
}
|
|
end -= 1
|
|
}
|
|
guard let match else { return [Self.unknownID] }
|
|
ids.append(match)
|
|
start = end
|
|
}
|
|
return ids
|
|
}
|
|
}
|