#!/usr/bin/env python3 """The typing quality gate for the iOS keyboard, on the same held-out inputs and scoring as android/benchmark/typing/quality-gate.py (#2374). python3 scripts/typing-gate.py run --destination 'id=' [en es de fr it nb] python3 scripts/typing-gate.py prepare OUT [langs] # inputs only python3 scripts/typing-gate.py score OUT [langs] # score outputs already in OUT `prepare` reads the Android benchmark's held-out sentences (held-out/.tsv.gz, the accent supplements) and its accent-keys.json, critical-cases.json and thresholds.json; it writes OUT/input.json for the opt-in TypingGateReplayTests, which types every input through FullKeyboardEngine on the simulator and writes OUT/output-.json. `run` does all three, one xcodebuild per language (a fresh process each, so one language's UITextChecker state never meets another's). Inputs, as quality-gate.py builds them: - noisy: the 2,000 sentences' hit keys and taps, moved onto Hex's layout key by key from the synthetic layout they were made on (402 pt wide, rows at y 66.5 / 122.5 / 178.5, space bar 64.76…303.1 x 212…257); a tap Hex's keys don't take as a letter or space (outside them, or on 123, shift or delete) goes to the nearest letter or space bar; - clean: each reference typed right at the key centres, a letter without a key of its own picked from its key's long-press menu, an apostrophe without a key from the 123 page; - accents: up to 500 references with a hold-only letter, typed with its substitute; - critical: the named cases, at the key centres; - lowercase: each reference typed all lowercase at the key centres, the keyboard's shift left as it is (it capitalises a sentence's start), for the capitals report below. Scoring is quality-gate.py's: word edits case-insensitively; correct words changed on the clean pass; accent words bucketed by whether the plain spelling is in the model or .valid. iOS also reports capitals, which the three numbers can't see as they ignore case: on the lowercase pass, of the reference words after a sentence's first with a capital (names, "I"), how many come out with it, and of those without, how many gain one. A report, not a threshold. Outputs also lose the . , ! ? ; : and quotes the engine may add, and ’ reads as '. """ import argparse from collections import Counter import gzip import json import os from pathlib import Path import re import struct import subprocess import sys import unicodedata IOS = Path(__file__).resolve().parents[1] BENCH = IOS.parent / 'android/benchmark/typing' MODELS = IOS / 'HEXKeyboard/TypingModel' ANDROID_MODELS = IOS.parent / 'android/app/src/main/assets/typing' LANGUAGES = ['en', 'es', 'de', 'fr', 'it', 'nb', 'pl'] ROW_Y, SPACE = [66.5, 122.5, 178.5], [64.76, 212.0, 238.34, 45.0] ROWS = {'en': ['qwertyuiop', 'asdfghjkl', 'zxcvbnm'], 'es': ['qwertyuiop', 'asdfghjklñ', 'zxcvbnm'], 'de': ['qwertzuiopü', 'asdfghjklöä', 'yxcvbnm'], 'nb': ['qwertyuiopå', 'asdfghjkløæ', 'zxcvbnm'], 'fr': ['azertyuiop', 'qsdfghjklm', "wxcvbn'"], 'it': ['qwertyuiop', 'asdfghjkl', 'zxcvbnm'], 'pl': ['qwertyuiop', 'asdfghjkl', 'zxcvbnm']} def config(name): return json.loads((BENCH / name).read_text(encoding='utf-8')) def geometry(lang): """The synthetic layout the held-out taps were made on (synthetic_sets.py, iOS's measured one for English and Spanish): key frames x, y, w, h.""" pitch = 36.0 if lang in ('de', 'nb') else 39.5 keys = {} for r, letters in enumerate(ROWS[lang]): for i, c in enumerate(letters): x = 21 + pitch * i if len(letters) == 11 else 201 + (i - (len(letters) - 1) / 2) * pitch keys[c] = [x - (pitch - 6) / 2, ROW_Y[r] - 22.5, pitch - 6, 45.0] keys['space'] = SPACE return keys def centred(text, keys, holds, lang): """Taps at the key centres for `text`: [-1, -1] picks a letter from its key's long-press menu, [0, 0] types an apostrophe from the 123 page.""" taps, want = [], [] for ch in text: low = ch.lower() if ch == ' ': taps.append([SPACE[0] + SPACE[2] / 2, SPACE[1] + SPACE[3] / 2]) elif low in keys: k = keys[low] taps.append([round(k[0] + k[2] / 2, 2), round(k[1] + k[3] / 2, 2)]) elif ch == "'": taps.append([0, 0]) elif low in holds: taps.append([-1, -1]) else: raise ValueError(f'{lang}: no key for {ch!r} in {text!r}') want.append(ch) return taps, ''.join(want) def model_words(path): b = path.read_bytes() words, _, blob, letters = struct.unpack_from('<4I', b, 8) chars = [' '] * (letters + 1) for i in range(letters): scalar, code = struct.unpack_from(' limits['max_correct_changed_percent']: failed.append('correct words changed') if status == 'scored' and restoration['unambiguous']['percent'] < limits['min_accents_restored_percent']: failed.append('unambiguous accents restored') failed += [c['name'] for c in critical if not c['passed']] results.append(dict(language=lang, reference_words=total, raw_errors=raw, noisy_errors=errors, noisy_wer_percent=100 * errors / total, error_reduction_percent=reduction, correct_changed=changed, correct_changed_percent=change_rate, accent_restoration=restoration, unambiguous_status=status, ambiguous_harm=tally(harm, opportunities), critical_cases=critical, capitals=dict(names=tally(names_kept, names), wrongly_capitalised=tally(plain_capitalised, plain)), failed=failed, passed=not failed)) print('Lang Noisy WER Reduction Changed Unambig. Ambig. Harm Critical Gate Names kept Capital added') rate = lambda m: '—' if m['percent'] is None else f"{m['percent']:.2f}%" for r in results: un = rate(r['accent_restoration']['unambiguous']) if r['unambiguous_status'] == 'scored' else r['unambiguous_status'] print(f"{r['language']:4} {r['noisy_wer_percent']:9.2f}% {r['error_reduction_percent']:9.2f}%" f" {r['correct_changed_percent']:8.2f}% {un:>8} {rate(r['accent_restoration']['ambiguous']):>7}" f" {rate(r['ambiguous_harm']):>6} {sum(c['passed'] for c in r['critical_cases']):5}/{len(r['critical_cases'])}" f" {'PASS' if r['passed'] else 'FAIL'}" f" {rate(r['capitals']['names']):>9} {rate(r['capitals']['wrongly_capitalised']):>13}") (out / 'typing-quality.json').write_text(json.dumps(dict(engine='ios-simulator', thresholds=limits, languages=results), ensure_ascii=False, indent=2) + '\n') return int(any(not r['passed'] for r in results)) def run(out, langs, destination): prepare(out, langs) for lang in langs: env = dict(os.environ, TEST_RUNNER_HEX_TYPING_GATE=str(out), TEST_RUNNER_HEX_TYPING_GATE_LANGUAGE=lang) log = out / f'xcodebuild-{lang}.log' with log.open('w') as f: code = subprocess.call(['xcodebuild', 'test', '-project', str(IOS / 'HEXMobile.xcodeproj'), '-scheme', 'HEXMobile', '-destination', destination, '-only-testing:HEXMobileTests/TypingGateReplayTests', '-parallel-testing-enabled', 'NO', 'SWIFT_OPTIMIZATION_LEVEL=-O'], cwd=IOS, env=env, stdout=f, stderr=subprocess.STDOUT) if code or not (out / f'output-{lang}.json').exists(): raise SystemExit(f'{lang}: xcodebuild failed ({code}); see {log}') print(f'{lang}: replayed', file=sys.stderr) return score(out, langs) def main(): parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) parser.add_argument('command', choices=['run', 'prepare', 'score']) parser.add_argument('out', nargs='?', type=Path, default=IOS / 'build/typing-gate') parser.add_argument('languages', nargs='*') parser.add_argument('--destination', help="xcodebuild destination for run, such as 'id='") args = parser.parse_intermixed_args() langs = args.languages or LANGUAGES for lang in langs: if lang not in LANGUAGES: parser.error(f'unknown language {lang}') out = args.out.resolve() if args.command == 'prepare': return prepare(out, langs) if args.command == 'score': return score(out, langs) if not args.destination: parser.error('run needs --destination') return run(out, langs, args.destination) if __name__ == '__main__': raise SystemExit(main())