mirror of
https://github.com/anomalyco/hex.git
synced 2026-11-22 21:23:02 +01:00
What changed - New experiment, off by default (KeyboardParakeetSettings, App Group): the keyboard's own dictation transcribes with Parakeet on the Neural Engine in the keyboard's process (HEXKeyboard/KeyboardParakeet.swift): V2 for English, Ultra for a language the Multilingual model experiment dictates once its files are there (Dictate and the Hex key then stop opening Hex for it, once the keyboard has it loaded). Everything after transcription (styles, cleanup, punctuation, insertion, held results) is the same code path; KeyboardSpeech is unchanged and is what runs whenever Parakeet can't. - The keyboard loads the model after its first frame and launch work, off the main thread (preprocessor on the CPU, the rest on the CPU and Neural Engine, one chunk at a time, never the GPU), keeps it while its process lives and releases it on a memory warning. It falls back to Apple's speech recognition, silently, when the model is missing, still loading (the first compile), not on the Neural Engine (neural ledger, else Core ML's compute plan), short of memory (KeyboardParakeetBudget: limit = footprint + os_proc_available_memory must be 96 MB for V2 / 115 MB for Ultra, with headroom to load and to transcribe), busy, empty or failing. - Parakeet's files move from the app's Application Support into the App Group (SpeechModelStore, Library/Application Support/SpeechModels), renamed in place at launch (moveLegacyDownloads, partial downloads included) instead of downloaded again; folders are marked complete (.hex-complete) and the keyboard loads only a marked one. ParakeetTranscriber and ParakeetUltraModel point there; turning the multilingual model off still deletes it (and any old copy). - Turning the switch on downloads Parakeet V2 (464 MB, KeyboardParakeetDownload, FluidAudio's resumable download); off stops and deletes it unless Hex itself dictates with V2. The row says what is happening in a few words (Downloading the speech model, Downloaded. The keyboard gets it ready the next time it opens, Getting ready in the keyboard, Ready in the keyboard, or that the keyboard can't use it now), with Simplified Chinese; the keyboard writes its state to the App Group, and "getting ready" shows only while fresh (3 min). - Each keyboard dictation with the switch on logs its engine, the keyboard's limit, footprint before and at peak, load and transcription time (keyboard-capture-probe.log / os_log). - FluidAudio is linked into the keyboard target. README updated. Why - Ryan: a better model than Apple's should run from the keyboard with the Hex app not running. The probe (~/dev/experiments/2026-11-22-hex-keyboard-asr/native) showed it fits: Neural Engine memory is outside the keyboard's footprint, the limit on an iPhone 17 Pro is 177 MB. Verified - Studio simulator (iPhone 17 Pro, iOS 27.2, fresh): xcodebuild test, 856 tests, 0 failures, 3 skipped (as on dev). scripts/code-shape.py passes. - iPhone 17 Pro, iOS 27.2, Release, Hex app force-quit: switch on, 464 MB downloaded, first load in Hex's keyboard 15.4 s (compiling), then a dictation from the Mac's speakers: engine=parakeet-v2 limit=177 MB footprint before=56.1 MB peak=72.0 MB transcribe=0.28 s for 9.4 s of audio, neural 468 MB, text exact. Switch off: the same clip with Apple's speech recognition, 0.34 s, footprint as on dev; the model's files deleted. - Switch off against dev (12+12 cold keyboard launches, Release): first appearance unchanged (median 491 vs 479 ms), footprint at launch work +0.2 MB (18.3 -> 18.5 MB median), keyboard binary 5.92 -> 12.64 MB. The 0.2 MB is FluidAudio's writable data (164 KB __DATA + 49 KB __DATA_CONST) that dyld fixes up at every launch; accepted for the experiment (follow-up: load the Parakeet code from a framework only when the switch is on).
164 lines
No EOL
5.9 KiB
YAML
164 lines
No EOL
5.9 KiB
YAML
name: HEXMobile
|
||
options:
|
||
minimumXcodeGenVersion: 2.46.0
|
||
bundleIdPrefix: anomaly.hex
|
||
deploymentTarget:
|
||
iOS: "18.0"
|
||
xcodeVersion: "2660"
|
||
packages:
|
||
FluidAudio:
|
||
url: https://github.com/FluidInference/FluidAudio.git
|
||
exactVersion: 0.15.5
|
||
settings:
|
||
base:
|
||
DEVELOPMENT_TEAM: 5NZ4Q7NXJ4
|
||
SWIFT_VERSION: 6.0
|
||
# The compiler finds the interface's strings for the String Catalogs (Localizable.xcstrings).
|
||
SWIFT_EMIT_LOC_STRINGS: YES
|
||
LOCALIZATION_PREFERS_STRING_CATALOGS: YES
|
||
targets:
|
||
HEXMobile:
|
||
type: application
|
||
platform: iOS
|
||
sources:
|
||
- HEXMobile
|
||
- HEXShared
|
||
- HEXActivity
|
||
- path: ../app/AppIcon.icon
|
||
buildPhase: resources
|
||
dependencies:
|
||
- package: FluidAudio
|
||
- target: HEXKeyboard
|
||
embed: true
|
||
- target: HEXWidgets
|
||
embed: true
|
||
info:
|
||
path: HEXMobile/Info.plist
|
||
properties:
|
||
CFBundleDisplayName: Hex
|
||
CFBundleShortVersionString: $(MARKETING_VERSION)
|
||
CFBundleVersion: $(CURRENT_PROJECT_VERSION)
|
||
CADisableMinimumFrameDurationOnPhone: true
|
||
CFBundleURLTypes:
|
||
- CFBundleTypeRole: Editor
|
||
CFBundleURLName: anomaly.hex.mobile.keyboard-session
|
||
CFBundleURLSchemes:
|
||
- hex-dictation
|
||
NSMicrophoneUsageDescription: Hex turns what you say into text on this iPhone. Audio never leaves it and is deleted once it's transcribed.
|
||
NSSpeechRecognitionUsageDescription: Hex uses Apple's on-device speech recognition to turn what you say into text. Nothing you say is sent to a server.
|
||
NSSupportsLiveActivities: true
|
||
ITSAppUsesNonExemptEncryption: false
|
||
UIBackgroundModes:
|
||
- audio
|
||
UILaunchScreen:
|
||
UIColorName: LaunchBackground
|
||
UISupportedInterfaceOrientations:
|
||
- UIInterfaceOrientationPortrait
|
||
- UIInterfaceOrientationLandscapeLeft
|
||
- UIInterfaceOrientationLandscapeRight
|
||
UISupportedInterfaceOrientations~ipad:
|
||
- UIInterfaceOrientationPortrait
|
||
- UIInterfaceOrientationPortraitUpsideDown
|
||
- UIInterfaceOrientationLandscapeLeft
|
||
- UIInterfaceOrientationLandscapeRight
|
||
settings:
|
||
base:
|
||
ASSETCATALOG_COMPILER_ACCENT_COLOR_NAME: AccentColor
|
||
ASSETCATALOG_COMPILER_APPICON_NAME: AppIcon
|
||
CODE_SIGN_STYLE: Automatic
|
||
CODE_SIGN_ENTITLEMENTS: HEXMobile/HEXMobile.entitlements
|
||
CURRENT_PROJECT_VERSION: 1
|
||
GENERATE_INFOPLIST_FILE: NO
|
||
MARKETING_VERSION: 9.0.0
|
||
TARGETED_DEVICE_FAMILY: "1"
|
||
OTHER_LDFLAGS: $(inherited) -weak_framework FoundationModels
|
||
OTHER_SWIFT_FLAGS: $(inherited) -Xfrontend -disable-autolink-framework -Xfrontend FoundationModels
|
||
PRODUCT_BUNDLE_IDENTIFIER: anomaly.hex.mobile
|
||
PRODUCT_NAME: HEX
|
||
scheme:
|
||
gatherCoverageData: false
|
||
testTargets:
|
||
- HEXMobileTests
|
||
HEXKeyboard:
|
||
type: app-extension
|
||
platform: iOS
|
||
sources:
|
||
- HEXKeyboard
|
||
- HEXShared
|
||
# The app's String Catalog, so the keyboard's words (the Hex key menu first) follow
|
||
# the iPhone's language where Hex is translated.
|
||
- HEXMobile/Localizable.xcstrings
|
||
dependencies:
|
||
# Experiments › Speech model in keyboard: Parakeet in the keyboard (KeyboardParakeet).
|
||
- package: FluidAudio
|
||
info:
|
||
path: HEXKeyboard/Info.plist
|
||
properties:
|
||
CFBundleDisplayName: Hex Keyboard
|
||
NSMicrophoneUsageDescription: Hex turns what you say into text on this iPhone. Audio never leaves it and is deleted once it's transcribed.
|
||
NSSpeechRecognitionUsageDescription: Hex uses Apple's on-device speech recognition to turn what you say into text. Nothing you say is sent to a server.
|
||
CFBundleShortVersionString: $(MARKETING_VERSION)
|
||
CFBundleVersion: $(CURRENT_PROJECT_VERSION)
|
||
NSExtension:
|
||
NSExtensionAttributes:
|
||
IsASCIICapable: true
|
||
PrefersRightToLeft: false
|
||
PrimaryLanguage: en-US
|
||
RequestsOpenAccess: true
|
||
NSExtensionPointIdentifier: com.apple.keyboard-service
|
||
NSExtensionPrincipalClass: $(PRODUCT_MODULE_NAME).KeyboardViewController
|
||
settings:
|
||
base:
|
||
APPLICATION_EXTENSION_API_ONLY: YES
|
||
CODE_SIGN_ENTITLEMENTS: HEXKeyboard/HEXKeyboard.entitlements
|
||
COREML_CODEGEN_LANGUAGE: None
|
||
CODE_SIGN_STYLE: Automatic
|
||
CURRENT_PROJECT_VERSION: 1
|
||
GENERATE_INFOPLIST_FILE: NO
|
||
MARKETING_VERSION: 9.0.0
|
||
TARGETED_DEVICE_FAMILY: "1"
|
||
PRODUCT_BUNDLE_IDENTIFIER: anomaly.hex.mobile.keyboard
|
||
PRODUCT_NAME: HEXKeyboard
|
||
SKIP_INSTALL: YES
|
||
HEXWidgets:
|
||
type: app-extension
|
||
platform: iOS
|
||
sources:
|
||
- HEXWidgets
|
||
- HEXActivity
|
||
- HEXShared/HexTheme.swift
|
||
info:
|
||
path: HEXWidgets/Info.plist
|
||
properties:
|
||
CFBundleDisplayName: Hex
|
||
CFBundleShortVersionString: $(MARKETING_VERSION)
|
||
CFBundleVersion: $(CURRENT_PROJECT_VERSION)
|
||
NSExtension:
|
||
NSExtensionPointIdentifier: com.apple.widgetkit-extension
|
||
settings:
|
||
base:
|
||
APPLICATION_EXTENSION_API_ONLY: YES
|
||
CODE_SIGN_STYLE: Automatic
|
||
CURRENT_PROJECT_VERSION: 1
|
||
GENERATE_INFOPLIST_FILE: NO
|
||
MARKETING_VERSION: 9.0.0
|
||
TARGETED_DEVICE_FAMILY: "1"
|
||
PRODUCT_BUNDLE_IDENTIFIER: anomaly.hex.mobile.widgets
|
||
PRODUCT_NAME: HEXWidgets
|
||
SKIP_INSTALL: YES
|
||
HEXMobileTests:
|
||
type: bundle.unit-test
|
||
platform: iOS
|
||
sources:
|
||
- HEXMobileTests
|
||
- path: ../tests/fixtures/vocabulary/audio/term-hex-1.wav
|
||
buildPhase: resources
|
||
dependencies:
|
||
- target: HEXMobile
|
||
settings:
|
||
base:
|
||
BUNDLE_LOADER: $(TEST_HOST)
|
||
CODE_SIGN_STYLE: Automatic
|
||
GENERATE_INFOPLIST_FILE: YES
|
||
PRODUCT_BUNDLE_IDENTIFIER: anomaly.hex.mobile.tests
|
||
TEST_HOST: $(BUILT_PRODUCTS_DIR)/HEX.app/$(BUNDLE_EXECUTABLE_FOLDER_PATH)/HEX |