hex/ios/HEXMobile/DictationController+ModelPreparation.swift
Dax c9fc9563ef
iOS: speech model preparation that never answers fails with Try again (#2372)
Apple's speech daemon can stop answering during preparation: the asset
reservation, the status checks before and after a download, the language lookup,
the network check and the release had no time limit. They are now watched at
60 s with StallWatch's suspension-aware deadline, a reservation or release that
never answers is forgotten so the next attempt asks again, and a call whose
success another caller already gave up on is retried rather than believed. A
keyboard check that only hangs counts no strike toward Hex's handoff.

A preparation a newer one replaces, or one the preparation watch ends, is
cancelled, and the keyboard's publish and handoff paths ignore a cancelled
check, so a late answer can't reopen the row. The keyboard's model check retries
after a real background trip. The multilingual download's restart cancels and
awaits the old transfer before resuming its .partial file, and a preparation
that ends in the background is retried on return.
2026-11-22 12:31:22 -05:00

263 lines
13 KiB
Swift
Raw Permalink Blame History

This file contains ambiguous Unicode characters

This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.

import Foundation
/// Getting the engine Hex dictates with ready: `prepareModel`, and Experiments ›
/// Multilingual model (Parakeet) downloading and loading beside the engine that dictates
/// meanwhile, including what the app does with it in the background.
extension DictationController {
// MARK: - Model Preparation
/// Experiments › Multilingual model (Parakeet) turned on or off. On, the engine Hex
/// dictates with stays while Parakeet Ultra downloads and loads beside it; a load that
/// ended Hex gets another try. Off, the download stops and is deleted, and Hex prepares
/// the engine it picks without it if Ultra was the one dictating (now, or once the
/// dictation in flight is done).
func multilingualParakeetChanged() {
let settings = MultilingualParakeetSettings()
if settings.isEnabled {
ParakeetLoadGuard(model: ParakeetUltraTranscriber.loadGuardModel).forget()
multilingualDownload.settle()
if isModelReady || phase == .preparingModel {
syncMultilingualParakeet()
return
}
} else {
settings.isModelReady = false
let download = multilingualDownload
Task {
// Nothing writes to the folder any more once `stop` returns.
await download.stop()
ParakeetUltraModel.remove()
}
if engineName != ParakeetUltraTranscriber.name, isModelReady, phase != .preparingModel {
// Ultra wasn't dictating: drop a load of it beside the engine that is.
modelPrepGeneration &+= 1
return
}
}
switch phase {
case .recording, .transcribing, .requestingPermission:
pendingLanguagePreparation = true
case .modelRequired, .preparingModel, .ready, .failed:
prepareModel()
}
}
/// `fallbacksLeft` bounds how often Automatic moves on to another language after one
/// whose model won't install here; each move sets a language aside, so it can't loop.
func prepareModel(fallbacksLeft: Int = 2, candidate preset: (any LocalTranscribing)? = nil) {
modelPrepGeneration &+= 1
let prepGen = modelPrepGeneration
// The preparation this replaces may still be inside a Speech call: cancel it so its work
// stops instead of piling up behind the new one.
preparationTask?.cancel()
// A load beside the engine this replaces is dropped; the end of this one decides again.
multilingualDownload.dropLoad()
startMultilingualDownloadIfWanted()
phase = .preparingModel
copied = false
preparingMultilingualParakeet = false
modelPreparation = ModelPreparationProgress(
fractionCompleted: 0,
detail: String(localized: "Checking the speech model")
)
preparationFailedInBackground = false
watchPreparation(prepGen)
// A few more seconds from iOS when Hex goes to the background mid-download or mid-compile.
let backgroundTime = BackgroundTime("Prepare speech model")
preparationTask = Task {
defer {
backgroundTime.end()
if self.modelPrepGeneration == prepGen { self.preparationTask = nil }
}
do {
let candidate = try await self.prepareCandidate(preset, generation: prepGen)
guard self.modelPrepGeneration == prepGen else { return }
self.languageOffer = nil
self.transcriber = candidate
self.transcriptionLanguage = await candidate.languageName
self.transcriptionLanguageID = await candidate.languageIdentifier
self.keyboardBridge.setModelInstalled(true)
self.isModelReady = true
self.phase = .ready
if self.preparingMultilingualParakeet {
self.multilingualParakeetLoaded()
}
self.publishKeyboardSnapshot()
if self.pendingHandoff != nil {
self.activateWarmSessionAndRunHandoff()
}
self.syncMultilingualParakeet()
} catch {
guard self.modelPrepGeneration == prepGen else { return }
if self.preparingMultilingualParakeet, fallbacksLeft > 0, !(error is CancellationError) {
// Parakeet Ultra didn't load: the engine Hex uses without it dictates, and
// Hex says why, with Try again.
self.multilingualParakeetFailedToLoad(error)
self.prepareModel(fallbacksLeft: fallbacksLeft - 1)
return
}
let unavailable = error as? SpeechModelUnavailable
if fallbacksLeft > 0, let notice = Self.switchNotice(for: error) {
// Automatic set the language aside, or Parakeet had no room: prepare what
// Hex picks now.
self.languageNotice = notice
self.prepareModel(fallbacksLeft: fallbacksLeft - 1)
return
}
self.languageOffer = unavailable.flatMap { $0.automatic ? nil : $0.fallback }
self.preparationFailedInBackground = self.isInBackground && Self.isDownloadInterruption(error)
// This only differs from Parakeet's cache in one case: a re-preparation that
// fails after an earlier success, where the engine that prepared before is still
// loaded and can dictate. A first failure on Apple Speech has no model to keep.
self.keyboardBridge.setModelInstalled(self.isModelReady || ParakeetTranscriber.hasCachedModel)
self.pendingHandoff = nil
self.phase = .failed(UserFacingError.message(for: error, context: .model))
self.publishKeyboardSnapshot()
// A downloaded Parakeet Ultra can still dictate.
self.syncMultilingualParakeet()
}
}
}
/// Ends a preparation that has said nothing new for too long, in plain words with Try
/// again (Set up), instead of a bar that never moves: `firstReport` (30 s) before it has
/// said anything (Apple's speech service or asset store not answering leaves "Checking
/// the speech model" up for good, TestFlight build 126), `stall` (120 s) after, longer
/// than the downloads' own stall limits. Time while iOS had Hex suspended doesn't count.
/// The work it ends can't always be cancelled (an XPC call that never returns); its
/// result is dropped, and Try again starts a new one.
func watchPreparation(_ prepGen: UInt64) {
preparationChangedAt = .now
preparationReported = false
Task { [weak self] in
let clock = ContinuousClock()
var polled = clock.now
while true {
try? await Task.sleep(for: .milliseconds(500))
guard let self, self.modelPrepGeneration == prepGen, self.phase == .preparingModel else { return }
let now = clock.now
if now - polled > .seconds(5) {
self.preparationChangedAt = now
}
polled = now
let limit = self.preparationReported ? self.preparationLimits.stall : self.preparationLimits.firstReport
guard now - self.preparationChangedAt >= limit else { continue }
print("speech model preparation said nothing new for \(limit): ended")
self.modelPrepGeneration &+= 1
self.preparationTask?.cancel()
self.preparationTask = nil
self.preparationFailedInBackground = self.isInBackground
self.pendingHandoff = nil
self.keyboardBridge.setModelInstalled(self.isModelReady || ParakeetTranscriber.hasCachedModel)
self.phase = .failed(String(localized: "The speech model isn't ready. Try again."))
self.publishKeyboardSnapshot()
return
}
}
}
/// Resolves the engine and gets it ready, naming it before its model loads so the window
/// and the keyboard already say which one is coming. Throws `CancellationError` when a newer
/// preparation replaced this one before it picked.
private func prepareCandidate(
_ preset: (any LocalTranscribing)?,
generation: UInt64
) async throws -> any LocalTranscribing {
let candidate: any LocalTranscribing
if let preset {
candidate = preset
} else {
candidate = await transcriberFactory()
}
guard modelPrepGeneration == generation else { throw CancellationError() }
try Task.checkCancellation()
engineName = candidate.displayName
preparingMultilingualParakeet = candidate is ParakeetUltraTranscriber
try await candidate.prepare { [weak self] progress in
Task { @MainActor [weak self] in
guard let self, self.modelPrepGeneration == generation else { return }
if progress.fractionCompleted != self.modelPreparation.fractionCompleted
|| progress.detail != self.modelPreparation.detail {
self.preparationChangedAt = .now
self.preparationReported = true
}
self.modelPreparation = progress
}
}
return candidate
}
/// Parakeet V2 dictates and leaves no room to load Parakeet Ultra beside it (English
/// before iOS 26): Ultra takes its place and loads as it would at launch, rather than
/// never getting its first load. False when V2 isn't the engine.
func replaceParakeetWithMultilingual(language: String) -> Bool {
guard transcriber is ParakeetTranscriber, phase != .preparingModel else { return false }
transcriber = ParakeetTranscriber()
isModelReady = false
prepareModel(candidate: ParakeetUltraTranscriber(language: language))
return true
}
/// Loads Parakeet Ultra while the engine Hex has keeps dictating, and switches to it once
/// it's loaded; a dictation in flight isn't interrupted. A failure keeps that engine and
/// says why. Only while no other preparation runs.
func loadMultilingualParakeetBeside(language: String) {
guard phase != .preparingModel else { return }
modelPrepGeneration &+= 1
let prepGen = modelPrepGeneration
multilingualDownload.loading()
let backgroundTime = BackgroundTime("Prepare the multilingual model")
Task {
defer { backgroundTime.end() }
let candidate = ParakeetUltraTranscriber(language: language)
do {
try await candidate.prepare { _ in }
guard self.modelPrepGeneration == prepGen else { return }
self.transcriber = candidate
self.engineName = candidate.displayName
self.transcriptionLanguage = await candidate.languageName
self.transcriptionLanguageID = await candidate.languageIdentifier
self.languageOffer = nil
self.isModelReady = true
self.keyboardBridge.setModelInstalled(true)
switch self.phase {
case .modelRequired, .failed:
self.phase = .ready
default:
break
}
self.multilingualParakeetLoaded()
self.publishKeyboardSnapshot()
} catch {
guard self.modelPrepGeneration == prepGen else { return }
self.multilingualParakeetFailedToLoad(error)
}
}
}
func appMovedToBackground() {
isInBackground = true
multilingualDownload.appMovedToBackground()
}
/// Picks up a download that iOS cut off while Hex was in the background; FluidAudio
/// resumes each file from the bytes it has.
func appBecameActive() {
isInBackground = false
refreshWarmDuration()
multilingualDownload.appBecameActive()
guard preparationFailedInBackground else { return }
preparationFailedInBackground = false
if case .failed = phase, !isModelReady {
prepareModel()
}
}
nonisolated static func isDownloadInterruption(_ error: Error) -> Bool {
if error is StallWatch.Stalled || error is URLError || error is SpeechModelUnresponsive {
return true
}
return (error as? ParakeetTranscriber.TranscriberError) == .downloadFailed
}
}