hex/src/transcription/gpu_health.rs
Dax 1ef7c54d02
windows: retry the GPU once after a gpu-start crash, and show where speech runs (#2351)
gpu-start.pending now holds the build's version and the crashed starts under
it: a launch that crashed starting the GPU is retried once, and a second crash
keeps the CPU until Auto or GPU is picked again in Settings › Speech engine,
or the next Hex update. Re-picking clears the marker even when the setting
already says GPU or this launch chose the CPU, and a marker from another build
is deleted when it is read.

Settings › Speech engine says when the crash forced the CPU, and `hex status`
prints the loaded model's backend ("Vulkan — AMD Radeon RX 9060 XT", or "CPU
(GPU start crashed twice)").
2026-11-22 05:53:58 -05:00

447 lines
16 KiB
Rust

//! GPUs that hung or failed while Hex transcribed on them (#856), remembered
//! per device and driver in `gpu-failures.json` in Application Support, so
//! Auto keeps that GPU off and transcribes on the CPU (or another GPU) until
//! the driver changes. A GPU context reset or device loss comes back from
//! ggml as an error, never as something Hex could retry safely on the same
//! device: ggml keeps the lost Vulkan device for the process, and asking a
//! hung driver again can hang the whole desktop's GPU again.
use std::path::PathBuf;
use std::sync::Mutex;
use std::sync::atomic::{AtomicBool, Ordering};
use serde::{Deserialize, Serialize};
use transcribe_cpp::{Backend, Device, DeviceType};
/// At most this many remembered failures; the oldest go first.
const KEPT: usize = 16;
#[derive(Clone, Debug, Deserialize, PartialEq, Serialize)]
pub(crate) struct Failure {
/// The device: its description, kind, and hardware id.
pub(crate) device: String,
/// What its driver was: [`driver`] when it failed.
pub(crate) driver: String,
/// The error, for the log and Settings.
pub(crate) reason: String,
/// Seconds since the Unix epoch.
pub(crate) at: u64,
/// It worked, but slower than the CPU (#484), rather than failing.
#[serde(default, skip_serializing_if = "std::ops::Not::not")]
pub(crate) slow: bool,
}
/// The failures on disk, read once.
static FAILURES: Mutex<Option<Vec<Failure>>> = Mutex::new(None);
fn path() -> Option<PathBuf> {
crate::app_paths::support_dir()
.ok()
.map(|dir| dir.join("gpu-failures.json"))
}
fn with_failures<R>(work: impl FnOnce(&mut Vec<Failure>) -> R) -> R {
let mut failures = FAILURES
.lock()
.unwrap_or_else(|poisoned| poisoned.into_inner());
let failures = failures.get_or_insert_with(|| {
path()
.and_then(|path| std::fs::read(path).ok())
.and_then(|bytes| serde_json::from_slice(&bytes).ok())
.unwrap_or_default()
});
work(failures)
}
fn save(failures: &[Failure]) {
let Some(path) = path() else { return };
let written = (|| -> std::io::Result<()> {
if let Some(dir) = path.parent() {
std::fs::create_dir_all(dir)?;
}
let staged = path.with_extension(format!("json.{}", std::process::id()));
std::fs::write(&staged, serde_json::to_vec_pretty(failures)?)?;
std::fs::rename(&staged, &path)
})();
if let Err(error) = written {
tracing::warn!(%error, path = %path.display(), "could not save the GPU failure record");
}
}
/// `device` as the failure record names it.
pub(crate) fn device_key(device: &Device) -> String {
format!(
"{} ({}, {})",
device.description,
device.kind,
device.device_id.as_deref().unwrap_or("-")
)
}
/// Whether `device` failed under the driver installed now.
pub(crate) fn failed(device: &Device) -> bool {
let (key, driver) = (device_key(device), driver());
with_failures(|failures| {
failures
.iter()
.any(|failure| failure.device == key && failure.driver == driver)
})
}
/// Remembers that `device` failed under the driver installed now.
pub(crate) fn record(device: &Device, reason: &str) {
remember(device, reason, false);
}
/// Remembers that `device` ran the model slower than the CPU under the
/// driver installed now (#484).
pub(crate) fn record_slow(device: &Device, reason: &str) {
remember(device, reason, true);
}
/// What to remember or show about `device` failing for `reason` now.
fn snapshot(device: String, reason: &str, slow: bool) -> Failure {
Failure {
device,
driver: driver(),
reason: reason.chars().take(400).collect(),
at: std::time::SystemTime::now()
.duration_since(std::time::UNIX_EPOCH)
.map_or(0, |elapsed| elapsed.as_secs()),
slow,
}
}
fn remember(device: &Device, reason: &str, slow: bool) {
let failure = snapshot(device_key(device), reason, slow);
if slow {
tracing::info!(
device = failure.device,
driver = failure.driver,
reason = failure.reason,
"the CPU runs the model faster than this GPU; Auto uses the CPU on it until its driver changes"
);
} else {
tracing::warn!(
device = failure.device,
driver = failure.driver,
reason = failure.reason,
"the GPU failed while transcribing; Auto transcribes on the CPU on it until its driver changes"
);
}
with_failures(|failures| {
failures.retain(|old| old.device != failure.device || old.driver != failure.driver);
failures.push(failure);
let excess = failures.len().saturating_sub(KEPT);
failures.drain(..excess);
save(failures);
});
}
/// Forgets `device`'s failure under the driver installed now, after it
/// worked when chosen outright.
pub(crate) fn forget(device: &Device) {
let (key, driver) = (device_key(device), driver());
with_failures(|failures| {
let before = failures.len();
failures.retain(|failure| failure.device != key || failure.driver != driver);
if failures.len() != before {
tracing::info!(device = key, "the GPU worked again; Auto uses it again");
save(failures);
}
});
}
/// Whether the last model Auto loaded runs on the CPU. Another healthy GPU
/// can run it while a failed one stays off, so a remembered failure alone
/// doesn't mean Auto uses the CPU (#966).
static AUTO_ON_CPU: AtomicBool = AtomicBool::new(false);
/// Records where the last model loaded: on the CPU under Auto, or not.
pub(crate) fn set_auto_on_cpu(on_cpu: bool) {
AUTO_ON_CPU.store(on_cpu, Ordering::Relaxed);
}
/// The newest failure that keeps a GPU off now, for Settings, only while
/// Auto actually runs the model on the CPU.
pub(crate) fn current() -> Option<Failure> {
if !AUTO_ON_CPU.load(Ordering::Relaxed) {
return None;
}
let driver = driver();
with_failures(|failures| {
failures
.iter()
.rev()
.find(|failure| failure.driver == driver)
.cloned()
})
}
/// The dictation engine's model switched to the CPU after a GPU ran out of
/// memory while transcribing. Only the engine publishes it, so another copy
/// in this process (the local API's own model, model preparation, or a
/// superseded model swap) cannot speak for dictation. It is never written to
/// `gpu-failures.json`: another copy of a model can fill a GPU (#1165), and
/// ggml keeps the failed device for the process, so a later GPU load or a
/// restart is what gets the GPU back.
static OUT_OF_MEMORY: Mutex<Option<Failure>> = Mutex::new(None);
/// What to remember about `device` running out of memory while transcribing,
/// for the model to carry to the engine.
pub(crate) fn out_of_memory_failure(device: &Device, reason: &str) -> Failure {
snapshot(device_key(device), reason, false)
}
/// Records what the dictation engine's model says: the out-of-memory
/// fallback it took, or `None` once it runs on a GPU again.
pub(crate) fn set_out_of_memory(fallback: Option<&Failure>) {
let mut current = OUT_OF_MEMORY
.lock()
.unwrap_or_else(|poisoned| poisoned.into_inner());
if current.as_ref() == fallback {
return;
}
let had = current.is_some();
*current = fallback.cloned();
match &*current {
Some(failure) => tracing::warn!(
device = failure.device,
reason = failure.reason,
"the GPU ran out of memory while transcribing; the dictation engine uses the CPU until a GPU load works or Hex restarts"
),
None if had => tracing::info!(
"the dictation engine loaded the model on a GPU again; forgetting the out-of-memory fallback"
),
None => {}
}
}
/// The dictation engine's out-of-memory fallback, which keeps this process on
/// the CPU now, for Settings. Never a record read from disk.
pub(crate) fn out_of_memory_fallback() -> Option<Failure> {
OUT_OF_MEMORY
.lock()
.unwrap_or_else(|poisoned| poisoned.into_inner())
.clone()
}
/// This launch registered the CPU alone because the GPU crashed twice
/// starting for transcription (Windows' `gpu-start.pending`, #662). Only that
/// process knows: Settings and `hex status` read it here.
static CRASH_FALLBACK: AtomicBool = AtomicBool::new(false);
/// Records that this launch runs the CPU because the GPU crashed twice
/// starting. Only the Windows marker forces it.
#[cfg(any(target_os = "windows", test))]
pub(crate) fn set_crash_fallback() {
CRASH_FALLBACK.store(true, Ordering::Relaxed);
}
/// Whether this launch runs the CPU because the GPU crashed twice starting.
pub(crate) fn crash_fallback() -> bool {
CRASH_FALLBACK.load(Ordering::Relaxed)
}
/// The GPUs ggml can run on, best first: discrete before integrated.
pub(crate) fn gpus() -> Vec<Device> {
let mut gpus = transcribe_cpp::devices()
.into_iter()
.filter(|device| matches!(device.device_type, DeviceType::Gpu | DeviceType::Igpu))
.collect::<Vec<_>>();
gpus.sort_by_key(|device| device.device_type != DeviceType::Gpu);
gpus
}
/// The backend that runs on `device` alone.
pub(crate) fn backend(device: &Device) -> Option<Backend> {
match device.kind.as_str() {
"vulkan" => Some(Backend::Vulkan),
"cuda" => Some(Backend::Cuda),
"rocm" => Some(Backend::Rocm),
"metal" => Some(Backend::Metal),
_ => None,
}
}
/// Whether `error` says the device failed, rather than the request or the
/// model: the errors a hung or lost GPU, or one out of memory, returns.
pub(crate) fn device_error(error: &color_eyre::eyre::Report) -> bool {
use transcribe_cpp::Error;
error.chain().any(|cause| {
cause.downcast_ref::<Error>().is_some_and(|error| {
matches!(
error,
Error::Backend(_) | Error::OutOfMemory(_) | Error::ModelLoad(_) | Error::Other(_)
)
})
})
}
/// Whether `error` says the GPU ran out of memory. That is not a driver
/// fault: another copy of a model (a model switch loading beside the running
/// one, or a CLI check while Hex holds the model) can fill a small GPU, so
/// the GPU is not remembered as failed for it (#1165).
pub(crate) fn out_of_memory(error: &color_eyre::eyre::Report) -> bool {
error.chain().any(|cause| {
if matches!(
cause.downcast_ref::<transcribe_cpp::Error>(),
Some(transcribe_cpp::Error::OutOfMemory(_))
) {
return true;
}
let message = cause.to_string().to_ascii_lowercase();
message.contains("out of memory")
|| message.contains("outofdevicememory")
|| message.contains("out_of_device_memory")
|| message.contains("failed to allocate")
})
}
/// What a GPU's driver is now, so an update tries it again. Linux: the
/// kernel (i915, xe, amdgpu, nouveau live there) and every installed Vulkan
/// driver's library, by size and modification time (a Mesa or vendor
/// update replaces them). Windows: this build of Hex.
pub(crate) fn driver() -> String {
static DRIVER: std::sync::OnceLock<String> = std::sync::OnceLock::new();
DRIVER.get_or_init(driver_now).clone()
}
#[cfg(target_os = "linux")]
fn driver_now() -> String {
use std::hash::{Hash, Hasher};
use std::os::unix::fs::MetadataExt;
let kernel = std::fs::read_to_string("/proc/sys/kernel/osrelease").unwrap_or_default();
let mut directories = vec![PathBuf::from("/etc/vulkan/icd.d")];
let data = std::env::var("XDG_DATA_DIRS")
.ok()
.filter(|dirs| !dirs.is_empty())
.unwrap_or_else(|| "/usr/local/share:/usr/share".into());
directories.extend(
data.split(':')
.filter(|dir| !dir.is_empty())
.map(|dir| PathBuf::from(dir).join("vulkan/icd.d")),
);
let mut libraries = Vec::new();
for directory in directories {
let Ok(entries) = std::fs::read_dir(&directory) else {
continue;
};
for entry in entries.flatten() {
let manifest = entry.path();
let library = std::fs::read(&manifest)
.ok()
.and_then(|bytes| serde_json::from_slice::<serde_json::Value>(&bytes).ok())
.and_then(|json| json["ICD"]["library_path"].as_str().map(PathBuf::from));
let Some(library) = library else { continue };
let library = if library.is_absolute() {
library
} else if library.components().count() > 1 {
directory.join(library)
} else {
// A bare name the loader finds on the library path.
[
"/usr/lib",
"/usr/lib64",
"/usr/lib/x86_64-linux-gnu",
"/run/opengl-driver/lib",
]
.iter()
.map(|dir| PathBuf::from(dir).join(&library))
.find(|path| path.is_file())
.unwrap_or(library)
};
let stamp = std::fs::metadata(&library)
.map(|metadata| (metadata.len(), metadata.mtime()))
.unwrap_or_default();
libraries.push((library, stamp));
}
}
libraries.sort();
libraries.dedup();
let mut hasher = std::collections::hash_map::DefaultHasher::new();
libraries.hash(&mut hasher);
format!("{} vulkan-{:016x}", kernel.trim(), hasher.finish())
}
#[cfg(not(target_os = "linux"))]
fn driver_now() -> String {
crate::VERSION.to_owned()
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn only_device_errors_count() {
use color_eyre::eyre::WrapErr;
let lost: color_eyre::Result<()> =
Err(transcribe_cpp::Error::Backend("device lost".into()))
.wrap_err("transcription failed");
assert!(device_error(&lost.unwrap_err()));
let short: color_eyre::Result<()> =
Err(transcribe_cpp::Error::InputTooShort("x".into())).wrap_err("transcription failed");
assert!(!device_error(&short.unwrap_err()));
assert!(!device_error(&color_eyre::eyre::eyre!("cancelled")));
}
#[test]
fn out_of_memory_is_not_a_driver_fault() {
use color_eyre::eyre::WrapErr;
let vulkan: color_eyre::Result<()> = Err(transcribe_cpp::Error::Backend(
"vk::Device::allocateMemory: ErrorOutOfDeviceMemory".into(),
))
.wrap_err("could not load");
assert!(out_of_memory(&vulkan.unwrap_err()));
let lost: color_eyre::Result<()> =
Err(transcribe_cpp::Error::Backend("device lost".into())).wrap_err("failed");
assert!(!out_of_memory(&lost.unwrap_err()));
}
#[test]
fn no_note_while_auto_runs_on_a_gpu() {
// A healthy GPU runs the model while another one's failure is
// remembered (#966): Settings names no failure.
set_auto_on_cpu(false);
assert_eq!(current(), None);
}
#[test]
fn a_crash_fallback_lasts_this_launch() {
// The marker forced this process onto the CPU: only a restart (or a
// GPU chosen in Settings) tries the GPU again (#662).
assert!(!crash_fallback());
set_crash_fallback();
assert!(crash_fallback());
}
#[test]
fn an_out_of_memory_fallback_lasts_until_a_gpu_load_works() {
// Not `gpu-failures.json`: the engine publishes its model's fallback
// and a later GPU load clears it.
let failure = Failure {
device: "Test GPU (gpu, 0000-0000)".into(),
driver: driver(),
reason: "out of memory".into(),
at: 0,
slow: false,
};
set_out_of_memory(None);
assert!(out_of_memory_fallback().is_none());
set_out_of_memory(Some(&failure));
assert_eq!(out_of_memory_fallback(), Some(failure.clone()));
// Publishing the same fallback again changes nothing.
set_out_of_memory(Some(&failure));
assert_eq!(out_of_memory_fallback(), Some(failure));
set_out_of_memory(None);
assert!(out_of_memory_fallback().is_none());
}
#[cfg(target_os = "linux")]
#[test]
fn the_driver_names_the_kernel() {
let kernel = std::fs::read_to_string("/proc/sys/kernel/osrelease").unwrap();
assert!(driver().starts_with(kernel.trim()));
}
}