mirror of
https://github.com/anomalyco/hex.git
synced 2026-11-22 21:23:02 +01:00
gpu-start.pending now holds the build's version and the crashed starts under
it: a launch that crashed starting the GPU is retried once, and a second crash
keeps the CPU until Auto or GPU is picked again in Settings › Speech engine,
or the next Hex update. Re-picking clears the marker even when the setting
already says GPU or this launch chose the CPU, and a marker from another build
is deleted when it is read.
Settings › Speech engine says when the crash forced the CPU, and `hex status`
prints the loaded model's backend ("Vulkan — AMD Radeon RX 9060 XT", or "CPU
(GPU start crashed twice)").
447 lines
16 KiB
Rust
447 lines
16 KiB
Rust
//! GPUs that hung or failed while Hex transcribed on them (#856), remembered
|
|
//! per device and driver in `gpu-failures.json` in Application Support, so
|
|
//! Auto keeps that GPU off and transcribes on the CPU (or another GPU) until
|
|
//! the driver changes. A GPU context reset or device loss comes back from
|
|
//! ggml as an error, never as something Hex could retry safely on the same
|
|
//! device: ggml keeps the lost Vulkan device for the process, and asking a
|
|
//! hung driver again can hang the whole desktop's GPU again.
|
|
|
|
use std::path::PathBuf;
|
|
use std::sync::Mutex;
|
|
use std::sync::atomic::{AtomicBool, Ordering};
|
|
|
|
use serde::{Deserialize, Serialize};
|
|
use transcribe_cpp::{Backend, Device, DeviceType};
|
|
|
|
/// At most this many remembered failures; the oldest go first.
|
|
const KEPT: usize = 16;
|
|
|
|
#[derive(Clone, Debug, Deserialize, PartialEq, Serialize)]
|
|
pub(crate) struct Failure {
|
|
/// The device: its description, kind, and hardware id.
|
|
pub(crate) device: String,
|
|
/// What its driver was: [`driver`] when it failed.
|
|
pub(crate) driver: String,
|
|
/// The error, for the log and Settings.
|
|
pub(crate) reason: String,
|
|
/// Seconds since the Unix epoch.
|
|
pub(crate) at: u64,
|
|
/// It worked, but slower than the CPU (#484), rather than failing.
|
|
#[serde(default, skip_serializing_if = "std::ops::Not::not")]
|
|
pub(crate) slow: bool,
|
|
}
|
|
|
|
/// The failures on disk, read once.
|
|
static FAILURES: Mutex<Option<Vec<Failure>>> = Mutex::new(None);
|
|
|
|
fn path() -> Option<PathBuf> {
|
|
crate::app_paths::support_dir()
|
|
.ok()
|
|
.map(|dir| dir.join("gpu-failures.json"))
|
|
}
|
|
|
|
fn with_failures<R>(work: impl FnOnce(&mut Vec<Failure>) -> R) -> R {
|
|
let mut failures = FAILURES
|
|
.lock()
|
|
.unwrap_or_else(|poisoned| poisoned.into_inner());
|
|
let failures = failures.get_or_insert_with(|| {
|
|
path()
|
|
.and_then(|path| std::fs::read(path).ok())
|
|
.and_then(|bytes| serde_json::from_slice(&bytes).ok())
|
|
.unwrap_or_default()
|
|
});
|
|
work(failures)
|
|
}
|
|
|
|
fn save(failures: &[Failure]) {
|
|
let Some(path) = path() else { return };
|
|
let written = (|| -> std::io::Result<()> {
|
|
if let Some(dir) = path.parent() {
|
|
std::fs::create_dir_all(dir)?;
|
|
}
|
|
let staged = path.with_extension(format!("json.{}", std::process::id()));
|
|
std::fs::write(&staged, serde_json::to_vec_pretty(failures)?)?;
|
|
std::fs::rename(&staged, &path)
|
|
})();
|
|
if let Err(error) = written {
|
|
tracing::warn!(%error, path = %path.display(), "could not save the GPU failure record");
|
|
}
|
|
}
|
|
|
|
/// `device` as the failure record names it.
|
|
pub(crate) fn device_key(device: &Device) -> String {
|
|
format!(
|
|
"{} ({}, {})",
|
|
device.description,
|
|
device.kind,
|
|
device.device_id.as_deref().unwrap_or("-")
|
|
)
|
|
}
|
|
|
|
/// Whether `device` failed under the driver installed now.
|
|
pub(crate) fn failed(device: &Device) -> bool {
|
|
let (key, driver) = (device_key(device), driver());
|
|
with_failures(|failures| {
|
|
failures
|
|
.iter()
|
|
.any(|failure| failure.device == key && failure.driver == driver)
|
|
})
|
|
}
|
|
|
|
/// Remembers that `device` failed under the driver installed now.
|
|
pub(crate) fn record(device: &Device, reason: &str) {
|
|
remember(device, reason, false);
|
|
}
|
|
|
|
/// Remembers that `device` ran the model slower than the CPU under the
|
|
/// driver installed now (#484).
|
|
pub(crate) fn record_slow(device: &Device, reason: &str) {
|
|
remember(device, reason, true);
|
|
}
|
|
|
|
/// What to remember or show about `device` failing for `reason` now.
|
|
fn snapshot(device: String, reason: &str, slow: bool) -> Failure {
|
|
Failure {
|
|
device,
|
|
driver: driver(),
|
|
reason: reason.chars().take(400).collect(),
|
|
at: std::time::SystemTime::now()
|
|
.duration_since(std::time::UNIX_EPOCH)
|
|
.map_or(0, |elapsed| elapsed.as_secs()),
|
|
slow,
|
|
}
|
|
}
|
|
|
|
fn remember(device: &Device, reason: &str, slow: bool) {
|
|
let failure = snapshot(device_key(device), reason, slow);
|
|
if slow {
|
|
tracing::info!(
|
|
device = failure.device,
|
|
driver = failure.driver,
|
|
reason = failure.reason,
|
|
"the CPU runs the model faster than this GPU; Auto uses the CPU on it until its driver changes"
|
|
);
|
|
} else {
|
|
tracing::warn!(
|
|
device = failure.device,
|
|
driver = failure.driver,
|
|
reason = failure.reason,
|
|
"the GPU failed while transcribing; Auto transcribes on the CPU on it until its driver changes"
|
|
);
|
|
}
|
|
with_failures(|failures| {
|
|
failures.retain(|old| old.device != failure.device || old.driver != failure.driver);
|
|
failures.push(failure);
|
|
let excess = failures.len().saturating_sub(KEPT);
|
|
failures.drain(..excess);
|
|
save(failures);
|
|
});
|
|
}
|
|
|
|
/// Forgets `device`'s failure under the driver installed now, after it
|
|
/// worked when chosen outright.
|
|
pub(crate) fn forget(device: &Device) {
|
|
let (key, driver) = (device_key(device), driver());
|
|
with_failures(|failures| {
|
|
let before = failures.len();
|
|
failures.retain(|failure| failure.device != key || failure.driver != driver);
|
|
if failures.len() != before {
|
|
tracing::info!(device = key, "the GPU worked again; Auto uses it again");
|
|
save(failures);
|
|
}
|
|
});
|
|
}
|
|
|
|
/// Whether the last model Auto loaded runs on the CPU. Another healthy GPU
|
|
/// can run it while a failed one stays off, so a remembered failure alone
|
|
/// doesn't mean Auto uses the CPU (#966).
|
|
static AUTO_ON_CPU: AtomicBool = AtomicBool::new(false);
|
|
|
|
/// Records where the last model loaded: on the CPU under Auto, or not.
|
|
pub(crate) fn set_auto_on_cpu(on_cpu: bool) {
|
|
AUTO_ON_CPU.store(on_cpu, Ordering::Relaxed);
|
|
}
|
|
|
|
/// The newest failure that keeps a GPU off now, for Settings, only while
|
|
/// Auto actually runs the model on the CPU.
|
|
pub(crate) fn current() -> Option<Failure> {
|
|
if !AUTO_ON_CPU.load(Ordering::Relaxed) {
|
|
return None;
|
|
}
|
|
let driver = driver();
|
|
with_failures(|failures| {
|
|
failures
|
|
.iter()
|
|
.rev()
|
|
.find(|failure| failure.driver == driver)
|
|
.cloned()
|
|
})
|
|
}
|
|
|
|
/// The dictation engine's model switched to the CPU after a GPU ran out of
|
|
/// memory while transcribing. Only the engine publishes it, so another copy
|
|
/// in this process (the local API's own model, model preparation, or a
|
|
/// superseded model swap) cannot speak for dictation. It is never written to
|
|
/// `gpu-failures.json`: another copy of a model can fill a GPU (#1165), and
|
|
/// ggml keeps the failed device for the process, so a later GPU load or a
|
|
/// restart is what gets the GPU back.
|
|
static OUT_OF_MEMORY: Mutex<Option<Failure>> = Mutex::new(None);
|
|
|
|
/// What to remember about `device` running out of memory while transcribing,
|
|
/// for the model to carry to the engine.
|
|
pub(crate) fn out_of_memory_failure(device: &Device, reason: &str) -> Failure {
|
|
snapshot(device_key(device), reason, false)
|
|
}
|
|
|
|
/// Records what the dictation engine's model says: the out-of-memory
|
|
/// fallback it took, or `None` once it runs on a GPU again.
|
|
pub(crate) fn set_out_of_memory(fallback: Option<&Failure>) {
|
|
let mut current = OUT_OF_MEMORY
|
|
.lock()
|
|
.unwrap_or_else(|poisoned| poisoned.into_inner());
|
|
if current.as_ref() == fallback {
|
|
return;
|
|
}
|
|
let had = current.is_some();
|
|
*current = fallback.cloned();
|
|
match &*current {
|
|
Some(failure) => tracing::warn!(
|
|
device = failure.device,
|
|
reason = failure.reason,
|
|
"the GPU ran out of memory while transcribing; the dictation engine uses the CPU until a GPU load works or Hex restarts"
|
|
),
|
|
None if had => tracing::info!(
|
|
"the dictation engine loaded the model on a GPU again; forgetting the out-of-memory fallback"
|
|
),
|
|
None => {}
|
|
}
|
|
}
|
|
|
|
/// The dictation engine's out-of-memory fallback, which keeps this process on
|
|
/// the CPU now, for Settings. Never a record read from disk.
|
|
pub(crate) fn out_of_memory_fallback() -> Option<Failure> {
|
|
OUT_OF_MEMORY
|
|
.lock()
|
|
.unwrap_or_else(|poisoned| poisoned.into_inner())
|
|
.clone()
|
|
}
|
|
|
|
/// This launch registered the CPU alone because the GPU crashed twice
|
|
/// starting for transcription (Windows' `gpu-start.pending`, #662). Only that
|
|
/// process knows: Settings and `hex status` read it here.
|
|
static CRASH_FALLBACK: AtomicBool = AtomicBool::new(false);
|
|
|
|
/// Records that this launch runs the CPU because the GPU crashed twice
|
|
/// starting. Only the Windows marker forces it.
|
|
#[cfg(any(target_os = "windows", test))]
|
|
pub(crate) fn set_crash_fallback() {
|
|
CRASH_FALLBACK.store(true, Ordering::Relaxed);
|
|
}
|
|
|
|
/// Whether this launch runs the CPU because the GPU crashed twice starting.
|
|
pub(crate) fn crash_fallback() -> bool {
|
|
CRASH_FALLBACK.load(Ordering::Relaxed)
|
|
}
|
|
|
|
/// The GPUs ggml can run on, best first: discrete before integrated.
|
|
pub(crate) fn gpus() -> Vec<Device> {
|
|
let mut gpus = transcribe_cpp::devices()
|
|
.into_iter()
|
|
.filter(|device| matches!(device.device_type, DeviceType::Gpu | DeviceType::Igpu))
|
|
.collect::<Vec<_>>();
|
|
gpus.sort_by_key(|device| device.device_type != DeviceType::Gpu);
|
|
gpus
|
|
}
|
|
|
|
/// The backend that runs on `device` alone.
|
|
pub(crate) fn backend(device: &Device) -> Option<Backend> {
|
|
match device.kind.as_str() {
|
|
"vulkan" => Some(Backend::Vulkan),
|
|
"cuda" => Some(Backend::Cuda),
|
|
"rocm" => Some(Backend::Rocm),
|
|
"metal" => Some(Backend::Metal),
|
|
_ => None,
|
|
}
|
|
}
|
|
|
|
/// Whether `error` says the device failed, rather than the request or the
|
|
/// model: the errors a hung or lost GPU, or one out of memory, returns.
|
|
pub(crate) fn device_error(error: &color_eyre::eyre::Report) -> bool {
|
|
use transcribe_cpp::Error;
|
|
error.chain().any(|cause| {
|
|
cause.downcast_ref::<Error>().is_some_and(|error| {
|
|
matches!(
|
|
error,
|
|
Error::Backend(_) | Error::OutOfMemory(_) | Error::ModelLoad(_) | Error::Other(_)
|
|
)
|
|
})
|
|
})
|
|
}
|
|
|
|
/// Whether `error` says the GPU ran out of memory. That is not a driver
|
|
/// fault: another copy of a model (a model switch loading beside the running
|
|
/// one, or a CLI check while Hex holds the model) can fill a small GPU, so
|
|
/// the GPU is not remembered as failed for it (#1165).
|
|
pub(crate) fn out_of_memory(error: &color_eyre::eyre::Report) -> bool {
|
|
error.chain().any(|cause| {
|
|
if matches!(
|
|
cause.downcast_ref::<transcribe_cpp::Error>(),
|
|
Some(transcribe_cpp::Error::OutOfMemory(_))
|
|
) {
|
|
return true;
|
|
}
|
|
let message = cause.to_string().to_ascii_lowercase();
|
|
message.contains("out of memory")
|
|
|| message.contains("outofdevicememory")
|
|
|| message.contains("out_of_device_memory")
|
|
|| message.contains("failed to allocate")
|
|
})
|
|
}
|
|
|
|
/// What a GPU's driver is now, so an update tries it again. Linux: the
|
|
/// kernel (i915, xe, amdgpu, nouveau live there) and every installed Vulkan
|
|
/// driver's library, by size and modification time (a Mesa or vendor
|
|
/// update replaces them). Windows: this build of Hex.
|
|
pub(crate) fn driver() -> String {
|
|
static DRIVER: std::sync::OnceLock<String> = std::sync::OnceLock::new();
|
|
DRIVER.get_or_init(driver_now).clone()
|
|
}
|
|
|
|
#[cfg(target_os = "linux")]
|
|
fn driver_now() -> String {
|
|
use std::hash::{Hash, Hasher};
|
|
use std::os::unix::fs::MetadataExt;
|
|
let kernel = std::fs::read_to_string("/proc/sys/kernel/osrelease").unwrap_or_default();
|
|
let mut directories = vec![PathBuf::from("/etc/vulkan/icd.d")];
|
|
let data = std::env::var("XDG_DATA_DIRS")
|
|
.ok()
|
|
.filter(|dirs| !dirs.is_empty())
|
|
.unwrap_or_else(|| "/usr/local/share:/usr/share".into());
|
|
directories.extend(
|
|
data.split(':')
|
|
.filter(|dir| !dir.is_empty())
|
|
.map(|dir| PathBuf::from(dir).join("vulkan/icd.d")),
|
|
);
|
|
let mut libraries = Vec::new();
|
|
for directory in directories {
|
|
let Ok(entries) = std::fs::read_dir(&directory) else {
|
|
continue;
|
|
};
|
|
for entry in entries.flatten() {
|
|
let manifest = entry.path();
|
|
let library = std::fs::read(&manifest)
|
|
.ok()
|
|
.and_then(|bytes| serde_json::from_slice::<serde_json::Value>(&bytes).ok())
|
|
.and_then(|json| json["ICD"]["library_path"].as_str().map(PathBuf::from));
|
|
let Some(library) = library else { continue };
|
|
let library = if library.is_absolute() {
|
|
library
|
|
} else if library.components().count() > 1 {
|
|
directory.join(library)
|
|
} else {
|
|
// A bare name the loader finds on the library path.
|
|
[
|
|
"/usr/lib",
|
|
"/usr/lib64",
|
|
"/usr/lib/x86_64-linux-gnu",
|
|
"/run/opengl-driver/lib",
|
|
]
|
|
.iter()
|
|
.map(|dir| PathBuf::from(dir).join(&library))
|
|
.find(|path| path.is_file())
|
|
.unwrap_or(library)
|
|
};
|
|
let stamp = std::fs::metadata(&library)
|
|
.map(|metadata| (metadata.len(), metadata.mtime()))
|
|
.unwrap_or_default();
|
|
libraries.push((library, stamp));
|
|
}
|
|
}
|
|
libraries.sort();
|
|
libraries.dedup();
|
|
let mut hasher = std::collections::hash_map::DefaultHasher::new();
|
|
libraries.hash(&mut hasher);
|
|
format!("{} vulkan-{:016x}", kernel.trim(), hasher.finish())
|
|
}
|
|
|
|
#[cfg(not(target_os = "linux"))]
|
|
fn driver_now() -> String {
|
|
crate::VERSION.to_owned()
|
|
}
|
|
|
|
#[cfg(test)]
|
|
mod tests {
|
|
use super::*;
|
|
|
|
#[test]
|
|
fn only_device_errors_count() {
|
|
use color_eyre::eyre::WrapErr;
|
|
let lost: color_eyre::Result<()> =
|
|
Err(transcribe_cpp::Error::Backend("device lost".into()))
|
|
.wrap_err("transcription failed");
|
|
assert!(device_error(&lost.unwrap_err()));
|
|
let short: color_eyre::Result<()> =
|
|
Err(transcribe_cpp::Error::InputTooShort("x".into())).wrap_err("transcription failed");
|
|
assert!(!device_error(&short.unwrap_err()));
|
|
assert!(!device_error(&color_eyre::eyre::eyre!("cancelled")));
|
|
}
|
|
|
|
#[test]
|
|
fn out_of_memory_is_not_a_driver_fault() {
|
|
use color_eyre::eyre::WrapErr;
|
|
let vulkan: color_eyre::Result<()> = Err(transcribe_cpp::Error::Backend(
|
|
"vk::Device::allocateMemory: ErrorOutOfDeviceMemory".into(),
|
|
))
|
|
.wrap_err("could not load");
|
|
assert!(out_of_memory(&vulkan.unwrap_err()));
|
|
let lost: color_eyre::Result<()> =
|
|
Err(transcribe_cpp::Error::Backend("device lost".into())).wrap_err("failed");
|
|
assert!(!out_of_memory(&lost.unwrap_err()));
|
|
}
|
|
|
|
#[test]
|
|
fn no_note_while_auto_runs_on_a_gpu() {
|
|
// A healthy GPU runs the model while another one's failure is
|
|
// remembered (#966): Settings names no failure.
|
|
set_auto_on_cpu(false);
|
|
assert_eq!(current(), None);
|
|
}
|
|
|
|
#[test]
|
|
fn a_crash_fallback_lasts_this_launch() {
|
|
// The marker forced this process onto the CPU: only a restart (or a
|
|
// GPU chosen in Settings) tries the GPU again (#662).
|
|
assert!(!crash_fallback());
|
|
set_crash_fallback();
|
|
assert!(crash_fallback());
|
|
}
|
|
|
|
#[test]
|
|
fn an_out_of_memory_fallback_lasts_until_a_gpu_load_works() {
|
|
// Not `gpu-failures.json`: the engine publishes its model's fallback
|
|
// and a later GPU load clears it.
|
|
let failure = Failure {
|
|
device: "Test GPU (gpu, 0000-0000)".into(),
|
|
driver: driver(),
|
|
reason: "out of memory".into(),
|
|
at: 0,
|
|
slow: false,
|
|
};
|
|
set_out_of_memory(None);
|
|
assert!(out_of_memory_fallback().is_none());
|
|
set_out_of_memory(Some(&failure));
|
|
assert_eq!(out_of_memory_fallback(), Some(failure.clone()));
|
|
// Publishing the same fallback again changes nothing.
|
|
set_out_of_memory(Some(&failure));
|
|
assert_eq!(out_of_memory_fallback(), Some(failure));
|
|
set_out_of_memory(None);
|
|
assert!(out_of_memory_fallback().is_none());
|
|
}
|
|
|
|
#[cfg(target_os = "linux")]
|
|
#[test]
|
|
fn the_driver_names_the_kernel() {
|
|
let kernel = std::fs::read_to_string("/proc/sys/kernel/osrelease").unwrap();
|
|
assert!(driver().starts_with(kernel.trim()));
|
|
}
|
|
}
|