//! Pause a browser that is audibly playing while you dictate, and play it //! again only when Hex saw it go silent right after its own pause. //! //! macOS no longer tells other apps what is playing, so Hex listens to the //! browsers' own output: the level says whether the system pause reached a //! browser, and only that proof allows the play at the end. use std::ffi::c_void; use std::ptr::NonNull; use std::sync::atomic::{AtomicU32, AtomicU64, Ordering}; use std::sync::{Arc, OnceLock}; use std::time::Duration; use block2::RcBlock; use color_eyre::eyre::{Result, eyre}; use objc2::AnyThread; use objc2_core_audio::{ AudioDeviceCreateIOProcIDWithBlock, AudioDeviceDestroyIOProcID, AudioDeviceIOProcID, AudioDeviceStart, AudioDeviceStop, AudioHardwareCreateProcessTap, AudioHardwareDestroyAggregateDevice, AudioHardwareDestroyProcessTap, AudioObjectID, CATapDescription, CATapMuteBehavior, }; use objc2_core_audio_types::{AudioBuffer, AudioBufferList, AudioTimeStamp}; use objc2_foundation::{NSArray, NSNumber, NSString}; use crate::microphone_activity::BrowserProcesses; /// Sound: a peak above this in every listen window. const SOUND_DBFS: f32 = -60.0; /// Silence: a peak below this in consecutive windows soon after the pause. const SILENCE_DBFS: f32 = -90.0; const WINDOW: Duration = Duration::from_millis(100); const LISTEN_WINDOWS: usize = 3; /// The silence must start in the first windows after the pause: a sound /// that ends later on its own is not the pause reaching the browser. const CONFIRM_WINDOWS: usize = 5; const SILENT_WINDOWS: usize = 2; /// The last listen before play. const RESUME_LISTEN: Duration = Duration::from_millis(300); /// After play, the browser must sound within this many windows (2 s). If it /// does not, the play reached something else, so Hex pauses it again. const PLAY_CHECK_WINDOWS: usize = 20; /// A browser that closed its output opens it again within this many windows /// of play (about 200 ms on Brave). If none does, the play reached another /// app, so Hex pauses it again sooner. const OPEN_CHECK_WINDOWS: usize = 10; /// How long a new tap may take to deliver its first cycle. const TAP_WARM_UP: Duration = Duration::from_millis(300); /// The explicit MediaRemote commands. Never the toggle: it can start music. #[derive(Clone, Copy, Debug, Eq, PartialEq)] #[repr(u32)] pub(crate) enum MediaCommand { Play = 0, Pause = 1, } pub(crate) trait BrowserAudio { /// A browser is using the microphone: probably a call. fn in_call(&mut self) -> bool; /// Peak of the browsers' output over `window`, in dBFS; None when it cannot listen. fn peak_dbfs(&mut self, window: Duration) -> Option; fn send(&mut self, command: MediaCommand) -> bool; /// A browser process runs and every one has closed its audio output. fn output_closed(&mut self) -> bool; } /// Proof that Hex paused a sounding browser; only this allows play. #[derive(Debug)] pub(crate) struct PausedBrowser(()); #[cfg(test)] impl PausedBrowser { pub(crate) fn for_test() -> Self { Self(()) } } pub(crate) fn pause(audio: &mut impl BrowserAudio) -> Option { if audio.in_call() { tracing::info!("left the browser alone: it is using the microphone"); return None; } for _ in 0..LISTEN_WINDOWS { match audio.peak_dbfs(WINDOW) { Some(peak) if peak > SOUND_DBFS => {} Some(_) => { tracing::info!("left the browser alone: it is silent"); return None; } None => { tracing::info!("left the browser alone: cannot hear it"); return None; } } } if !audio.send(MediaCommand::Pause) { tracing::info!("the system refused the pause command"); return None; } let mut silent = 0; for _ in 0..CONFIRM_WINDOWS { if audio .peak_dbfs(WINDOW) .is_some_and(|peak| peak < SILENCE_DBFS) { silent += 1; if silent == SILENT_WINDOWS { tracing::info!("paused the browser"); return Some(PausedBrowser(())); } } else { silent = 0; } } tracing::info!("sent pause but the browser still plays; it will not get play"); None } pub(crate) fn resume(audio: &mut impl BrowserAudio, _paused: PausedBrowser) { let mut heard = true; match audio.peak_dbfs(RESUME_LISTEN) { Some(peak) if peak <= SOUND_DBFS => {} Some(_) => { tracing::info!("the browser already plays again; sent no play"); return; } // A pause longer than about 10 s closes the browser's output, and a // tap on closed outputs delivers nothing: that browser is still paused. None if audio.output_closed() => heard = false, None => { tracing::info!("cannot hear the browser; sent no play"); return; } } if !audio.send(MediaCommand::Play) { tracing::info!("the system refused the play command"); return; } // The tap delivers again once play opens the output, about 300 ms later. for window in 1..=PLAY_CHECK_WINDOWS { match audio.peak_dbfs(WINDOW) { Some(peak) if peak > SOUND_DBFS => { tracing::info!("resumed the browser"); return; } Some(_) => heard = true, None if !heard => { if window == OPEN_CHECK_WINDOWS && audio.output_closed() { break; } } None => { tracing::info!("sent play but cannot hear the browser any more"); return; } } } if !heard && !audio.output_closed() { tracing::info!("sent play but cannot hear the browser"); return; } // A closed or reloaded tab stays silent, and the play went to whatever // macOS now treats as playing: pause that again. tracing::info!("the browser stayed silent after play; sent pause again"); audio.send(MediaCommand::Pause); } /// The browsers' own output, heard through a private Core Audio process tap, /// and the system media commands. Build one per pause or resume: the tap /// covers the browser processes running when it first listens. pub(crate) struct NativeBrowserAudio { processes: BrowserProcesses, tap: Option, started: bool, } impl NativeBrowserAudio { /// Hears only browsers with open output: a closed output is silent, so /// most dictations open no tap at all. pub(crate) fn for_pause() -> Self { Self::new(BrowserProcesses::Playing) } /// Hears every browser: a paused one closes its output a few seconds later. pub(crate) fn for_resume() -> Self { Self::new(BrowserProcesses::All) } fn new(processes: BrowserProcesses) -> Self { Self { processes, tap: None, started: false, } } } impl BrowserAudio for NativeBrowserAudio { fn in_call(&mut self) -> bool { crate::microphone_activity::active_microphone_applications() .map(|applications| applications.iter().any(|app| app.is_browser())) .unwrap_or(true) } fn peak_dbfs(&mut self, window: Duration) -> Option { if !self.started { self.started = true; self.tap = BrowserTap::start(self.processes) .inspect_err(|error| tracing::warn!(%error, "could not listen to the browser")) .ok() .flatten(); } self.tap.as_ref()?.peak_dbfs(window) } fn send(&mut self, command: MediaCommand) -> bool { send_media_command(command) } fn output_closed(&mut self) -> bool { use crate::microphone_activity::browser_process_objects; // A quit browser is not a paused one: play would reach another app. browser_process_objects(BrowserProcesses::All).is_ok_and(|all| !all.is_empty()) && browser_process_objects(BrowserProcesses::Playing).is_ok_and(|open| open.is_empty()) } } struct BrowserTap { tap: AudioObjectID, aggregate: AudioObjectID, io_proc: AudioDeviceIOProcID, /// The largest |sample| since the last reset, as f32 bits. Non-negative /// floats order the same as their bits, so `fetch_max` keeps the peak. peak: Arc, callbacks: Arc, } impl BrowserTap { /// None when no browser process matches. fn start(processes: BrowserProcesses) -> Result> { let processes = crate::microphone_activity::browser_process_objects(processes)?; if processes.is_empty() { return Ok(None); } let processes: Vec<_> = processes.into_iter().map(NSNumber::new_u32).collect(); let processes = NSArray::from_retained_slice(&processes); // SAFETY: Standard CATapDescription initialization and setters. let description = unsafe { let description = CATapDescription::initMonoMixdownOfProcesses(CATapDescription::alloc(), &processes); description.setName(&NSString::from_str("Hex browser level")); description.setPrivate(true); description.setMuteBehavior(CATapMuteBehavior::Unmuted); description }; let mut tap: AudioObjectID = 0; // SAFETY: `description` is a valid tap description; `tap` is writable. check( unsafe { AudioHardwareCreateProcessTap(Some(&description), &mut tap) }, "could not create the browser audio tap", )?; let mut source = Self { tap, aggregate: 0, io_proc: None, peak: Arc::new(AtomicU32::new(0)), callbacks: Arc::new(AtomicU64::new(0)), }; // SAFETY: Reading the UUID of a live description. let tap_uid = unsafe { description.UUID() }.UUIDString(); source.aggregate = crate::meeting::create_tap_aggregate( &tap_uid, "Hex Browser Level", &format!("anomaly.hex.browser-tap.{}", std::process::id()), )?; let peak = source.peak.clone(); let callbacks = source.callbacks.clone(); let block = RcBlock::new( move |_now: NonNull, input: NonNull, _input_time: NonNull, _output: NonNull, _output_time: NonNull| { // SAFETY: Core Audio passes a valid list of float buffers for this cycle. let cycle_peak = unsafe { buffer_peak(input) }; peak.fetch_max(cycle_peak.to_bits(), Ordering::Relaxed); callbacks.fetch_add(1, Ordering::Relaxed); }, ); let mut io_proc: AudioDeviceIOProcID = None; // SAFETY: The aggregate exists; Core Audio copies the block. check( unsafe { AudioDeviceCreateIOProcIDWithBlock( NonNull::from(&mut io_proc), source.aggregate, None, RcBlock::as_ptr(&block), ) }, "could not attach to the browser audio tap", )?; source.io_proc = io_proc; // SAFETY: Starting the IOProc created above. check( unsafe { AudioDeviceStart(source.aggregate, source.io_proc) }, "could not start the browser audio tap", )?; let warm_up = std::time::Instant::now(); while source.callbacks.load(Ordering::Relaxed) == 0 && warm_up.elapsed() < TAP_WARM_UP { std::thread::sleep(Duration::from_millis(10)); } Ok(Some(source)) } /// None when the tap delivered nothing during `window`. fn peak_dbfs(&self, window: Duration) -> Option { let callbacks = self.callbacks.load(Ordering::Relaxed); self.peak.store(0, Ordering::Relaxed); std::thread::sleep(window); if self.callbacks.load(Ordering::Relaxed) == callbacks { return None; } let peak = f32::from_bits(self.peak.load(Ordering::Relaxed)); Some(if peak > 0.0 { 20.0 * peak.log10() } else { -200.0 }) } } impl Drop for BrowserTap { fn drop(&mut self) { // SAFETY: Each identifier was created by this tap and is torn down // once, in reverse order of creation. unsafe { if self.aggregate != 0 { if self.io_proc.is_some() { AudioDeviceStop(self.aggregate, self.io_proc); AudioDeviceDestroyIOProcID(self.aggregate, self.io_proc); } AudioHardwareDestroyAggregateDevice(self.aggregate); } if self.tap != 0 { AudioHardwareDestroyProcessTap(self.tap); } } } } /// The largest |sample| in one IO cycle. /// /// # Safety /// `list` must point at a valid `AudioBufferList` of 32-bit float samples. unsafe fn buffer_peak(list: NonNull) -> f32 { let list = unsafe { list.as_ref() }; let buffers: &[AudioBuffer] = unsafe { std::slice::from_raw_parts(list.mBuffers.as_ptr(), list.mNumberBuffers as usize) }; buffers .iter() .filter(|buffer| !buffer.mData.is_null()) .flat_map(|buffer| { let len = buffer.mDataByteSize as usize / size_of::(); unsafe { std::slice::from_raw_parts(buffer.mData.cast::(), len) } }) .fold(0.0, |peak: f32, sample| peak.max(sample.abs())) } fn check(status: i32, context: &str) -> Result<()> { if status == 0 { Ok(()) } else { Err(eyre!("{context} (OSStatus {status})")) } } type SendCommand = unsafe extern "C" fn(u32, *const c_void) -> bool; /// `MRMediaRemoteSendCommand` from the private MediaRemote framework: the /// same pause and play as the media keys, sent to the app macOS has as Now Playing. fn send_media_command(command: MediaCommand) -> bool { static SEND: OnceLock> = OnceLock::new(); let send = SEND.get_or_init(|| { // SAFETY: dlopen and dlsym with static C strings; the symbol has // this signature on every macOS that ships the framework. unsafe { let handle = libc::dlopen( c"/System/Library/PrivateFrameworks/MediaRemote.framework/MediaRemote".as_ptr(), libc::RTLD_LAZY, ); if handle.is_null() { return None; } let symbol = libc::dlsym(handle, c"MRMediaRemoteSendCommand".as_ptr()); (!symbol.is_null()).then(|| std::mem::transmute::<*mut c_void, SendCommand>(symbol)) } }); let Some(send) = send else { tracing::warn!("MediaRemote is not available"); return false; }; // SAFETY: A command number and no options dictionary. unsafe { send(command as u32, std::ptr::null()) } } #[cfg(test)] mod tests { use std::collections::VecDeque; use std::time::Duration; use super::*; struct Fake { in_call: bool, peaks: VecDeque>, sent: Vec, send_ok: bool, listened: usize, output_closed: bool, } impl Fake { fn new(peaks: impl IntoIterator>) -> Self { Self { in_call: false, peaks: peaks.into_iter().collect(), sent: Vec::new(), send_ok: true, listened: 0, output_closed: false, } } } impl BrowserAudio for Fake { fn in_call(&mut self) -> bool { self.in_call } fn peak_dbfs(&mut self, _window: Duration) -> Option { self.listened += 1; self.peaks.pop_front().flatten() } fn send(&mut self, command: MediaCommand) -> bool { self.sent.push(command); self.send_ok } fn output_closed(&mut self) -> bool { self.output_closed } } /// The three listen windows of a browser that plays. fn playing() -> [Option; 3] { [Some(-20.0); 3] } #[test] #[ignore = "needs a browser playing audio and System Audio Recording"] fn native_tap_hears_a_playing_browser() { let peak = NativeBrowserAudio::for_pause().peak_dbfs(WINDOW); eprintln!("browser peak: {peak:?} dBFS"); assert!(peak.is_some_and(|peak| peak > SOUND_DBFS)); } #[test] #[ignore = "needs a browser playing audio for 15 s and System Audio Recording"] fn native_resume_after_a_long_pause_plays_the_browser() { let _ = tracing_subscriber::fmt().with_test_writer().try_init(); let paused = pause(&mut NativeBrowserAudio::for_pause()).expect("paused the browser"); std::thread::sleep(Duration::from_secs(15)); let mut audio = NativeBrowserAudio::for_resume(); eprintln!("output closed after 15 s: {}", audio.output_closed()); resume(&mut audio, paused); // One browser tap per process at a time: its aggregate UID is per pid. drop(audio); std::thread::sleep(Duration::from_millis(500)); let peak = NativeBrowserAudio::for_pause().peak_dbfs(WINDOW); eprintln!("browser peak after resume: {peak:?} dBFS"); assert!(peak.is_some_and(|peak| peak > SOUND_DBFS)); } #[test] fn a_silent_browser_is_left_alone() { let mut audio = Fake::new([Some(-120.0)]); assert!(pause(&mut audio).is_none()); assert_eq!(audio.sent, []); } #[test] fn a_short_sound_is_not_a_playing_browser() { for listen in [ [Some(-20.0), Some(-120.0), Some(-120.0)], [Some(-120.0), Some(-20.0), Some(-20.0)], [Some(-20.0), Some(-20.0), None], ] { let mut audio = Fake::new(listen.into_iter().chain([Some(-200.0); 5])); assert!(pause(&mut audio).is_none(), "{listen:?}"); assert_eq!(audio.sent, [], "{listen:?}"); } } #[test] fn a_call_is_never_paused() { let mut audio = Fake::new(playing()); audio.in_call = true; assert!(pause(&mut audio).is_none()); assert_eq!(audio.sent, []); assert_eq!(audio.listened, 0); } #[test] fn a_confirmed_pause_is_resumed() { let mut audio = Fake::new(playing().into_iter().chain([ Some(-30.0), Some(-200.0), Some(-200.0), Some(-200.0), Some(-200.0), Some(-25.0), ])); let paused = pause(&mut audio).expect("the pause is confirmed"); resume(&mut audio, paused); assert_eq!(audio.sent, [MediaCommand::Pause, MediaCommand::Play]); } #[test] fn a_play_the_browser_never_takes_is_paused_again() { let mut audio = Fake::new(std::iter::repeat_n(Some(-200.0), 40)); resume(&mut audio, PausedBrowser::for_test()); assert_eq!(audio.sent, [MediaCommand::Play, MediaCommand::Pause]); } #[test] fn a_browser_that_plays_late_in_the_check_keeps_playing() { let mut audio = Fake::new(std::iter::repeat_n(Some(-200.0), 19).chain([Some(-25.0)])); resume(&mut audio, PausedBrowser::for_test()); assert_eq!(audio.sent, [MediaCommand::Play]); } #[test] fn a_browser_it_stops_hearing_after_play_gets_nothing_more() { let mut audio = Fake::new([Some(-200.0), Some(-200.0)]); resume(&mut audio, PausedBrowser::for_test()); assert_eq!(audio.sent, [MediaCommand::Play]); } #[test] fn two_sounding_tabs_never_get_play() { let mut audio = Fake::new( playing() .into_iter() .chain(std::iter::repeat_n(Some(-25.0), 10)), ); assert!(pause(&mut audio).is_none()); assert_eq!(audio.sent, [MediaCommand::Pause]); } #[test] fn a_late_silence_is_not_a_confirmed_pause() { let mut audio = Fake::new( playing() .into_iter() .chain([Some(-25.0); 4]) .chain([Some(-200.0); 6]), ); assert!(pause(&mut audio).is_none()); assert_eq!(audio.sent, [MediaCommand::Pause]); } #[test] fn one_quiet_window_is_not_a_confirmed_pause() { let mut audio = Fake::new(playing().into_iter().chain([ Some(-200.0), Some(-25.0), Some(-200.0), Some(-25.0), Some(-25.0), Some(-200.0), Some(-200.0), ])); assert!(pause(&mut audio).is_none()); assert_eq!(audio.sent, [MediaCommand::Pause]); } #[test] fn no_level_sends_nothing() { let mut audio = Fake::new([None]); assert!(pause(&mut audio).is_none()); assert_eq!(audio.sent, []); } #[test] fn a_browser_already_playing_gets_no_play() { let mut audio = Fake::new(playing().into_iter().chain([Some(-200.0), Some(-200.0)])); let paused = pause(&mut audio).expect("the pause is confirmed"); audio.peaks = VecDeque::from([Some(-20.0)]); resume(&mut audio, paused); assert_eq!(audio.sent, [MediaCommand::Pause]); } #[test] fn a_browser_it_cannot_hear_gets_no_play() { let mut audio = Fake::new([None]); resume(&mut audio, PausedBrowser::for_test()); assert_eq!(audio.sent, []); } #[test] fn a_browser_heard_before_play_that_goes_quiet_is_left_alone() { let mut audio = Fake::new([Some(-200.0), None]); audio.output_closed = true; resume(&mut audio, PausedBrowser::for_test()); assert_eq!(audio.sent, [MediaCommand::Play]); assert_eq!(audio.listened, 2); } #[test] fn a_browser_that_closed_its_output_gets_play() { // A pause longer than about 10 s: the browser closed its output, so // the tap delivers nothing until play opens it again. let mut audio = Fake::new([None, None, None, None, Some(-25.0)]); audio.output_closed = true; resume(&mut audio, PausedBrowser::for_test()); assert_eq!(audio.sent, [MediaCommand::Play]); } #[test] fn a_closed_browser_that_never_opens_after_play_is_paused_again() { let mut audio = Fake::new(std::iter::repeat_n(None, 40)); audio.output_closed = true; resume(&mut audio, PausedBrowser::for_test()); assert_eq!(audio.sent, [MediaCommand::Play, MediaCommand::Pause]); // A closed tab sends the play to another app: stop it after 1 s, not 2. assert_eq!(audio.listened, 1 + 10); } #[test] fn a_refused_command_is_not_confirmed() { let mut audio = Fake::new(playing().into_iter().chain([Some(-200.0); 5])); audio.send_ok = false; assert!(pause(&mut audio).is_none()); assert_eq!(audio.sent, [MediaCommand::Pause]); } }