mirror of
https://github.com/anomalyco/hex.git
synced 2026-11-22 21:23:02 +01:00
* Screenshots: Export opens a save panel; recordings dim around their target * Gate the dim windows to macOS * Screenshot name test: the date is macOS-only
1053 lines
38 KiB
Rust
1053 lines
38 KiB
Rust
//! Window geometry and cursor log for Screen-mode captures.
|
||
//!
|
||
//! While a full-screen capture records, a low-rate worker writes down which windows were on
|
||
//! screen (front to back) and where the cursor was, so `scrawl autocrop` can work out which
|
||
//! window the narration and ink were about. Snapshots are deduped (only stored when the window
|
||
//! list changes), cursor samples are deduped by movement, and window titles stay in
|
||
//! `scrawl.json`: they never go into `summary.md` or the clipboard.
|
||
|
||
use std::collections::HashMap;
|
||
use std::sync::atomic::{AtomicBool, Ordering};
|
||
use std::sync::{Arc, Mutex};
|
||
use std::thread::{self, JoinHandle};
|
||
use std::time::{Duration, Instant};
|
||
|
||
use crate::model::TargetFrame;
|
||
|
||
/// How often the worker wakes to sample the cursor and check its triggers.
|
||
const TICK: Duration = Duration::from_millis(66);
|
||
/// Snapshots while the cursor moves are at most this often (about 2 Hz).
|
||
const MOVING_INTERVAL: Duration = Duration::from_millis(500);
|
||
/// A snapshot even when nothing seems to happen, to catch windows opening without input.
|
||
const IDLE_INTERVAL: Duration = Duration::from_millis(2_000);
|
||
/// Cursor samples closer than this to the previous one (in capture points) are dropped.
|
||
const CURSOR_MIN_MOVE_PT: f64 = 2.0;
|
||
|
||
pub use scrawl_router::model::{CursorSample, WindowEntry, WindowSnapshot};
|
||
|
||
/// What the worker collected over one capture.
|
||
#[derive(Clone, Debug, Default, PartialEq)]
|
||
pub struct ActivityLog {
|
||
pub windows: Vec<WindowSnapshot>,
|
||
pub cursor: Vec<CursorSample>,
|
||
/// Calls to the window list and the time they took, for the log line at the end.
|
||
pub snapshot_calls: u32,
|
||
pub snapshot_time: Duration,
|
||
/// When the finish sequence began and how long stopping capture streams took.
|
||
pub stop_started: Option<Instant>,
|
||
pub stop_ms: Option<u64>,
|
||
}
|
||
|
||
impl ActivityLog {
|
||
/// Stores a snapshot unless the window list is identical to the last stored one.
|
||
pub fn push_windows(&mut self, t: u64, windows: Vec<WindowEntry>) -> bool {
|
||
if self
|
||
.windows
|
||
.last()
|
||
.is_some_and(|last| last.windows == windows)
|
||
{
|
||
return false;
|
||
}
|
||
self.windows.push(WindowSnapshot { t, windows });
|
||
true
|
||
}
|
||
|
||
/// Stores a cursor sample (capture points) unless it is within `CURSOR_MIN_MOVE_PT` of the
|
||
/// last one or outside the capture.
|
||
pub fn push_cursor(&mut self, t: u64, x_pt: f64, y_pt: f64, frame: TargetFrame) -> bool {
|
||
let (w, h) = (frame.width.max(1.0), frame.height.max(1.0));
|
||
if !(0.0..=w).contains(&x_pt) || !(0.0..=h).contains(&y_pt) {
|
||
return false;
|
||
}
|
||
let nx = round4(x_pt / w);
|
||
let ny = round4(y_pt / h);
|
||
if let Some(CursorSample(_, px, py)) = self.cursor.last() {
|
||
let dx = (nx - px) as f64 * w;
|
||
let dy = (ny - py) as f64 * h;
|
||
if dx.hypot(dy) < CURSOR_MIN_MOVE_PT {
|
||
return false;
|
||
}
|
||
}
|
||
self.cursor.push(CursorSample(t, nx, ny));
|
||
true
|
||
}
|
||
}
|
||
|
||
fn round4(v: f64) -> f32 {
|
||
((v * 10_000.0).round() / 10_000.0) as f32
|
||
}
|
||
|
||
/// One window as the window server reported it, in global points.
|
||
#[derive(Clone, Debug, PartialEq)]
|
||
pub struct RawWindow {
|
||
pub id: u32,
|
||
pub pid: i32,
|
||
pub owner: String,
|
||
pub title: String,
|
||
pub bounds: [f64; 4],
|
||
pub layer: i32,
|
||
pub alpha: f64,
|
||
}
|
||
|
||
/// What the log needs to know about a window's app.
|
||
#[derive(Clone, Debug, PartialEq)]
|
||
pub struct AppMeta {
|
||
pub bundle_id: Option<String>,
|
||
/// A Dock app (`NSApplicationActivationPolicyRegular`), not an agent or overlay.
|
||
pub regular: bool,
|
||
}
|
||
|
||
/// Highest window level that is still an app's own window: floating panels, status-level
|
||
/// launchers (25) and pop-up menus (101). Above are screen savers, shields and overlays.
|
||
const PANEL_LAYER_MAX: i32 = 101;
|
||
|
||
/// The app whose floating panel has the keys while `front` stays the active app. Launchers
|
||
/// (Invoke, Spotlight, Raycast) are non-activating panels, so `frontmostApplication` still names
|
||
/// the app behind them. `focused` is the app Accessibility says has the keys; it counts when it
|
||
/// shows a window at a normal or floating level.
|
||
#[cfg_attr(not(target_os = "macos"), allow(dead_code))]
|
||
pub fn panel_front(
|
||
raw: &[RawWindow],
|
||
front: Option<i32>,
|
||
focused: Option<i32>,
|
||
own_pid: i32,
|
||
) -> Option<i32> {
|
||
let focused = focused.filter(|pid| Some(*pid) != front && *pid != own_pid)?;
|
||
raw.iter()
|
||
.any(|w| {
|
||
w.pid == focused
|
||
&& w.alpha > 0.01
|
||
&& (0..=PANEL_LAYER_MAX).contains(&w.layer)
|
||
&& w.bounds[2] >= 24.0
|
||
&& w.bounds[3] >= 16.0
|
||
})
|
||
.then_some(focused)
|
||
}
|
||
|
||
/// Whether the topmost normal window on screen (in `raw`, front to back) is this process's, such
|
||
/// as Hex's Settings. Its overlays, HUDs and panels sit above the normal level and don't count.
|
||
pub fn own_window_in_front(raw: &[RawWindow], own_pid: i32) -> bool {
|
||
raw.iter()
|
||
.find(|w| w.layer == 0 && w.alpha > 0.01 && w.bounds[2] >= 80.0 && w.bounds[3] >= 80.0)
|
||
.is_some_and(|w| w.pid == own_pid)
|
||
}
|
||
|
||
/// Whether `w` is the host's capture chrome rather than one of its app windows: a window the
|
||
/// host named (`excluded`, its capsule, ink and picker), or any window of this process above
|
||
/// the normal level (overlays, HUDs and panels). The host's normal windows, such as Hex's
|
||
/// Settings, are windows like any other app's: a capture can be about them.
|
||
pub fn is_own_chrome(w: &RawWindow, own_pid: i32, excluded: &[u32]) -> bool {
|
||
excluded.contains(&w.id) || (w.pid == own_pid && w.layer != 0)
|
||
}
|
||
|
||
/// Keeps normal windows (plus regular apps' panels and menus, and the frontmost app's even when
|
||
/// it's a menu bar app), the menu bar and the Dock that overlap the capture, converts them to
|
||
/// capture points and marks the frontmost app's window: its first normal one, else its largest
|
||
/// panel (a launcher's). The host's own chrome ([`is_own_chrome`]) is left out.
|
||
///
|
||
/// The Dock's own window covers the whole display, so it is replaced by `dock` (the strip the
|
||
/// Dock reserves, in global points) or dropped when the Dock hides.
|
||
pub fn filter_windows(
|
||
raw: &[RawWindow],
|
||
frame: TargetFrame,
|
||
own_pid: i32,
|
||
excluded: &[u32],
|
||
frontmost_pid: Option<i32>,
|
||
dock: Option<[f64; 4]>,
|
||
app_meta: &mut dyn FnMut(i32) -> AppMeta,
|
||
) -> Vec<WindowEntry> {
|
||
let bounds_of = |w: &RawWindow| match (w.layer == 20 && w.owner == "Dock", dock) {
|
||
(true, Some(rect)) => Some(rect),
|
||
(true, None) => None,
|
||
(false, _) => Some(w.bounds),
|
||
};
|
||
let visible = |w: &RawWindow| {
|
||
let Some([x, y, width, height]) = bounds_of(w) else {
|
||
return false;
|
||
};
|
||
let overlaps = x < frame.x + frame.width
|
||
&& x + width > frame.x
|
||
&& y < frame.y + frame.height
|
||
&& y + height > frame.y;
|
||
!is_own_chrome(w, own_pid, excluded)
|
||
&& w.alpha > 0.01
|
||
&& width >= 24.0
|
||
&& height >= 16.0
|
||
&& overlaps
|
||
};
|
||
let front_id = frontmost_pid.and_then(|pid| {
|
||
let own = || raw.iter().filter(|w| w.pid == pid && visible(w));
|
||
own()
|
||
.find(|w| w.layer == 0)
|
||
.or_else(|| {
|
||
own()
|
||
.filter(|w| (1..=PANEL_LAYER_MAX).contains(&w.layer))
|
||
.max_by(|a, b| {
|
||
(a.bounds[2] * a.bounds[3]).total_cmp(&(b.bounds[2] * b.bounds[3]))
|
||
})
|
||
})
|
||
.map(|w| w.id)
|
||
});
|
||
let mut out = Vec::new();
|
||
for w in raw {
|
||
if !visible(w) {
|
||
continue;
|
||
}
|
||
let Some([x, y, width, height]) = bounds_of(w) else {
|
||
continue;
|
||
};
|
||
let is_dock = w.layer == 20 && w.owner == "Dock";
|
||
let menubar = w.layer == 24 && w.title == "Menubar";
|
||
let meta = app_meta(w.pid);
|
||
// Floating panels, sheets and pop-up menus count when a regular app or the frontmost
|
||
// one owns them; other high layers are system chrome, notifications and overlays.
|
||
let panel = (1..=PANEL_LAYER_MAX).contains(&w.layer)
|
||
&& (meta.regular || Some(w.pid) == frontmost_pid);
|
||
if !(w.layer == 0 || menubar || is_dock || panel) {
|
||
continue;
|
||
}
|
||
out.push(WindowEntry {
|
||
id: w.id,
|
||
app: w.owner.clone(),
|
||
bundle_id: meta.bundle_id,
|
||
regular: meta.regular,
|
||
title: w.title.clone(),
|
||
bounds: [
|
||
(x - frame.x).round() as i32,
|
||
(y - frame.y).round() as i32,
|
||
width.round() as i32,
|
||
height.round() as i32,
|
||
],
|
||
layer: w.layer,
|
||
frontmost: Some(w.id) == front_id,
|
||
});
|
||
}
|
||
out
|
||
}
|
||
|
||
/// Apps whose windows are macOS asking the user something, never what a capture is about:
|
||
/// UserNotificationCenter shows the privacy alerts (ScreenCaptureKit's "bypass the system
|
||
/// private window picker" among them), SecurityAgent the password sheets.
|
||
#[cfg_attr(not(target_os = "macos"), allow(dead_code))]
|
||
const SYSTEM_PROMPT_OWNERS: [&str; 2] = ["UserNotificationCenter", "SecurityAgent"];
|
||
|
||
#[cfg_attr(not(target_os = "macos"), allow(dead_code))]
|
||
fn is_system_prompt(w: &RawWindow) -> bool {
|
||
w.alpha > 0.01 && SYSTEM_PROMPT_OWNERS.contains(&w.owner.as_str())
|
||
}
|
||
|
||
/// System prompts (see `SYSTEM_PROMPT_OWNERS`) overlapping `frame`, by window id, sorted.
|
||
#[cfg_attr(not(target_os = "macos"), allow(dead_code))]
|
||
pub fn system_prompts_over(raw: &[RawWindow], frame: TargetFrame) -> Vec<u32> {
|
||
let mut ids: Vec<u32> = raw
|
||
.iter()
|
||
.filter(|w| {
|
||
let [x, y, width, height] = w.bounds;
|
||
is_system_prompt(w)
|
||
&& x < frame.x + frame.width
|
||
&& x + width > frame.x
|
||
&& y < frame.y + frame.height
|
||
&& y + height > frame.y
|
||
})
|
||
.map(|w| w.id)
|
||
.collect();
|
||
ids.sort_unstable();
|
||
ids
|
||
}
|
||
|
||
/// A macOS question is in view: a system prompt, or an alert in System Settings (a second
|
||
/// window of its own, like the "Quit & Reopen" that follows a Screen Recording grant) while
|
||
/// System Settings is in front. One left behind another window isn't asking anyone.
|
||
#[cfg_attr(not(target_os = "macos"), allow(dead_code))]
|
||
pub fn macos_is_asking(raw: &[RawWindow]) -> bool {
|
||
let normal = || raw.iter().filter(|w| w.layer == 0 && w.alpha > 0.01);
|
||
raw.iter().any(is_system_prompt)
|
||
|| (normal()
|
||
.next()
|
||
.is_some_and(|w| w.owner == "System Settings")
|
||
&& normal().filter(|w| w.owner == "System Settings").count() > 1)
|
||
}
|
||
|
||
/// Whether a global point is on a system prompt (they float above app windows), so a click
|
||
/// there is the user answering macOS rather than part of what they're showing.
|
||
#[cfg_attr(not(target_os = "macos"), allow(dead_code))]
|
||
pub fn system_prompt_at(raw: &[RawWindow], x: f64, y: f64) -> bool {
|
||
raw.iter().any(|w| {
|
||
let [wx, wy, ww, wh] = w.bounds;
|
||
is_system_prompt(w) && x >= wx && x <= wx + ww && y >= wy && y <= wy + wh
|
||
})
|
||
}
|
||
|
||
/// Whether the topmost window under a global point (in `raw`, front to back) is one `ours`
|
||
/// accepts, looking through `passthrough` windows, which let clicks fall to whatever is below
|
||
/// them.
|
||
pub fn owned_window_at(
|
||
raw: &[RawWindow],
|
||
x: f64,
|
||
y: f64,
|
||
passthrough: &[u32],
|
||
ours: impl Fn(&RawWindow) -> bool,
|
||
) -> bool {
|
||
raw.iter()
|
||
.filter(|w| {
|
||
let [wx, wy, ww, wh] = w.bounds;
|
||
w.alpha > 0.01 && x >= wx && x <= wx + ww && y >= wy && y <= wy + wh
|
||
})
|
||
.find(|w| !passthrough.contains(&w.id))
|
||
.is_some_and(ours)
|
||
}
|
||
|
||
/// Reads the on-screen window list (front to back) from the window server.
|
||
#[cfg(target_os = "macos")]
|
||
pub fn raw_windows() -> Vec<RawWindow> {
|
||
use core_graphics::window::{
|
||
kCGNullWindowID, kCGWindowListExcludeDesktopElements, kCGWindowListOptionOnScreenOnly,
|
||
};
|
||
listed(
|
||
kCGWindowListOptionOnScreenOnly | kCGWindowListExcludeDesktopElements,
|
||
kCGNullWindowID,
|
||
)
|
||
}
|
||
|
||
/// Where window `id` is now, in global top-left points, asking the window
|
||
/// server for that one window alone: cheap enough to follow a window as
|
||
/// it's dragged. `None` once it's gone.
|
||
#[cfg(target_os = "macos")]
|
||
pub fn window_bounds(id: u32) -> Option<[f64; 4]> {
|
||
use core_graphics::window::kCGWindowListOptionIncludingWindow;
|
||
listed(kCGWindowListOptionIncludingWindow, id)
|
||
.into_iter()
|
||
.find(|window| window.id == id)
|
||
.map(|window| window.bounds)
|
||
}
|
||
|
||
#[cfg(target_os = "macos")]
|
||
fn listed(
|
||
option: core_graphics::window::CGWindowListOption,
|
||
relative_to: core_graphics::window::CGWindowID,
|
||
) -> Vec<RawWindow> {
|
||
use core_foundation::base::{CFType, TCFType};
|
||
use core_foundation::dictionary::{CFDictionary, CFDictionaryRef};
|
||
use core_foundation::number::CFNumber;
|
||
use core_foundation::string::CFString;
|
||
use core_graphics::window::copy_window_info;
|
||
|
||
let Some(list) = copy_window_info(option, relative_to) else {
|
||
return Vec::new();
|
||
};
|
||
let key = CFString::from_static_string;
|
||
let (k_number, k_pid, k_owner, k_name, k_layer, k_alpha, k_bounds) = (
|
||
key("kCGWindowNumber"),
|
||
key("kCGWindowOwnerPID"),
|
||
key("kCGWindowOwnerName"),
|
||
key("kCGWindowName"),
|
||
key("kCGWindowLayer"),
|
||
key("kCGWindowAlpha"),
|
||
key("kCGWindowBounds"),
|
||
);
|
||
let (k_x, k_y, k_w, k_h) = (key("X"), key("Y"), key("Width"), key("Height"));
|
||
let number = |dict: &CFDictionary<CFString, CFType>, k: &CFString| -> Option<f64> {
|
||
dict.find(k)?.downcast::<CFNumber>()?.to_f64()
|
||
};
|
||
let string = |dict: &CFDictionary<CFString, CFType>, k: &CFString| -> String {
|
||
dict.find(k)
|
||
.and_then(|v| v.downcast::<CFString>())
|
||
.map(|s| s.to_string())
|
||
.unwrap_or_default()
|
||
};
|
||
let mut out = Vec::new();
|
||
for item in list.iter() {
|
||
let ptr = *item as CFDictionaryRef;
|
||
if ptr.is_null() {
|
||
continue;
|
||
}
|
||
let dict: CFDictionary<CFString, CFType> =
|
||
unsafe { CFDictionary::wrap_under_get_rule(ptr) };
|
||
let Some(bounds) = dict
|
||
.find(&k_bounds)
|
||
.and_then(|v| v.downcast::<CFDictionary>())
|
||
else {
|
||
continue;
|
||
};
|
||
let bounds: CFDictionary<CFString, CFType> =
|
||
unsafe { CFDictionary::wrap_under_get_rule(bounds.as_concrete_TypeRef()) };
|
||
out.push(RawWindow {
|
||
id: number(&dict, &k_number).unwrap_or(0.0) as u32,
|
||
pid: number(&dict, &k_pid).unwrap_or(0.0) as i32,
|
||
owner: string(&dict, &k_owner),
|
||
title: string(&dict, &k_name),
|
||
bounds: [
|
||
number(&bounds, &k_x).unwrap_or(0.0),
|
||
number(&bounds, &k_y).unwrap_or(0.0),
|
||
number(&bounds, &k_w).unwrap_or(0.0),
|
||
number(&bounds, &k_h).unwrap_or(0.0),
|
||
],
|
||
layer: number(&dict, &k_layer).unwrap_or(0.0) as i32,
|
||
alpha: number(&dict, &k_alpha).unwrap_or(1.0),
|
||
});
|
||
}
|
||
out
|
||
}
|
||
|
||
/// Wayland lists no windows to an app: the window log stays empty on Linux.
|
||
#[cfg(target_os = "linux")]
|
||
pub fn raw_windows() -> Vec<RawWindow> {
|
||
Vec::new()
|
||
}
|
||
|
||
#[cfg(target_os = "linux")]
|
||
pub fn dock_rect(_frame: TargetFrame) -> Option<[f64; 4]> {
|
||
None
|
||
}
|
||
|
||
#[cfg(target_os = "linux")]
|
||
pub fn snapshot(
|
||
_frame: TargetFrame,
|
||
_dock: Option<[f64; 4]>,
|
||
_excluded: &[u32],
|
||
_cache: &mut HashMap<i32, AppMeta>,
|
||
) -> Vec<WindowEntry> {
|
||
Vec::new()
|
||
}
|
||
|
||
/// Reads the visible top-level windows (front to back). Like CGWindowList: the taskbar is the
|
||
/// `Dock` (layer 20), always-on-top and tool windows sit above layer 0, and cloaked (on another
|
||
/// virtual desktop) or minimized windows have no alpha.
|
||
#[cfg(target_os = "windows")]
|
||
pub fn raw_windows() -> Vec<RawWindow> {
|
||
let mut names: HashMap<u32, String> = HashMap::new();
|
||
crate::win::top_level_windows()
|
||
.into_iter()
|
||
.map(|w| {
|
||
let taskbar = matches!(w.class.as_str(), "Shell_TrayWnd" | "Shell_SecondaryTrayWnd");
|
||
let owner = if taskbar {
|
||
"Dock".to_owned()
|
||
} else {
|
||
names
|
||
.entry(w.pid)
|
||
.or_insert_with(|| {
|
||
crate::win::app_identity(w.pid)
|
||
.map(|(name, _)| name)
|
||
.unwrap_or_default()
|
||
})
|
||
.clone()
|
||
};
|
||
let layer = if taskbar {
|
||
20
|
||
} else if w.is_app_window() && !w.topmost() {
|
||
0
|
||
} else {
|
||
3
|
||
};
|
||
RawWindow {
|
||
id: w.hwnd as u32,
|
||
pid: w.pid as i32,
|
||
owner,
|
||
title: w.title.clone(),
|
||
bounds: w.rect,
|
||
layer,
|
||
alpha: if w.on_screen() { 1.0 } else { 0.0 },
|
||
}
|
||
})
|
||
.collect()
|
||
}
|
||
|
||
#[cfg(target_os = "windows")]
|
||
fn app_meta_of(pid: i32) -> AppMeta {
|
||
let id = crate::win::app_identity(pid as u32).map(|(_, id)| id);
|
||
let regular = crate::win::top_level_windows()
|
||
.iter()
|
||
.any(|w| w.pid == pid as u32 && w.is_app_window());
|
||
AppMeta {
|
||
bundle_id: id,
|
||
regular,
|
||
}
|
||
}
|
||
|
||
/// The taskbar's strip on the display at `frame` (global pixels): the difference between the
|
||
/// display and its work area.
|
||
#[cfg(target_os = "windows")]
|
||
pub fn dock_rect(frame: TargetFrame) -> Option<[f64; 4]> {
|
||
let monitor = crate::win::monitors().into_iter().find(|m| {
|
||
let [x, y, w, h] = m.rect;
|
||
(x - frame.x).abs() < 1.0
|
||
&& (y - frame.y).abs() < 1.0
|
||
&& (w - frame.width).abs() < 1.0
|
||
&& (h - frame.height).abs() < 1.0
|
||
})?;
|
||
let [fx, fy, fw, fh] = monitor.rect;
|
||
let [vx, vy, vw, vh] = monitor.work;
|
||
let (left, right, top, bottom) = (
|
||
vx - fx,
|
||
(fx + fw) - (vx + vw),
|
||
vy - fy,
|
||
(fy + fh) - (vy + vh),
|
||
);
|
||
let min = 8.0;
|
||
if bottom >= min {
|
||
Some([fx, vy + vh, fw, bottom])
|
||
} else if top >= min {
|
||
Some([fx, fy, fw, top])
|
||
} else if left >= min {
|
||
Some([fx, vy, left, vh])
|
||
} else if right >= min {
|
||
Some([vx + vw, vy, right, vh])
|
||
} else {
|
||
None
|
||
}
|
||
}
|
||
|
||
/// One timed snapshot of the windows overlapping `frame`.
|
||
#[cfg(target_os = "windows")]
|
||
pub fn snapshot(
|
||
frame: TargetFrame,
|
||
dock: Option<[f64; 4]>,
|
||
excluded: &[u32],
|
||
cache: &mut HashMap<i32, AppMeta>,
|
||
) -> Vec<WindowEntry> {
|
||
let raw = raw_windows();
|
||
let front = crate::capture::frontmost_external_pid();
|
||
filter_windows(
|
||
&raw,
|
||
frame,
|
||
std::process::id() as i32,
|
||
excluded,
|
||
front,
|
||
dock,
|
||
&mut |pid| cache.entry(pid).or_insert_with(|| app_meta_of(pid)).clone(),
|
||
)
|
||
}
|
||
|
||
#[cfg(target_os = "macos")]
|
||
fn app_meta_of(pid: i32) -> AppMeta {
|
||
use objc2_app_kit::{NSApplicationActivationPolicy, NSRunningApplication};
|
||
match NSRunningApplication::runningApplicationWithProcessIdentifier(pid) {
|
||
Some(app) => AppMeta {
|
||
bundle_id: app.bundleIdentifier().map(|b| b.to_string()),
|
||
regular: app.activationPolicy() == NSApplicationActivationPolicy::Regular,
|
||
},
|
||
None => AppMeta {
|
||
bundle_id: None,
|
||
regular: false,
|
||
},
|
||
}
|
||
}
|
||
|
||
/// The strip the Dock reserves on the display at `frame` (global top-left points), from the
|
||
/// difference between the screen's frame and visible frame. `None` when the Dock hides or off
|
||
/// the main thread (`NSScreen` is main-thread only).
|
||
#[cfg(target_os = "macos")]
|
||
pub fn dock_rect(frame: TargetFrame) -> Option<[f64; 4]> {
|
||
use objc2::MainThreadMarker;
|
||
use objc2_app_kit::NSScreen;
|
||
let mtm = MainThreadMarker::new()?;
|
||
let screens = NSScreen::screens(mtm);
|
||
// AppKit's origin is the primary display's bottom-left corner, y up.
|
||
let primary_h = screens.firstObject()?.frame().size.height;
|
||
let flip = |r: objc2_foundation::NSRect| {
|
||
[
|
||
r.origin.x,
|
||
primary_h - r.origin.y - r.size.height,
|
||
r.size.width,
|
||
r.size.height,
|
||
]
|
||
};
|
||
let screen = screens.iter().find(|s| {
|
||
let [x, y, w, h] = flip(s.frame());
|
||
(x - frame.x).abs() < 1.0
|
||
&& (y - frame.y).abs() < 1.0
|
||
&& (w - frame.width).abs() < 1.0
|
||
&& (h - frame.height).abs() < 1.0
|
||
})?;
|
||
let [fx, fy, fw, fh] = flip(screen.frame());
|
||
let [vx, vy, vw, vh] = flip(screen.visibleFrame());
|
||
let (left, right, bottom) = (vx - fx, (fx + fw) - (vx + vw), (fy + fh) - (vy + vh));
|
||
// A hidden Dock still leaves a few points at the edge.
|
||
let min = 8.0;
|
||
if bottom >= min {
|
||
Some([fx, vy + vh, fw, bottom])
|
||
} else if left >= min {
|
||
Some([fx, vy, left, vh])
|
||
} else if right >= min {
|
||
Some([vx + vw, vy, right, vh])
|
||
} else {
|
||
None
|
||
}
|
||
}
|
||
|
||
/// One timed snapshot of the windows overlapping `frame`.
|
||
#[cfg(target_os = "macos")]
|
||
pub fn snapshot(
|
||
frame: TargetFrame,
|
||
dock: Option<[f64; 4]>,
|
||
excluded: &[u32],
|
||
cache: &mut HashMap<i32, AppMeta>,
|
||
) -> Vec<WindowEntry> {
|
||
objc2::rc::autoreleasepool(|_| {
|
||
let raw = raw_windows();
|
||
let front = crate::capture::frontmost_external_pid();
|
||
filter_windows(
|
||
&raw,
|
||
frame,
|
||
std::process::id() as i32,
|
||
excluded,
|
||
front,
|
||
dock,
|
||
&mut |pid| cache.entry(pid).or_insert_with(|| app_meta_of(pid)).clone(),
|
||
)
|
||
})
|
||
}
|
||
|
||
/// Background worker that logs windows and the cursor during a Screen-mode capture.
|
||
///
|
||
/// Snapshots are taken at the start, right after `request` is raised (click, ink or laser
|
||
/// start), when the frontmost app changes, at most every 500 ms while the cursor moves, every
|
||
/// 2 s otherwise, and once more at the end. Cursor samples are taken at 15 Hz.
|
||
pub struct WindowLogger {
|
||
stopping: Arc<AtomicBool>,
|
||
worker: Option<JoinHandle<ActivityLog>>,
|
||
}
|
||
|
||
impl WindowLogger {
|
||
/// Call on the main thread, so the Dock's strip can be read from `NSScreen`. `excluded`
|
||
/// is the host's chrome by window id (`OwnWindows::excluded`), left out of every snapshot.
|
||
pub fn start(
|
||
frame: TargetFrame,
|
||
started_at: Instant,
|
||
request: Arc<AtomicBool>,
|
||
excluded: Arc<Mutex<Vec<u32>>>,
|
||
) -> Self {
|
||
let stopping = Arc::new(AtomicBool::new(false));
|
||
let stop = Arc::clone(&stopping);
|
||
let dock = dock_rect(frame);
|
||
let worker = thread::Builder::new()
|
||
.name("scrawl-window-log".into())
|
||
.spawn(move || run(frame, dock, started_at, &request, &excluded, &stop))
|
||
.ok();
|
||
Self { stopping, worker }
|
||
}
|
||
|
||
pub fn stop(mut self) -> ActivityLog {
|
||
self.stopping.store(true, Ordering::Release);
|
||
let log = self
|
||
.worker
|
||
.take()
|
||
.and_then(|w| {
|
||
w.thread().unpark();
|
||
w.join().ok()
|
||
})
|
||
.unwrap_or_default();
|
||
if log.snapshot_calls > 0 {
|
||
crate::log::scrawl_log(format!(
|
||
"Window log: {} snapshots taken, {} stored, {:.2} ms each; {} cursor samples",
|
||
log.snapshot_calls,
|
||
log.windows.len(),
|
||
log.snapshot_time.as_secs_f64() * 1_000.0 / f64::from(log.snapshot_calls),
|
||
log.cursor.len()
|
||
));
|
||
}
|
||
log
|
||
}
|
||
}
|
||
|
||
impl Drop for WindowLogger {
|
||
fn drop(&mut self) {
|
||
self.stopping.store(true, Ordering::Release);
|
||
if let Some(ref w) = self.worker {
|
||
w.thread().unpark();
|
||
}
|
||
}
|
||
}
|
||
|
||
fn run(
|
||
frame: TargetFrame,
|
||
dock: Option<[f64; 4]>,
|
||
started_at: Instant,
|
||
request: &AtomicBool,
|
||
excluded: &Mutex<Vec<u32>>,
|
||
stopping: &AtomicBool,
|
||
) -> ActivityLog {
|
||
let mut log = ActivityLog::default();
|
||
let mut cache = HashMap::new();
|
||
let mut last_snapshot: Option<Instant> = None;
|
||
let mut last_front = None;
|
||
let mut cursor_at_snapshot: Option<[f32; 2]> = None;
|
||
let mut take = |log: &mut ActivityLog, t: u64| {
|
||
let began = Instant::now();
|
||
let excluded = excluded
|
||
.lock()
|
||
.unwrap_or_else(std::sync::PoisonError::into_inner)
|
||
.clone();
|
||
let windows = snapshot(frame, dock, &excluded, &mut cache);
|
||
log.snapshot_calls += 1;
|
||
log.snapshot_time += began.elapsed();
|
||
log.push_windows(t, windows);
|
||
};
|
||
loop {
|
||
let finishing = stopping.load(Ordering::Acquire);
|
||
let t = started_at.elapsed().as_millis() as u64;
|
||
let cursor = crate::capture::current_mouse_screen();
|
||
if let Some([x, y]) = cursor {
|
||
log.push_cursor(t, f64::from(x) - frame.x, f64::from(y) - frame.y, frame);
|
||
}
|
||
let front = crate::capture::frontmost_external_pid();
|
||
let since = last_snapshot.map(|s| s.elapsed());
|
||
let moved = cursor != cursor_at_snapshot;
|
||
let due = !finishing
|
||
&& (since.is_none()
|
||
|| request.swap(false, Ordering::AcqRel)
|
||
|| front != last_front
|
||
|| (moved && since.is_some_and(|s| s >= MOVING_INTERVAL))
|
||
|| since.is_some_and(|s| s >= IDLE_INTERVAL));
|
||
if due {
|
||
take(&mut log, t);
|
||
last_snapshot = Some(Instant::now());
|
||
last_front = front;
|
||
cursor_at_snapshot = cursor;
|
||
}
|
||
if finishing {
|
||
return log;
|
||
}
|
||
thread::park_timeout(TICK);
|
||
}
|
||
}
|
||
|
||
#[cfg(test)]
|
||
mod tests {
|
||
use super::*;
|
||
|
||
fn frame() -> TargetFrame {
|
||
TargetFrame {
|
||
x: 0.0,
|
||
y: 0.0,
|
||
width: 1000.0,
|
||
height: 500.0,
|
||
scale: 2.0,
|
||
}
|
||
}
|
||
|
||
fn entry(id: u32, x: i32) -> WindowEntry {
|
||
WindowEntry {
|
||
id,
|
||
app: format!("App {id}"),
|
||
bundle_id: None,
|
||
regular: true,
|
||
title: String::new(),
|
||
bounds: [x, 0, 400, 300],
|
||
layer: 0,
|
||
frontmost: id == 1,
|
||
}
|
||
}
|
||
|
||
#[test]
|
||
fn clicks_on_scrawl_windows_are_found_through_the_ink_overlay() {
|
||
let window = |id, pid, bounds| RawWindow {
|
||
id,
|
||
pid,
|
||
owner: String::new(),
|
||
title: String::new(),
|
||
bounds,
|
||
layer: 0,
|
||
alpha: 1.0,
|
||
};
|
||
let own = 7;
|
||
let ours = |w: &RawWindow| w.pid == own;
|
||
let raw = [
|
||
window(1, own, [0.0, 0.0, 1728.0, 1117.0]), // the ink overlay, click-through
|
||
window(2, own, [700.0, 6.0, 320.0, 44.0]), // the recording control
|
||
window(3, 99, [100.0, 30.0, 1200.0, 900.0]), // an app
|
||
];
|
||
assert!(
|
||
owned_window_at(&raw, 800.0, 20.0, &[1], ours),
|
||
"the stop button"
|
||
);
|
||
assert!(
|
||
!owned_window_at(&raw, 400.0, 500.0, &[1], ours),
|
||
"the app under the overlay"
|
||
);
|
||
assert!(
|
||
!owned_window_at(&raw, 1500.0, 1000.0, &[1], ours),
|
||
"the desktop"
|
||
);
|
||
assert!(
|
||
owned_window_at(&raw, 400.0, 500.0, &[], ours),
|
||
"an overlay that takes clicks"
|
||
);
|
||
}
|
||
|
||
#[test]
|
||
fn window_snapshots_are_stored_only_when_the_list_changes() {
|
||
let mut log = ActivityLog::default();
|
||
assert!(log.push_windows(0, vec![entry(1, 0), entry(2, 100)]));
|
||
assert!(!log.push_windows(500, vec![entry(1, 0), entry(2, 100)]));
|
||
// A moved window, a reorder, and a focus change are all new snapshots.
|
||
assert!(log.push_windows(1_000, vec![entry(1, 10), entry(2, 100)]));
|
||
assert!(log.push_windows(1_500, vec![entry(2, 100), entry(1, 10)]));
|
||
let mut unfocused = vec![entry(2, 100), entry(1, 10)];
|
||
unfocused[1].frontmost = false;
|
||
assert!(log.push_windows(2_000, unfocused.clone()));
|
||
assert!(!log.push_windows(2_500, unfocused));
|
||
assert_eq!(
|
||
log.windows.iter().map(|s| s.t).collect::<Vec<_>>(),
|
||
vec![0, 1_000, 1_500, 2_000]
|
||
);
|
||
}
|
||
|
||
#[test]
|
||
fn logger_snapshots_at_start_and_right_after_a_request() {
|
||
let request = Arc::new(AtomicBool::new(false));
|
||
let logger = WindowLogger::start(
|
||
frame(),
|
||
Instant::now(),
|
||
Arc::clone(&request),
|
||
Arc::default(),
|
||
);
|
||
request.store(true, Ordering::Release);
|
||
// A slow window list (a loaded CI runner) can hold the worker for a while, so wait for
|
||
// the request to be taken instead of sleeping a fixed time. Once it's taken, the snapshot
|
||
// it asked for is under way and `stop` joins it.
|
||
let deadline = Instant::now() + Duration::from_secs(20);
|
||
while request.load(Ordering::Acquire) && Instant::now() < deadline {
|
||
thread::sleep(Duration::from_millis(10));
|
||
}
|
||
assert!(!request.load(Ordering::Acquire), "the request was taken");
|
||
let log = logger.stop();
|
||
assert!(log.snapshot_calls >= 2, "{} snapshots", log.snapshot_calls);
|
||
assert!(log.windows.len() <= log.snapshot_calls as usize);
|
||
}
|
||
|
||
#[test]
|
||
fn cursor_samples_skip_tiny_moves_and_points_outside_the_capture() {
|
||
let mut log = ActivityLog::default();
|
||
assert!(log.push_cursor(0, 500.0, 250.0, frame()));
|
||
assert!(!log.push_cursor(66, 501.0, 250.5, frame()));
|
||
assert!(log.push_cursor(132, 520.0, 250.0, frame()));
|
||
assert!(!log.push_cursor(198, -5.0, 250.0, frame()));
|
||
assert!(!log.push_cursor(264, 500.0, 900.0, frame()));
|
||
assert_eq!(
|
||
log.cursor,
|
||
vec![CursorSample(0, 0.5, 0.5), CursorSample(132, 0.52, 0.5)]
|
||
);
|
||
assert_eq!(
|
||
serde_json::to_string(&log.cursor[1]).unwrap(),
|
||
"[132,0.52,0.5]"
|
||
);
|
||
}
|
||
|
||
#[test]
|
||
fn filtering_keeps_overlapping_windows_menu_bar_and_dock_in_capture_points() {
|
||
let capture = TargetFrame {
|
||
x: 1000.0,
|
||
..frame()
|
||
};
|
||
let raw = |id, pid, owner: &str, title: &str, bounds, layer| RawWindow {
|
||
id,
|
||
pid,
|
||
owner: owner.into(),
|
||
title: title.into(),
|
||
bounds,
|
||
layer,
|
||
alpha: 1.0,
|
||
};
|
||
let windows = [
|
||
raw(1, 9, "Scrawl", "", [1000.0, 0.0, 300.0, 300.0], 25),
|
||
raw(
|
||
2,
|
||
10,
|
||
"Window Server",
|
||
"Menubar",
|
||
[1000.0, 0.0, 1000.0, 24.0],
|
||
24,
|
||
),
|
||
raw(3, 11, "Dock", "Dock", [1000.0, 0.0, 1000.0, 500.0], 20),
|
||
raw(4, 12, "Control Center", "", [1800.0, 0.0, 40.0, 24.0], 25),
|
||
raw(5, 13, "Safari", "Docs", [1100.5, 50.0, 600.0, 400.0], 0),
|
||
raw(6, 14, "Ghostty", "zsh", [1200.0, 80.0, 600.0, 400.0], 0),
|
||
raw(7, 13, "Safari", "Other", [1150.0, 60.0, 600.0, 400.0], 0),
|
||
raw(8, 15, "Notes", "", [0.0, 0.0, 600.0, 400.0], 0),
|
||
raw(9, 16, "Tiny", "", [1300.0, 300.0, 10.0, 10.0], 0),
|
||
raw(10, 13, "Safari", "", [1200.0, 30.0, 200.0, 120.0], 101),
|
||
raw(11, 14, "Ghostty", "", [1200.0, 30.0, 200.0, 120.0], 101),
|
||
];
|
||
let mut lookups = Vec::new();
|
||
let dock = Some([1000.0, 440.0, 1000.0, 60.0]);
|
||
let out = filter_windows(&windows, capture, 9, &[], Some(13), dock, &mut |pid| {
|
||
lookups.push(pid);
|
||
AppMeta {
|
||
bundle_id: Some(format!("pid.{pid}")),
|
||
regular: pid != 12 && pid != 14,
|
||
}
|
||
});
|
||
assert_eq!(
|
||
out.iter().map(|w| w.id).collect::<Vec<_>>(),
|
||
vec![2, 3, 5, 6, 7, 10],
|
||
"Safari's pop-up menu counts, not an agent app's high-layer window"
|
||
);
|
||
let safari = &out[2];
|
||
assert_eq!(safari.bounds, [101, 50, 600, 400]);
|
||
assert_eq!(safari.bundle_id.as_deref(), Some("pid.13"));
|
||
assert!(safari.frontmost);
|
||
assert!(
|
||
!out[4].frontmost,
|
||
"only the first window of the frontmost app"
|
||
);
|
||
assert!(!out[0].frontmost && !out[1].frontmost);
|
||
assert_eq!(
|
||
out[1].bounds,
|
||
[0, 440, 1000, 60],
|
||
"the Dock's strip, not its window"
|
||
);
|
||
assert!(!out[3].regular);
|
||
assert_eq!(lookups, vec![10, 11, 12, 13, 14, 13, 13, 14]);
|
||
// A hidden Dock has no strip, so its full-screen window is dropped.
|
||
let hidden = filter_windows(&windows, capture, 9, &[], Some(13), None, &mut |_| {
|
||
AppMeta {
|
||
bundle_id: None,
|
||
regular: true,
|
||
}
|
||
});
|
||
assert!(hidden.iter().all(|w| w.id != 3));
|
||
}
|
||
|
||
/// Hosted in an app (Hex), the capture keeps the host's normal windows, which a capture can
|
||
/// be about, and leaves out only its chrome: the windows it named and its overlays and panels.
|
||
#[test]
|
||
fn the_hosts_normal_windows_count_and_its_chrome_does_not() {
|
||
let raw = |id, title: &str, layer| RawWindow {
|
||
id,
|
||
pid: 9,
|
||
owner: "Hex".into(),
|
||
title: title.into(),
|
||
bounds: [100.0, 50.0, 600.0, 400.0],
|
||
layer,
|
||
alpha: 1.0,
|
||
};
|
||
let windows = [
|
||
raw(1, "", 27), // the capsule
|
||
raw(2, "", 25), // the ink overlay, or a HUD
|
||
raw(3, "", 3), // a floating panel
|
||
raw(4, "Picker", 0), // a normal-level window the host named
|
||
raw(5, "Settings", 0), // its Settings window
|
||
RawWindow {
|
||
pid: 13,
|
||
owner: "Safari".into(),
|
||
..raw(6, "Docs", 0)
|
||
},
|
||
];
|
||
let out = filter_windows(&windows, frame(), 9, &[4], Some(9), None, &mut |_| {
|
||
AppMeta {
|
||
bundle_id: Some("anomaly.hex".into()),
|
||
regular: true,
|
||
}
|
||
});
|
||
assert_eq!(out.iter().map(|w| w.id).collect::<Vec<_>>(), vec![5, 6]);
|
||
assert!(out[0].frontmost, "the host's own window can be the front");
|
||
assert_eq!(out[0].title, "Settings");
|
||
assert!(!is_own_chrome(&windows[5], 9, &[4]), "another app's window");
|
||
assert!(is_own_chrome(&windows[0], 9, &[]), "an overlay, unnamed");
|
||
assert!(
|
||
own_window_in_front(&windows, 9),
|
||
"Settings is the top normal window"
|
||
);
|
||
let chrome = [windows[0].clone(), windows[1].clone(), windows[5].clone()];
|
||
assert!(
|
||
!own_window_in_front(&chrome, 9),
|
||
"with only its chrome over Safari, the host isn't in front"
|
||
);
|
||
let behind = [windows[5].clone(), windows[4].clone()];
|
||
assert!(!own_window_in_front(&behind, 9), "Settings behind Safari");
|
||
}
|
||
|
||
#[test]
|
||
fn a_launchers_key_panel_is_the_front_over_the_active_app() {
|
||
let raw = |id, pid, owner: &str, bounds, layer| RawWindow {
|
||
id,
|
||
pid,
|
||
owner: owner.into(),
|
||
title: String::new(),
|
||
bounds,
|
||
layer,
|
||
alpha: 1.0,
|
||
};
|
||
// Invoke (pid 20, a menu bar app) shows its search panel and an actions menu over
|
||
// OpenCode (pid 13), which stays the active app.
|
||
let windows = [
|
||
raw(1, 20, "Invoke", [520.0, 150.0, 200.0, 180.0], 25),
|
||
raw(2, 20, "Invoke", [350.0, 110.0, 400.0, 260.0], 25),
|
||
raw(3, 21, "Hex", [450.0, 460.0, 100.0, 30.0], 25),
|
||
raw(4, 13, "OpenCode Beta", [0.0, 0.0, 1000.0, 500.0], 0),
|
||
];
|
||
assert_eq!(panel_front(&windows, Some(13), Some(20), 9), Some(20));
|
||
assert_eq!(panel_front(&windows, Some(13), Some(13), 9), None);
|
||
assert_eq!(
|
||
panel_front(&windows, Some(13), Some(9), 9),
|
||
None,
|
||
"Scrawl's own keys"
|
||
);
|
||
assert_eq!(
|
||
panel_front(&windows, Some(13), Some(22), 9),
|
||
None,
|
||
"an app with the keys but nothing on screen"
|
||
);
|
||
|
||
let out = filter_windows(&windows, frame(), 9, &[], Some(20), None, &mut |pid| {
|
||
AppMeta {
|
||
bundle_id: None,
|
||
regular: pid == 13,
|
||
}
|
||
});
|
||
assert_eq!(
|
||
out.iter().map(|w| w.id).collect::<Vec<_>>(),
|
||
vec![1, 2, 4],
|
||
"the frontmost menu bar app's panels count, another one's don't"
|
||
);
|
||
assert_eq!(
|
||
out.iter()
|
||
.filter(|w| w.frontmost)
|
||
.map(|w| w.id)
|
||
.collect::<Vec<_>>(),
|
||
vec![2],
|
||
"the launcher's largest panel, not the app behind it"
|
||
);
|
||
}
|
||
|
||
#[test]
|
||
fn system_prompts_are_found_over_the_capture_and_clicks_on_them_are_answers() {
|
||
let window = |id, owner: &str, bounds, layer| RawWindow {
|
||
id,
|
||
pid: 7,
|
||
owner: owner.into(),
|
||
title: String::new(),
|
||
bounds,
|
||
layer,
|
||
alpha: 1.0,
|
||
};
|
||
let raw = [
|
||
window(5, "UserNotificationCenter", [830.0, 188.0, 260.0, 348.0], 8),
|
||
window(6, "SecurityAgent", [2000.0, 100.0, 300.0, 200.0], 1000),
|
||
window(7, "Safari", [0.0, 0.0, 1000.0, 500.0], 0),
|
||
];
|
||
// `frame()` is 1000 × 500 at the origin: the password sheet is on another display.
|
||
assert_eq!(system_prompts_over(&raw, frame()), vec![5]);
|
||
assert!(system_prompt_at(&raw, 900.0, 300.0));
|
||
assert!(!system_prompt_at(&raw, 100.0, 100.0));
|
||
assert!(macos_is_asking(&raw));
|
||
let settings = [
|
||
window(8, "System Settings", [597.0, 132.0, 723.0, 625.0], 0),
|
||
window(9, "Safari", [0.0, 0.0, 1000.0, 500.0], 0),
|
||
];
|
||
assert!(!macos_is_asking(&settings));
|
||
let quit_and_reopen = [
|
||
window(10, "System Settings", [829.0, 342.0, 260.0, 204.0], 0),
|
||
settings[0].clone(),
|
||
];
|
||
assert!(macos_is_asking(&quit_and_reopen));
|
||
// The same alert behind Scrawl's window.
|
||
let behind = [
|
||
window(11, "Scrawl Helper", [650.0, 145.0, 620.0, 532.0], 0),
|
||
quit_and_reopen[0].clone(),
|
||
settings[0].clone(),
|
||
];
|
||
assert!(!macos_is_asking(&behind));
|
||
}
|
||
}
|