hex/crates/capture/src/capture.rs
Kit Langton 4f3a45e03a
feat(screenshots): a recording toolbar, smaller movies with sound, and playback in the editor (#2166)
* Screenshots recordings: quality settings, copy as a file, export size (t1607)

Labs › Screenshots gets a Recording group (Scale Retina to 1x, Max resolution, Frame rate, Quality), each with a hex settings key; defaults 1x, Original, 30 fps, High (HEVC 0.50). The editor copies a recording's movie file, and its export-size pop-up (Original, 1080p, 720p) shows an estimate for the span kept, re-encoding off the UI thread with progress on the pressed button. (t1607)

* Screenshots: ⌘⇧5 recording toolbar with microphone and computer audio mixed into the movie (t1607)

* Screenshots: compact recording toolbar, editor playback and tidier toolbar (t1607)

* Screenshots: simplify the toolbar still, export-size save and playhead tick (t1607)

* docs: the compact recording toolbar and editor playback (t1607)

* Screenshots: filmstrip cells keep the movie's shape; mic-off icon is macOS-only (t1607)

* Screenshots: trim editor goldens for the transport and filmstrip (t1607)

* Screenshots: a trimmed export keeps a whole video track; reencode has a deadline (t1607)

* Screenshots: recording sound isn't silenced; toolbar choices apply at once and save in order (t1607)

* Screenshots: Return and Escape reach an open toolbar menu (t1607)

* Screenshots: a menu-only open of the chosen microphone; no microphone keeps no audio; Annotate may take the record key while the Lab is off (t1607)

* Screenshots: an export lands whole or not at all, and reports only to the recording it came from (t1607)

* Screenshots: keep the exact-device open and partial exports macOS-only (t1607)
2026-11-21 08:26:36 -08:00

4210 lines
149 KiB
Rust
Raw Permalink Blame History

This file contains ambiguous Unicode characters

This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.

use std::path::{Path, PathBuf};
use std::sync::atomic::{AtomicBool, AtomicI32, AtomicU64, Ordering};
use std::sync::{Arc, Mutex};
use std::thread::{self, JoinHandle};
use std::time::{Duration, Instant};
use anyhow::{Context, Result};
use crate::log::scrawl_log;
use crate::model::{
CaptureMode, InteractionEvent, RgbaFrame, SCALE, Stroke, TargetFrame, TargetInfo,
};
// The operating system side of capture: global shortcuts, input monitoring, the frontmost app,
// displays and windows, and the screen recorder. Each platform module provides the same items.
#[cfg_attr(target_os = "macos", path = "capture/macos.rs")]
#[cfg_attr(target_os = "windows", path = "capture/windows.rs")]
#[cfg_attr(target_os = "linux", path = "capture/linux.rs")]
mod platform;
pub use platform::*;
pub const FRAME_SAMPLE_INTERVAL: Duration = Duration::from_millis(220);
/// Candidate frames only feed scene diffs and keyframe selection; the keyframes themselves are
/// decoded from `recording.mp4` at full size afterwards.
pub const CANDIDATE_EDGE: f64 = 480.0;
/// Without a recording to decode from, the candidates are the keyframes, so keep them sharper.
#[cfg_attr(target_os = "linux", allow(dead_code))]
pub(crate) const CANDIDATE_EDGE_NO_VIDEO: f64 = 1280.0;
/// The candidates' memory ceiling, however long the capture runs.
pub const CANDIDATE_BUDGET_BYTES: usize = 384 << 20;
/// The candidate frames of a capture, bounded in memory. When full it drops every other frame
/// (keeping the first and the ones taken for a click, stroke or shortcut) and samples half as
/// often from then on, so a long capture keeps its whole timeline, evenly, at a coarser step.
pub struct CandidateBuffer {
frames: Vec<(RgbaFrame, bool)>,
cap: usize,
budget: usize,
interval: Duration,
}
impl CandidateBuffer {
pub fn new(budget_bytes: usize) -> Self {
Self {
frames: Vec::new(),
cap: 0,
budget: budget_bytes,
interval: FRAME_SAMPLE_INTERVAL,
}
}
/// How often to sample now.
pub fn interval(&self) -> Duration {
self.interval
}
#[cfg(test)]
fn len(&self) -> usize {
self.frames.len()
}
pub fn push(&mut self, frame: RgbaFrame, urgent: bool) {
if self.cap == 0 {
self.cap = (self.budget / frame.rgba.len().max(1)).clamp(60, 1_200);
}
self.frames.push((frame, urgent));
if self.frames.len() >= self.cap {
self.thin();
}
}
fn thin(&mut self) {
let mut regular = 0_usize;
let mut kept: Vec<(RgbaFrame, bool)> = std::mem::take(&mut self.frames)
.into_iter()
.enumerate()
.filter(|(i, (_, urgent))| {
let keep = *i == 0 || *urgent || regular.is_multiple_of(2);
if !urgent {
regular += 1;
}
keep
})
.map(|(_, f)| f)
.collect();
// Mostly urgent frames (a storm of clicks): thin those too.
if kept.len() > self.cap * 3 / 4 {
kept = kept
.into_iter()
.enumerate()
.filter(|(i, _)| i.is_multiple_of(2))
.map(|(_, f)| f)
.collect();
}
self.frames = kept;
self.interval *= 2;
}
/// Forgets frames taken after `t_ms`, except the first.
#[cfg_attr(not(target_os = "macos"), allow(dead_code))]
pub fn drop_since(&mut self, t_ms: u64) {
let mut index = 0;
self.frames.retain(|(frame, _)| {
index += 1;
index == 1 || frame.t_ms <= t_ms
});
}
pub fn into_frames(self) -> Vec<RgbaFrame> {
self.frames.into_iter().map(|(f, _)| f).collect()
}
}
/// What a finished screen capture leaves behind.
pub struct ScreenRecording {
/// Small candidate frames for keyframe selection (see `CANDIDATE_EDGE`).
pub frames: Vec<RgbaFrame>,
/// The movie, which may still be finishing: only what reads it waits ([`Movie::wait`]).
pub video: Option<Movie>,
pub duration_ms: u64,
/// Why there is no picture of the screen, when no frame was recorded: the summary says so
/// instead of showing anything.
pub no_screen: Option<String>,
}
/// Why a screen capture recorded no frame: what the screen source said, or the platform's reason.
pub fn no_screen_reason(failure: Option<String>) -> String {
match failure {
Some(why) => why,
None if screen_capture_usable(false) => "the screen recorded no frames".into(),
None => format!("Screen Recording is off for {}", host_app_name()),
}
}
/// The app this process runs as: its `.app` bundle's name, else its executable's.
fn host_app_name() -> String {
let exe = std::env::current_exe().unwrap_or_default();
exe.ancestors()
.find(|dir| dir.extension().is_some_and(|ext| ext == "app"))
.unwrap_or(&exe)
.file_stem()
.map(|name| name.to_string_lossy().into_owned())
.unwrap_or_else(|| "this app".into())
}
impl ScreenRecording {
/// Blocks until nothing writes the movie any more, so its bundle can be removed.
pub fn discard(self) {
if let Some(movie) = self.video {
let _ = movie.wait();
}
}
}
/// A screen capture's movie. On macOS its file is still being finished when the capture stops:
/// the recording output closes it and its final frame is held to the stop.
pub struct Movie {
path: PathBuf,
state: MovieState,
}
enum MovieState {
Written {
offset_ms: u64,
},
#[cfg(target_os = "macos")]
Finishing(platform::FinishingMovie),
}
impl Movie {
/// A movie already written, starting `offset_ms` into the capture (on the frames' clock).
pub fn written(path: PathBuf, offset_ms: u64) -> Self {
Self {
path,
state: MovieState::Written { offset_ms },
}
}
#[cfg(target_os = "macos")]
pub(crate) fn finishing(path: PathBuf, finishing: platform::FinishingMovie) -> Self {
Self {
path,
state: MovieState::Finishing(finishing),
}
}
/// Where the movie is, finished or not.
pub fn path(&self) -> &Path {
&self.path
}
/// When the movie starts on the frames' clock, before it finishes: exact once written, an
/// estimate from when its recording started while it finishes.
pub fn offset_hint_ms(&self) -> u64 {
match &self.state {
MovieState::Written { offset_ms } => *offset_ms,
#[cfg(target_os = "macos")]
MovieState::Finishing(finishing) => finishing.offset_hint_ms(),
}
}
/// Blocks until the movie is finished. Returns its path and when it starts on the frames'
/// clock, or `None` when nothing playable was left.
pub fn wait(self) -> Option<(PathBuf, u64)> {
let offset_ms = match self.state {
MovieState::Written { offset_ms } => Some(offset_ms),
#[cfg(target_os = "macos")]
MovieState::Finishing(finishing) => finishing.wait(),
}?;
let written = self
.path
.metadata()
.is_ok_and(|m| m.is_file() && m.len() > 0);
written.then_some((self.path, offset_ms))
}
}
#[derive(Clone, Copy, Debug, Eq, PartialEq)]
pub enum Stage {
Recording,
Stopping,
Transcribing,
Rendering,
Writing,
Done,
Cancelled,
Failed,
}
impl Stage {
pub fn label(self) -> &'static str {
match self {
Self::Recording => "recording",
Self::Stopping => "stopping",
Self::Transcribing => "transcribing",
Self::Rendering => "rendering",
Self::Writing => "writing",
Self::Done => "done",
Self::Cancelled => "cancelled",
Self::Failed => "failed",
}
}
}
/// Live engine events emitted by the global input monitor, iPhone mirror and ingest server.
#[derive(Clone, Debug)]
pub enum Event {
/// ⌥⇧S (or the configured shortcut) went down; `shortcut::Shortcut` decides what it means.
CaptureShortcut,
PasteLatestShortcut,
RequestSnapshot,
/// The iPhone mirror ended on its own (unplugged, never found, or access denied).
PhoneEnded,
/// Lifecycle progress of an active or finishing capture.
Progress {
bundle: PathBuf,
stage: Stage,
},
/// A ScrawlKit recording was ingested from an iPhone and compiled into a `.scrawl` bundle.
IngestedBundle {
dir: PathBuf,
bundle: Box<crate::model::ScrawlBundle>,
status: Option<String>,
},
/// A capture bundle was delivered (or queued for retry) to a destination.
Delivered {
dir: PathBuf,
delivery: crate::delivery::Delivery,
},
}
#[derive(Clone, Debug, Default)]
pub struct OwnWindows {
/// Window IDs excluded from ScreenCaptureKit / Windows.Graphics.Capture recordings.
pub excluded: Vec<u32>,
/// Click-through window IDs (such as the ink overlay) ignored when checking whether a click
/// landed on the host's own UI.
pub passthrough: Vec<u32>,
/// Set, the recording shows the host's own windows: neither `excluded` nor the host's
/// other non-layer-0 windows are left out. Read while recording, so the host can flip it
/// mid-capture (a demo that shows the host's overlays).
pub recorded: Arc<AtomicBool>,
}
/// What the host provides. Every method has a default except `emit`; `emit` and `log` must return
/// without blocking.
pub trait Host: Send + Sync + 'static {
fn emit(&self, event: Event);
fn log(&self, line: &str) {
scrawl_log(line);
}
fn metric(&self, name: &str, ms: f64) {
crate::log::record_metric(name, ms);
}
fn clipboard(&self) -> Option<&dyn crate::handoff::Clipboard> {
None
}
/// Window IDs to leave out of recordings (overlay, capsule, host windows) and to let clicks
/// through (`excluded_window_ids` / `passthrough_window_ids`).
fn own_windows(&self) -> OwnWindows {
OwnWindows::default()
}
}
pub struct ChannelHost {
sender: flume::Sender<Event>,
}
impl ChannelHost {
pub fn new(sender: flume::Sender<Event>) -> Self {
Self { sender }
}
}
impl Host for ChannelHost {
fn emit(&self, event: Event) {
let _ = self.sender.try_send(event);
}
}
pub trait ConfigSource: Send + Sync {
fn engine_config(&self) -> Result<crate::handoff::EngineConfig>;
}
#[derive(Clone, Copy, Debug, Default)]
pub struct DefaultConfigSource;
impl ConfigSource for DefaultConfigSource {
fn engine_config(&self) -> Result<crate::handoff::EngineConfig> {
Ok(crate::handoff::engine_config())
}
}
#[derive(Clone, Debug, Default)]
pub enum Target {
/// Discover the display under the cursor (or focused window) when starting.
#[default]
ActiveDisplay,
/// Record a specific screen, window or region target.
Screen(TargetInfo),
/// Record a cabled iPhone (either a new session for `device` or an existing preview). With
/// [`AudioIn::External`] the mirror opens no microphone and `finish`'s audio is the
/// narration; otherwise the mirror records the Mac's microphone into the movie.
Phone {
device: Option<String>,
name: Option<String>,
},
}
#[derive(Clone, Copy, Debug, Default, Eq, PartialEq)]
pub enum AudioIn {
/// No engine-managed microphone stream; the host passes `Option<RecordedAudio>` to `finish`.
#[default]
External,
/// No audio at all.
None,
/// Open Scrawl's built-in `AudioRecorder` stream.
Microphone,
}
/// How a screen capture's movie is recorded (macOS; elsewhere always compact).
#[derive(Clone, Copy, Debug, Default, Eq, PartialEq)]
pub enum MovieQuality {
/// Sized for agents: H.264, 1280 px long edge (2560 for a whole display, for crops).
#[default]
Compact,
/// For people: hardware HEVC (`capture_core::Encoder::Hevc`) at the size, frame rate, and
/// quality asked.
Full(MovieSize),
}
/// A [`MovieQuality::Full`] movie's size, frame rate, and quality.
#[derive(Clone, Copy, Debug, Eq, PartialEq)]
pub struct MovieSize {
/// At most one pixel a point, so a Retina display records at its size in points.
pub one_x: bool,
/// The longest side in pixels; `None` keeps the source's (or one pixel a point's).
pub long_edge: Option<u32>,
pub fps: u32,
/// HEVC's constant quality, in hundredths.
pub quality: u8,
/// Also records other apps' audio (never Hex's) into `system.m4a`.
pub system_audio: bool,
}
impl Default for MovieSize {
/// Every pixel at up to 60 fps, at `capture_core`'s quality for Retina text.
fn default() -> Self {
Self {
one_x: false,
long_edge: None,
fps: 60,
quality: 70,
system_audio: false,
}
}
}
impl MovieSize {
/// The long edge a source `points` large records at: one pixel a point with
/// [`Self::one_x`], capped at [`Self::long_edge`]. `None` keeps every pixel.
pub fn long_edge_for(self, points: (f64, f64)) -> Option<u32> {
let longest = points.0.max(points.1);
let one_x =
(self.one_x && longest.is_finite() && longest >= 1.0).then(|| longest.round() as u32);
match (one_x, self.long_edge) {
(Some(a), Some(b)) => Some(a.min(b)),
(a, b) => a.or(b),
}
}
}
mod start;
mod token;
pub use start::Start;
pub use token::{OutputBounds, PortalToken};
#[derive(Clone, Debug, Default)]
pub struct RouteContext {
pub front_bundle_id: Option<String>,
pub captured_bundle_id: Option<String>,
pub app_pid: Option<i32>,
}
#[derive(Clone, Debug)]
pub struct Finished {
pub dir: PathBuf,
pub bundle: crate::model::ScrawlBundle,
pub summary: String,
pub words: usize,
pub duration_ms: u64,
pub route_hint: RouteContext,
/// Number of audio gap spans where microphone samples were missing or dropped (`RecordedAudio::gaps`).
pub gaps: usize,
/// Structured reason when no transcript was produced (`"no_audio"` when no audio was
/// captured, or `bundle.transcript.missing.reason` when transcription failed or is pending).
pub missing_transcript: Option<String>,
}
// Carbon HIToolbox modifier bits. The shortcut is stored with them on every platform.
pub const CARBON_CMD_KEY: u32 = 1 << 8;
pub const CARBON_SHIFT_KEY: u32 = 1 << 9;
pub const CARBON_OPTION_KEY: u32 = 1 << 11;
pub const CARBON_CONTROL_KEY: u32 = 1 << 12;
#[derive(Clone, Debug, PartialEq, Eq, serde::Serialize, serde::Deserialize)]
pub struct ShortcutConfig {
pub key_code: u32,
pub carbon_modifiers: u32,
pub label: String,
}
impl Default for ShortcutConfig {
fn default() -> Self {
Self::default_record()
}
}
/// Scrawl's two global shortcuts.
#[derive(Clone, Copy, Debug, PartialEq, Eq, serde::Serialize, serde::Deserialize)]
#[serde(rename_all = "kebab-case")]
pub enum ShortcutKind {
/// Start, switch or finish a recording (⌥⇧S).
Record,
/// Paste the latest capture into the app in front (⌥⇧V).
Paste,
}
impl ShortcutKind {
pub const ALL: [Self; 2] = [Self::Record, Self::Paste];
pub fn default_config(self) -> ShortcutConfig {
match self {
Self::Record => ShortcutConfig::default_record(),
Self::Paste => ShortcutConfig::default_paste(),
}
}
/// The letter of the default, on the active layout (Dvorak's S is not ANSI's).
fn default_char(self) -> char {
match self {
Self::Record => 's',
Self::Paste => 'v',
}
}
pub(crate) fn ansi_code(self) -> u16 {
match self {
Self::Record => KEY_S,
Self::Paste => KEY_V,
}
}
pub fn name(self) -> &'static str {
match self {
Self::Record => "Record",
Self::Paste => "Paste last capture",
}
}
/// Whether it can be turned off. Record can't: it's the way into Scrawl.
pub fn can_unset(self) -> bool {
self == Self::Paste
}
}
impl ShortcutConfig {
pub fn default_record() -> Self {
Self {
key_code: u32::from(layout_s_keycode()),
carbon_modifiers: CARBON_OPTION_KEY | CARBON_SHIFT_KEY,
label: shortcut_label(CARBON_OPTION_KEY | CARBON_SHIFT_KEY, "S"),
}
}
pub fn default_paste() -> Self {
Self {
key_code: u32::from(layout_v_keycode()),
carbon_modifiers: CARBON_OPTION_KEY | CARBON_SHIFT_KEY,
label: shortcut_label(CARBON_OPTION_KEY | CARBON_SHIFT_KEY, "V"),
}
}
pub fn defaults() -> (Self, Self) {
(Self::default_record(), Self::default_paste())
}
/// Default shortcuts using ANSI key codes (`KEY_S`, `KEY_V`) without querying macOS TIS/TSM,
/// safe to call off the main thread.
pub fn ansi_defaults() -> (Self, Self) {
(
Self {
key_code: u32::from(KEY_S),
carbon_modifiers: CARBON_OPTION_KEY | CARBON_SHIFT_KEY,
label: shortcut_label(CARBON_OPTION_KEY | CARBON_SHIFT_KEY, "S"),
},
Self {
key_code: u32::from(KEY_V),
carbon_modifiers: CARBON_OPTION_KEY | CARBON_SHIFT_KEY,
label: shortcut_label(CARBON_OPTION_KEY | CARBON_SHIFT_KEY, "V"),
},
)
}
/// No shortcut: nothing is registered and no key press matches. Only the paste shortcut can
/// be turned off.
pub fn unset() -> Self {
Self {
key_code: u32::MAX,
carbon_modifiers: 0,
label: String::new(),
}
}
pub fn is_unset(&self) -> bool {
self.label.is_empty()
}
/// Whether this is `kind`'s default, which also answers to its aliases (⌃⌥⇧, ⌘⌥⇧, ⌃⌥⌘
/// and the ANSI key).
pub fn is_default_for(&self, kind: ShortcutKind) -> bool {
self.carbon_modifiers == CARBON_OPTION_KEY | CARBON_SHIFT_KEY
&& shortcut_parts(&self.label).1 == kind.default_char().to_ascii_uppercase().to_string()
}
pub fn key_label(&self) -> &str {
shortcut_parts(&self.label).1
}
}
/// `⌃⌥⇧⌘` in the order macOS writes them, for Carbon modifier bits; on Windows the same
/// modifiers as `Ctrl+Win+Alt+Shift+` (ready to take a key).
pub fn modifier_symbols(carbon_modifiers: u32) -> String {
if cfg!(target_os = "windows") {
let label = shortcut_label(carbon_modifiers, "");
return label;
}
[
(CARBON_CONTROL_KEY, '⌃'),
(CARBON_OPTION_KEY, '⌥'),
(CARBON_SHIFT_KEY, '⇧'),
(CARBON_CMD_KEY, '⌘'),
]
.into_iter()
.filter(|(bit, _)| carbon_modifiers & bit != 0)
.map(|(_, symbol)| symbol)
.collect()
}
/// A shortcut as this platform writes it: `⌥⇧S` on macOS, `Alt+Shift+S` on Windows.
pub fn shortcut_label(carbon_modifiers: u32, key: &str) -> String {
let held = |bit: u32| carbon_modifiers & bit != 0;
if cfg!(target_os = "windows") {
let mut parts = Vec::new();
for (bit, name) in [
(CARBON_CONTROL_KEY, "Ctrl"),
(CARBON_CMD_KEY, "Win"),
(CARBON_OPTION_KEY, "Alt"),
(CARBON_SHIFT_KEY, "Shift"),
] {
if held(bit) {
parts.push(name);
}
}
parts.push(key);
parts.join("+")
} else {
let mut label = String::new();
for (bit, glyph) in [
(CARBON_CONTROL_KEY, '⌃'),
(CARBON_OPTION_KEY, '⌥'),
(CARBON_SHIFT_KEY, '⇧'),
(CARBON_CMD_KEY, '⌘'),
] {
if held(bit) {
label.push(glyph);
}
}
label.push_str(key);
label
}
}
/// The system paste keystroke: `⌘V` on macOS, `Ctrl+V` on Windows.
pub fn system_paste_label() -> &'static str {
if cfg!(target_os = "windows") {
"Ctrl+V"
} else {
"⌘V"
}
}
/// The keys of a shortcut label, one per keycap: `⌥⇧S` → `⌥`, `⇧`, `S`; `Alt+Shift+S` →
/// `Alt`, `Shift`, `S`.
pub fn shortcut_keys(label: &str) -> Vec<String> {
if label.contains('+') && label.len() > 1 {
let mut keys: Vec<String> = label
.split('+')
.filter(|key| !key.is_empty())
.map(str::to_owned)
.collect();
// `Ctrl++` binds the plus key itself.
if label.ends_with("++") {
keys.push("+".into());
}
keys
} else {
label.chars().map(String::from).collect()
}
}
/// A shortcut label split into its modifiers and key: `⌥⇧S` → (`⌥⇧`, `S`), and on Windows
/// `Alt+Shift+S` → (`Alt+Shift`, `S`).
pub fn shortcut_parts(label: &str) -> (&str, &str) {
// The key itself may be "+" (`Ctrl++`), or a letter outside ASCII.
let before_key = label.char_indices().last().map_or("", |(i, _)| &label[..i]);
if let Some(plus) = before_key.rfind('+') {
return (&label[..plus], &label[plus + 1..]);
}
let split = label
.char_indices()
.find(|(_, c)| !matches!(c, '⌃' | '⌥' | '⇧' | '⌘'))
.map_or(label.len(), |(i, _)| i);
label.split_at(split)
}
/// Whether a key press is the record shortcut. A custom one matches exactly; the default ⌥⇧S
/// also answers with ⌃ or ⌘ added and as ⌃⌥⌘S, on the active layout's S key (`s_code`) or the
/// ANSI one.
pub fn is_capture_shortcut(
cfg: &ShortcutConfig,
s_code: u16,
keycode: u16,
cmd: bool,
ctrl: bool,
alt: bool,
shift: bool,
) -> bool {
shortcut_matches(
ShortcutKind::Record,
cfg,
s_code,
keycode,
cmd,
ctrl,
alt,
shift,
)
}
/// `is_capture_shortcut` for any of Scrawl's shortcuts; `layout_code` is the default key on the
/// active layout.
#[allow(clippy::too_many_arguments)]
pub fn shortcut_matches(
kind: ShortcutKind,
cfg: &ShortcutConfig,
layout_code: u16,
keycode: u16,
cmd: bool,
ctrl: bool,
alt: bool,
shift: bool,
) -> bool {
if cfg.is_unset() {
return false;
}
let want_cmd = (cfg.carbon_modifiers & CARBON_CMD_KEY) != 0;
let want_ctrl = (cfg.carbon_modifiers & CARBON_CONTROL_KEY) != 0;
let want_alt = (cfg.carbon_modifiers & CARBON_OPTION_KEY) != 0;
let want_shift = (cfg.carbon_modifiers & CARBON_SHIFT_KEY) != 0;
let exact = u32::from(keycode) == cfg.key_code
&& cmd == want_cmd
&& ctrl == want_ctrl
&& alt == want_alt
&& shift == want_shift;
if exact {
return true;
}
let is_key = keycode == layout_code || keycode == kind.ansi_code();
cfg.is_default_for(kind) && is_key && alt && (shift || (ctrl && cmd))
}
/// The active layout's key for `kind`'s default letter (Windows' hooks and poller match on it).
#[cfg_attr(target_os = "macos", allow(dead_code))]
pub fn layout_default_keycode(kind: ShortcutKind) -> u16 {
key_code_for_char(kind.default_char()).unwrap_or(kind.ansi_code())
}
/// The key codes and Carbon modifiers `cfg` registers: itself, plus for a default its aliases
/// (⌃⌥⇧, ⌘⌥⇧, ⌃⌥⌘) on the active layout's key and the ANSI one.
pub fn hotkey_combos(kind: ShortcutKind, cfg: &ShortcutConfig) -> Vec<(u32, u32)> {
if cfg.is_unset() {
return Vec::new();
}
let mut combos = vec![(cfg.key_code, cfg.carbon_modifiers)];
if !cfg.is_default_for(kind) {
return combos;
}
let mut codes = vec![cfg.key_code];
if cfg.key_code != u32::from(kind.ansi_code()) {
codes.push(u32::from(kind.ansi_code()));
}
for code in codes {
for mods in [
CARBON_OPTION_KEY | CARBON_SHIFT_KEY,
CARBON_CONTROL_KEY | CARBON_OPTION_KEY | CARBON_SHIFT_KEY,
CARBON_CMD_KEY | CARBON_OPTION_KEY | CARBON_SHIFT_KEY,
CARBON_CONTROL_KEY | CARBON_OPTION_KEY | CARBON_CMD_KEY,
] {
if !combos.contains(&(code, mods)) {
combos.push((code, mods));
}
}
}
combos
}
#[cfg_attr(target_os = "linux", allow(dead_code))]
fn clean_label(input: &str) -> String {
let squashed = input.split_whitespace().collect::<Vec<_>>().join(" ");
let trimmed = squashed.trim();
if trimmed.chars().count() > 44 {
let prefix: String = trimmed.chars().take(43).collect();
format!("{}…", prefix.trim_end())
} else {
trimmed.to_string()
}
}
/// Which display to record: the one under `point`, else the one holding the focused window's
/// centre, else the first. Frames and points are global, top-left origin.
pub fn pick_display(
displays: &[[f64; 4]],
point: Option<[f64; 2]>,
focused: Option<[f64; 4]>,
) -> usize {
let inside = |p: [f64; 2]| {
displays
.iter()
.position(|&[x, y, w, h]| p[0] >= x && p[0] < x + w && p[1] >= y && p[1] < y + h)
};
point
.and_then(inside)
.or_else(|| focused.and_then(|[x, y, w, h]| inside([x + w / 2.0, y + h / 2.0])))
.unwrap_or(0)
}
pub fn selectable_window_at(windows: &[TargetInfo], x: f64, y: f64) -> Option<&TargetInfo> {
windows.iter().find(|window| {
let f = window.frame;
x >= f.x && x <= f.x + f.width && y >= f.y && y <= f.y + f.height
})
}
/// Resolves region metadata from the picker's snapshot, without another SCK query.
pub fn region_target(
x: f64,
y: f64,
width: f64,
height: f64,
windows: &[TargetInfo],
) -> TargetInfo {
let mid_x = x + width * 0.5;
let mid_y = y + height * 0.5;
let hit = selectable_window_at(windows, mid_x, mid_y);
let frontmost = frontmost_app_info();
// One display per recording: the area's (by its centre), trimmed to that display and at
// least `MIN_AREA` a side inside it, so this frame is exactly the recorded `sourceRect`.
let screens = crate::screens::displays();
let (display_id, scale, [x, y, width, height]) =
crate::screens::area_on_display(&screens, [x, y, width, height]).map_or_else(
|| {
let min = crate::screens::MIN_AREA;
let rect = [x, y, width.max(min), height.max(min)];
(main_display_id(), MAX_CAPTURE_SCALE, rect)
},
|(i, rect)| (screens[i].id, screens[i].scale, rect),
);
TargetInfo {
mode: CaptureMode::Region,
app: hit
.and_then(|window| window.app.clone())
.or_else(|| frontmost.as_ref().map(|(_, name, _)| name.clone())),
bundle_id: hit
.and_then(|window| window.bundle_id.clone())
.or_else(|| frontmost.and_then(|(_, _, bid)| bid)),
window_title: hit.and_then(|window| window.window_title.clone()),
window_id: None,
display_id,
frame: TargetFrame {
x,
y,
width,
height,
scale,
},
}
}
/// The capture size for `target` with its long edge at most `long_edge` px (and at most its
/// display's pixels: a 1x display isn't upscaled, and on Windows frames are pixels already,
/// `MAX_CAPTURE_SCALE`).
pub fn capture_size(target: &TargetInfo, long_edge: f64) -> (u32, u32) {
let max_dim = target.frame.width.max(target.frame.height).max(1.0);
let native = target.frame.scale.clamp(1.0, MAX_CAPTURE_SCALE);
let scale = (long_edge / max_dim).min(native);
(
((target.frame.width * scale).round() as u32).max(48) & !1,
((target.frame.height * scale).round() as u32).max(48) & !1,
)
}
/// What the platform's sampler thread records into. It runs until `stopping`, pushes candidate
/// frames (at once when `urgent`), and returns the movie it recorded.
pub(crate) struct Sampler {
pub target: TargetInfo,
pub video_output_path: Option<PathBuf>,
#[cfg_attr(not(target_os = "macos"), allow(dead_code))]
pub movie_quality: MovieQuality,
/// The screen-cast portal's token choice; ignored off Linux.
#[cfg_attr(not(target_os = "linux"), allow(dead_code))]
pub portal_token: PortalToken,
/// Scrawl's own windows to leave out of the recording.
#[cfg_attr(any(target_os = "windows", target_os = "linux"), allow(dead_code))]
pub excluded_window_ids: Arc<Mutex<Vec<u32>>>,
/// [`OwnWindows::recorded`]: while set, nothing of the host's is left out.
#[cfg_attr(any(target_os = "windows", target_os = "linux"), allow(dead_code))]
pub own_windows_recorded: Arc<AtomicBool>,
pub started_at: Instant,
/// `started_at` on the host clock, which places the movie's start on the frames' clock.
#[cfg(target_os = "macos")]
pub started_host: capture_core::HostTime,
pub stopping: Arc<AtomicBool>,
pub urgent: Arc<AtomicBool>,
pub frames: Arc<Mutex<CandidateBuffer>>,
/// Why the screen source gave up before the stop (Linux: the portal's picker was cancelled
/// or the stream never started), so the host can end the capture.
#[cfg_attr(not(target_os = "linux"), allow(dead_code))]
pub failure: Arc<Mutex<Option<String>>>,
/// Where the Windows recorder stores when its movie started (ms since `started_at`).
#[cfg(target_os = "windows")]
pub video_offset_ms: Arc<AtomicU64>,
}
/// Active screen + optional MP4 recorder (ScreenCaptureKit on macOS, Windows.Graphics.Capture
/// and Media Foundation on Windows).
pub struct ScreenCaptureSession {
started_at: Instant,
stopping: Arc<AtomicBool>,
urgent_snapshot: Arc<AtomicBool>,
frames: Arc<Mutex<CandidateBuffer>>,
failure: Arc<Mutex<Option<String>>>,
sampler_thread: Option<JoinHandle<Option<Movie>>>,
_recording: crate::retention::Recording,
}
impl ScreenCaptureSession {
pub fn start(
target: TargetInfo,
video_output_path: Option<PathBuf>,
excluded_window_ids: Arc<Mutex<Vec<u32>>>,
) -> Self {
Self::start_with_budget(
target,
video_output_path,
excluded_window_ids,
Arc::default(),
CANDIDATE_BUDGET_BYTES,
MovieQuality::Compact,
PortalToken::None,
)
}
pub fn start_with_budget(
target: TargetInfo,
video_output_path: Option<PathBuf>,
excluded_window_ids: Arc<Mutex<Vec<u32>>>,
own_windows_recorded: Arc<AtomicBool>,
budget_bytes: usize,
movie_quality: MovieQuality,
portal_token: PortalToken,
) -> Self {
let started_at = Instant::now();
#[cfg(target_os = "macos")]
let started_host = capture_core::HostTime::now();
let stopping = Arc::new(AtomicBool::new(false));
let urgent_snapshot = Arc::new(AtomicBool::new(true));
let frames = Arc::new(Mutex::new(CandidateBuffer::new(budget_bytes)));
let thread_stopping = Arc::clone(&stopping);
let thread_urgent = Arc::clone(&urgent_snapshot);
let thread_frames = Arc::clone(&frames);
let failure = Arc::new(Mutex::new(None));
let thread_failure = Arc::clone(&failure);
let sampler_target = target;
let sampler_thread = thread::Builder::new()
.name("scrawl-frame-sampler".into())
.spawn(move || {
platform::run_sampler(Sampler {
target: sampler_target,
video_output_path,
movie_quality,
portal_token,
excluded_window_ids,
own_windows_recorded,
started_at,
#[cfg(target_os = "macos")]
started_host,
stopping: thread_stopping,
urgent: thread_urgent,
frames: thread_frames,
failure: thread_failure,
#[cfg(target_os = "windows")]
video_offset_ms: Arc::new(AtomicU64::new(0)),
})
})
.ok();
Self {
started_at,
stopping,
urgent_snapshot,
frames,
failure,
sampler_thread,
_recording: crate::retention::Recording::start(),
}
}
/// Why the screen source gave up before the stop, if it did: nothing more of the screen
/// will be recorded.
pub fn failure(&self) -> Option<String> {
self.failure
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner)
.clone()
}
pub fn request_immediate_frame(&self) {
self.urgent_snapshot.store(true, Ordering::Release);
if let Some(ref handle) = self.sampler_thread {
handle.thread().unpark();
}
}
pub fn elapsed_ms(&self) -> u64 {
self.started_at.elapsed().as_millis() as u64
}
/// Stops at once: the movie keeps finishing on its own (see [`Movie`]).
pub fn stop(mut self) -> ScreenRecording {
let duration_ms = self.elapsed_ms();
self.stopping.store(true, Ordering::Release);
let video = self
.sampler_thread
.take()
.and_then(|handle| {
handle.thread().unpark();
handle.join().ok()
})
.flatten();
let buffer = std::mem::replace(
&mut *self
.frames
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner),
CandidateBuffer::new(0),
);
let frames = buffer.into_frames();
// Never a made-up frame: the capture goes with its voice and marks alone, and says why.
let no_screen = frames.is_empty().then(|| no_screen_reason(self.failure()));
ScreenRecording {
frames,
video,
duration_ms: duration_ms.max(100),
no_screen,
}
}
}
impl Drop for ScreenCaptureSession {
fn drop(&mut self) {
self.stopping.store(true, Ordering::Release);
if let Some(handle) = self.sampler_thread.take() {
handle.thread().unpark();
let _ = handle.join();
}
}
}
/// Shared state collected by the global input monitor during an active capture session.
/// Monotonic boot-time nanoseconds (`mach_absolute_time` scaled by `mach_timebase_info` on macOS,
/// matching CoreAudio and CGEvent timestamps).
pub fn boot_time_ns() -> u64 {
#[cfg(target_os = "macos")]
{
#[repr(C)]
struct MachTimebaseInfo {
numer: u32,
denom: u32,
}
unsafe extern "C" {
fn mach_absolute_time() -> u64;
fn mach_timebase_info(info: *mut MachTimebaseInfo) -> i32;
}
static TIMEBASE: std::sync::OnceLock<(u32, u32)> = std::sync::OnceLock::new();
let &(numer, denom) = TIMEBASE.get_or_init(|| {
let mut info = MachTimebaseInfo { numer: 0, denom: 0 };
let status = unsafe { mach_timebase_info(&mut info) };
if status == 0 && info.denom != 0 {
(info.numer, info.denom)
} else {
(1, 1)
}
});
let ticks = unsafe { mach_absolute_time() };
let nanos = u128::from(ticks) * u128::from(numer) / u128::from(denom);
u64::try_from(nanos).unwrap_or(u64::MAX)
}
#[cfg(not(target_os = "macos"))]
{
static EPOCH: std::sync::OnceLock<Instant> = std::sync::OnceLock::new();
EPOCH.get_or_init(Instant::now).elapsed().as_nanos() as u64
}
}
#[derive(Clone, Debug)]
pub struct InkSnapshot {
pub target: TargetFrame,
pub strokes: Vec<Stroke>,
pub live: Option<Stroke>,
pub laser: Option<Stroke>,
pub glow: bool,
}
/// A raw input event on the capture's clock, in global top-left screen points.
#[derive(Clone, Debug, PartialEq)]
pub enum InputEvent {
ButtonDown {
t_ms: u64,
x: f64,
y: f64,
right: bool,
shift_only: bool,
},
ButtonUp {
t_ms: u64,
x: f64,
y: f64,
right: bool,
shift_only: bool,
},
Scroll {
t_ms: u64,
x: f64,
y: f64,
dx: f64,
dy: f64,
},
Move {
t_ms: u64,
x: f64,
y: f64,
shift_only: bool,
},
Laser {
t_ms: u64,
x: f64,
y: f64,
},
KeyChord {
t_ms: u64,
shortcut: String,
},
Flag {
t_ms: u64,
},
}
/// Whether `shift` is held alone without other modifiers and is not left over from a multi-modifier
/// chord (such as the capture shortcut `⌥⇧S`) before `shift` was released.
pub fn pure_shift_state(
shift: bool,
other_modifiers: bool,
await_shift_release: &mut bool,
) -> bool {
if !shift {
*await_shift_release = false;
} else if other_modifiers {
*await_shift_release = true;
}
shift && !other_modifiers && !*await_shift_release
}
/// Ink and pointer input in global top-left points, ms since capture start.
#[derive(Clone)]
pub struct InputSink {
telemetry: Arc<SharedCaptureTelemetry>,
}
impl Default for InputSink {
fn default() -> Self {
let (tx, _rx) = flume::unbounded();
Self {
telemetry: SharedCaptureTelemetry::new(tx, ShortcutConfig::ansi_defaults()),
}
}
}
impl InputSink {
pub fn new(telemetry: Arc<SharedCaptureTelemetry>) -> Self {
Self { telemetry }
}
pub fn telemetry(&self) -> &Arc<SharedCaptureTelemetry> {
&self.telemetry
}
pub fn send(&self, event: InputEvent) {
self.telemetry.handle_input(event);
}
pub fn button_down(&self, t_ms: u64, x: f64, y: f64, right: bool, shift_only: bool) {
self.send(InputEvent::ButtonDown {
t_ms,
x,
y,
right,
shift_only,
});
}
pub fn button_up(&self, t_ms: u64, x: f64, y: f64, right: bool, shift_only: bool) {
self.send(InputEvent::ButtonUp {
t_ms,
x,
y,
right,
shift_only,
});
}
pub fn pointer_down(&self, t_ms: u64, x: f64, y: f64) {
let shift_only = self.telemetry.shift_held.load(Ordering::Acquire);
self.button_down(t_ms, x, y, false, shift_only);
}
pub fn pointer_up(&self, t_ms: u64, x: f64, y: f64, right: bool) {
let shift_only = self.telemetry.shift_held.load(Ordering::Acquire);
self.button_up(t_ms, x, y, right, shift_only);
}
pub fn shift_down(&self, t_ms: u64, x: f64, y: f64) {
self.button_down(t_ms, x, y, false, true);
}
pub fn shift_up(&self) -> bool {
self.telemetry.shift_up()
}
pub fn scroll(&self, t_ms: u64, x: f64, y: f64, dx: f64, dy: f64) {
self.send(InputEvent::Scroll { t_ms, x, y, dx, dy });
}
pub fn ink(&self, t_ms: u64, x: f64, y: f64, shift_held: bool) {
self.send(InputEvent::Move {
t_ms,
x,
y,
shift_only: shift_held,
});
}
pub fn laser(&self, t_ms: u64, x: f64, y: f64) {
self.send(InputEvent::Laser { t_ms, x, y });
}
pub fn key_chord(&self, t_ms: u64, shortcut: impl Into<String>) {
self.send(InputEvent::KeyChord {
t_ms,
shortcut: shortcut.into(),
});
}
pub fn flag(&self, t_ms: u64) {
self.send(InputEvent::Flag { t_ms });
}
pub fn snapshot(&self) -> InkSnapshot {
self.telemetry.ink_snapshot()
}
}
/// A drag's start: when (ms), where (normalized), its Accessibility label, and the frame.
type DragStart = (u64, f32, f32, Option<String>, TargetFrame);
pub struct SharedCaptureTelemetry {
pub active: AtomicBool,
pub selecting: AtomicBool,
pub shift_held: AtomicBool,
pub capture_mode: AtomicU64,
/// The record shortcut (`crate::settings` loads and saves it).
pub shortcut: Mutex<ShortcutConfig>,
pub paste_shortcut: Mutex<ShortcutConfig>,
/// Settings is recording a new shortcut: none of Scrawl's fire (`pause_shortcuts`).
pub shortcuts_paused: AtomicBool,
pub started_at: Mutex<Instant>,
pub started_at_boot_ns: AtomicU64,
pub picker_started_at: Mutex<Instant>,
pub picker_rect: Mutex<Option<([f32; 4], u8)>>,
pub target_frame: Mutex<TargetFrame>,
pub pointer_screen: Mutex<[f32; 2]>,
pub events: Mutex<Vec<InteractionEvent>>,
pub strokes: Mutex<Vec<Stroke>>,
pub flags: Mutex<Vec<u64>>,
pub active_laser: Mutex<Option<Stroke>>,
pub live_ink: Mutex<Option<Stroke>>,
pub excluded_window_ids: Arc<Mutex<Vec<u32>>>,
/// Raised when a click, ink stroke or shortcut starts, so the Screen-mode window log takes
/// a snapshot right away.
pub window_log_request: Arc<AtomicBool>,
pub ui_sender: flume::Sender<Event>,
/// Scrawl's windows that let clicks through (the ink overlay), skipped when deciding whether
/// a click landed on Scrawl itself.
pub passthrough_window_ids: Mutex<Vec<u32>>,
/// The `scrawl-helper` whose window is what's being recorded (the iPhone mirror): its clicks
/// count, unlike those on other helper windows (Settings, onboarding). 0 when none.
capture_source_pid: AtomicI32,
/// The last `on_scrawl_window` answer (when, where, what), reused for a burst of scrolls.
scrawl_hit: Mutex<Option<(Instant, f64, f64, bool)>>,
/// macOS's key event tap (`CFMachPortRef`), made once Accessibility is on (`check_key_tap`).
#[cfg_attr(any(target_os = "windows", target_os = "linux"), allow(dead_code))]
tap_port: AtomicU64,
/// The key event tap thread's run loop (retained), so `check_key_tap` can stop it.
#[cfg_attr(any(target_os = "windows", target_os = "linux"), allow(dead_code))]
tap_run_loop: AtomicU64,
/// A key tap thread is being started.
#[cfg_attr(any(target_os = "windows", target_os = "linux"), allow(dead_code))]
tap_starting: AtomicBool,
/// When the last key tap was tried (ms since `UNIX_EPOCH`), to space out retries.
#[cfg_attr(any(target_os = "windows", target_os = "linux"), allow(dead_code))]
tap_tried_ms: AtomicU64,
/// Whether key presses are arriving (macOS's key tap, Windows' keyboard hook).
tap_live: AtomicBool,
/// Esc went down since the UI loop last looked (`take_escape`).
escape_pressed: AtomicBool,
/// A Screen recording started straight from the shortcut: the overlay flashes an edge glow.
pub start_glow: AtomicBool,
/// Clicks are arriving (macOS's global mouse monitor, Windows' mouse hook). Until then the
/// poller records clicks from polled button state.
mouse_live: AtomicBool,
last_shortcut_ms: AtomicU64,
drag_start: Mutex<Option<DragStart>>,
/// The `agentContext` experiment was on when this capture started: clicks and strokes are
/// looked up in the Accessibility tree (`crate::ax`, macOS only).
agent_context: AtomicBool,
/// A ⇧-press inside the target: when, where (global points) and on which frame.
shift_press: Mutex<Option<(u64, f64, f64, TargetFrame)>>,
/// Monotonic session counter incremented each time a capture starts on this telemetry.
session_id: AtomicU64,
}
impl SharedCaptureTelemetry {
pub fn new(
ui_sender: flume::Sender<Event>,
shortcuts: (ShortcutConfig, ShortcutConfig),
) -> Arc<Self> {
let (record, paste) = shortcuts;
Arc::new(Self {
active: AtomicBool::new(false),
selecting: AtomicBool::new(false),
shift_held: AtomicBool::new(false),
capture_mode: AtomicU64::new(0),
shortcut: Mutex::new(record),
paste_shortcut: Mutex::new(paste),
shortcuts_paused: AtomicBool::new(false),
started_at: Mutex::new(Instant::now()),
started_at_boot_ns: AtomicU64::new(boot_time_ns()),
picker_started_at: Mutex::new(Instant::now()),
picker_rect: Mutex::new(None),
target_frame: Mutex::new(TargetFrame {
x: 0.0,
y: 0.0,
width: 1440.0,
height: 900.0,
scale: 2.0,
}),
pointer_screen: Mutex::new([0.0, 0.0]),
events: Mutex::new(Vec::new()),
strokes: Mutex::new(Vec::new()),
flags: Mutex::new(Vec::new()),
active_laser: Mutex::new(None),
live_ink: Mutex::new(None),
excluded_window_ids: Arc::new(Mutex::new(Vec::new())),
window_log_request: Arc::new(AtomicBool::new(false)),
ui_sender,
passthrough_window_ids: Mutex::new(Vec::new()),
capture_source_pid: AtomicI32::new(0),
scrawl_hit: Mutex::new(None),
tap_port: AtomicU64::new(0),
tap_run_loop: AtomicU64::new(0),
tap_starting: AtomicBool::new(false),
tap_tried_ms: AtomicU64::new(0),
tap_live: AtomicBool::new(false),
escape_pressed: AtomicBool::new(false),
start_glow: AtomicBool::new(false),
mouse_live: AtomicBool::new(false),
last_shortcut_ms: AtomicU64::new(0),
drag_start: Mutex::new(None),
agent_context: AtomicBool::new(false),
shift_press: Mutex::new(None),
session_id: AtomicU64::new(0),
})
}
/// Each source (Carbon or `RegisterHotKey`, the key tap or hook) reports the same press within
/// a few ms; a press after that is a new one, as when S is pressed again to switch modes.
pub fn emit_capture_shortcut(&self) {
if self.shortcuts_paused.load(Ordering::Acquire) {
return;
}
let now_ms = std::time::SystemTime::now()
.duration_since(std::time::UNIX_EPOCH)
.unwrap_or_default()
.as_millis() as u64;
let prev = self.last_shortcut_ms.load(Ordering::Acquire);
if now_ms.saturating_sub(prev) >= 90
&& self
.last_shortcut_ms
.compare_exchange(prev, now_ms, Ordering::AcqRel, Ordering::Acquire)
.is_ok()
{
scrawl_log("emit_capture_shortcut");
let _ = self.ui_sender.try_send(Event::CaptureShortcut);
}
}
pub fn emit_paste_shortcut(&self) {
if self.shortcuts_paused.load(Ordering::Acquire) {
return;
}
let now_ms = std::time::SystemTime::now()
.duration_since(std::time::UNIX_EPOCH)
.unwrap_or_default()
.as_millis() as u64;
let prev = self.last_shortcut_ms.load(Ordering::Acquire);
if now_ms.saturating_sub(prev) >= 400
&& self
.last_shortcut_ms
.compare_exchange(prev, now_ms, Ordering::AcqRel, Ordering::Acquire)
.is_ok()
{
scrawl_log("emit_paste_shortcut -> sending PasteLatestShortcut");
let _ = self.ui_sender.try_send(Event::PasteLatestShortcut);
}
}
pub fn matches_shortcut(
&self,
keycode: u16,
cmd: bool,
ctrl: bool,
alt: bool,
shift: bool,
) -> bool {
let cfg = self
.shortcut
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner)
.clone();
is_capture_shortcut(&cfg, layout_s_keycode(), keycode, cmd, ctrl, alt, shift)
}
/// The record and paste shortcuts.
pub fn shortcuts(&self) -> (ShortcutConfig, ShortcutConfig) {
let lock = |m: &Mutex<ShortcutConfig>| {
m.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner)
.clone()
};
(lock(&self.shortcut), lock(&self.paste_shortcut))
}
pub fn set_capture_mode(&self, mode: CaptureMode) {
let code = match mode {
CaptureMode::Display => 0,
CaptureMode::Region => 1,
CaptureMode::Window => 2,
CaptureMode::Device => 3,
};
self.capture_mode.store(code, Ordering::Release);
}
pub fn begin_picker_session(&self) {
*self
.picker_started_at
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner) = Instant::now();
*self
.picker_rect
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner) = None;
self.active.store(false, Ordering::Release);
self.selecting.store(true, Ordering::Release);
}
pub fn set_picker_rect(&self, rect: Option<([f32; 4], u8)>) {
*self
.picker_rect
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner) = rect;
}
pub fn started_at_boot_ns(&self) -> u64 {
self.started_at_boot_ns.load(Ordering::Acquire)
}
pub fn reset_for_session(&self, frame: TargetFrame) {
self.reset_for_session_at(frame, None);
}
pub fn reset_for_session_at(&self, frame: TargetFrame, t0_boot_ns: Option<u64>) {
let agent_context = crate::handoff::experiment(crate::handoff::AGENT_CONTEXT);
let _ = self.reset_for_session_at_with_agent_context(frame, t0_boot_ns, agent_context);
}
pub fn reset_for_session_at_with_agent_context(
&self,
frame: TargetFrame,
t0_boot_ns: Option<u64>,
agent_context: bool,
) -> u64 {
let session_id = self.session_id.fetch_add(1, Ordering::AcqRel) + 1;
self.selecting.store(false, Ordering::Release);
*self
.picker_rect
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner) = None;
let now_boot_ns = boot_time_ns();
let boot_ns = t0_boot_ns.unwrap_or(now_boot_ns);
self.started_at_boot_ns.store(boot_ns, Ordering::Release);
let started_instant = Instant::now()
.checked_sub(Duration::from_nanos(now_boot_ns.saturating_sub(boot_ns)))
.unwrap_or_else(Instant::now);
*self
.started_at
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner) = started_instant;
*self
.target_frame
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner) = frame;
self.events
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner)
.clear();
self.strokes
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner)
.clear();
self.flags
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner)
.clear();
*self
.active_laser
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner) = None;
*self
.live_ink
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner) = None;
*self
.drag_start
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner) = None;
*self
.shift_press
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner) = None;
self.excluded_window_ids
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner)
.clear();
self.shift_held.store(false, Ordering::Release);
self.window_log_request.store(false, Ordering::Release);
self.agent_context.store(agent_context, Ordering::Release);
if agent_context {
crate::ax::Prober::shared().begin();
}
self.active.store(true, Ordering::Release);
let clicks = if self.mouse_live.load(Ordering::Acquire) {
platform::MOUSE_SOURCE
} else {
"polled mouse buttons"
};
let keys = if self.tap_live.load(Ordering::Acquire) {
platform::KEY_SOURCE
} else {
platform::NO_KEY_SOURCE
};
scrawl_log(format!("session clicks from {clicks}, keys from {keys}"));
session_id
}
/// Whether Esc went down since the last call (a pending shortcut or an open picker cancels on it).
pub fn take_escape(&self) -> bool {
self.escape_pressed.swap(false, Ordering::AcqRel)
}
pub fn elapsed_ms(&self) -> u64 {
self.started_at
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner)
.elapsed()
.as_millis() as u64
}
pub fn contains_screen_point(&self, screen_x: f64, screen_y: f64) -> bool {
let frame = *self
.target_frame
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner);
frame.width >= 16.0
&& frame.height >= 16.0
&& frame.x > -10_000.0
&& screen_x >= frame.x
&& screen_x <= frame.x + frame.width
&& screen_y >= frame.y
&& screen_y <= frame.y + frame.height
}
/// Whether to look up what clicks and strokes land on: `agentContext` is on, the target is
/// this Mac's screen (not the iPhone mirror) and Accessibility is granted.
fn probing(&self) -> bool {
self.agent_context.load(Ordering::Acquire)
&& self.capture_mode.load(Ordering::Acquire) != 3
&& accessibility_granted()
}
/// A mouse button went down at a global point.
pub fn pointer_down(&self, t_ms: u64, screen_x: f64, screen_y: f64) {
if !self.contains_screen_point(screen_x, screen_y) {
return;
}
#[cfg(target_os = "macos")]
{
let raw = objc2::rc::autoreleasepool(|_| crate::windows::raw_windows());
if crate::windows::system_prompt_at(&raw, screen_x, screen_y) {
return;
}
}
let (nx, ny, _, _) = self.normalize_point(screen_x, screen_y);
self.window_log_request.store(true, Ordering::Release);
let start_frame = *self
.target_frame
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner);
*self
.drag_start
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner) =
Some((t_ms, nx, ny, None, start_frame));
if self.probing() {
crate::ax::Prober::shared().probe(crate::ax::Target::Pointer(t_ms), screen_x, screen_y);
}
}
/// A mouse button came up: records the click or drag that `pointer_down` began.
pub fn pointer_up(&self, t_ms: u64, screen_x: f64, screen_y: f64, right: bool) {
let Some((t0, x0, y0, start_label, start_frame)) = self
.drag_start
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner)
.take()
else {
return;
};
let end_frame = *self
.target_frame
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner);
if !same_target_frame(start_frame, end_frame) {
return;
}
let (nx, ny, fw, fh) = self.normalize_point(screen_x, screen_y);
let dist_px = ((nx - x0) * fw as f32).hypot((ny - y0) * fh as f32);
let _ = self.ui_sender.try_send(Event::RequestSnapshot);
let event = if dist_px < CLICK_SLOP_PT {
InteractionEvent::Click {
t: t0,
x: x0,
y: y0,
button: Some(if right { "right" } else { "left" }.into()),
label: start_label,
role: None,
}
} else {
InteractionEvent::Drag {
t: t0,
t1: t_ms,
x0,
y0,
x1: nx,
y1: ny,
label: start_label,
button: None,
path: Vec::new(),
}
};
self.events
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner)
.push(event);
}
/// The left button went down with only ⇧ held: drawing, never a click (`shift_up`).
pub fn shift_down(&self, t_ms: u64, screen_x: f64, screen_y: f64) {
if !self.contains_screen_point(screen_x, screen_y) {
return;
}
let frame = *self
.target_frame
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner);
*self
.shift_press
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner) =
Some((t_ms, screen_x, screen_y, frame));
}
/// The button of a ⇧-press came up (with or without ⇧ still held). A ⇧-press is part of
/// the drawing, never a click: returns whether one was waiting, so the release isn't taken
/// as a click's.
pub fn shift_up(&self) -> bool {
self.shift_press
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner)
.take()
.is_some()
}
/// The helper process whose window is the capture target (the iPhone mirror).
#[cfg_attr(any(target_os = "windows", target_os = "linux"), allow(dead_code))]
pub fn set_capture_source_pid(&self, pid: u32) {
self.capture_source_pid
.store(i32::try_from(pid).unwrap_or(0), Ordering::Release);
}
/// Whether the topmost window at a global point is Scrawl's own or a `scrawl-helper`'s other
/// than the capture source (ignoring the click-through ink overlay). Asked once per press, and
/// for scrolls, which come in bursts at one point, at most every 250 ms.
fn on_scrawl_window(&self, x: f64, y: f64) -> bool {
let mut hit = self
.scrawl_hit
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner);
if let Some((at, hx, hy, on)) = *hit
&& at.elapsed() < Duration::from_millis(250)
&& (hx - x).abs() <= 2.0
&& (hy - y).abs() <= 2.0
{
return on;
}
let on = self.scrawl_window_in(
&platform::raw_windows_now(),
x,
y,
platform::is_helper_process,
);
*hit = Some((Instant::now(), x, y, on));
on
}
fn scrawl_window_in(
&self,
windows: &[crate::windows::RawWindow],
x: f64,
y: f64,
is_helper: impl Fn(i32) -> bool,
) -> bool {
let passthrough = self
.passthrough_window_ids
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner)
.clone();
let excluded = self
.excluded_window_ids
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner)
.clone();
let source = self.capture_source_pid.load(Ordering::Acquire);
on_own_window(windows, (x, y), &passthrough, &excluded, source, is_helper)
}
pub fn normalize_point(&self, screen_x: f64, screen_y: f64) -> (f32, f32, u32, u32) {
let frame = *self
.target_frame
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner);
let nx = ((screen_x - frame.x) / frame.width.max(1.0)).clamp(0.0, 1.0) as f32;
let ny = ((screen_y - frame.y) / frame.height.max(1.0)).clamp(0.0, 1.0) as f32;
(
nx,
ny,
frame.width.round().max(1.0) as u32,
frame.height.round().max(1.0) as u32,
)
}
pub fn finish_active_laser(&self, _ax_point: Option<(f64, f64)>) {
if let Some(mut laser) = self
.active_laser
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner)
.take()
&& !laser.pts.is_empty()
{
if let Some(last) = laser.pts.as_chunks::<3>().0.last() {
let lx = last[0] as f32 / SCALE as f32;
let ly = last[1] as f32 / SCALE as f32;
laser.t1 = laser.t0.saturating_add(last[2].max(0) as u64);
self.events
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner)
.push(InteractionEvent::Point {
t: laser.t1,
x: lx,
y: ly,
label: None,
});
}
self.strokes
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner)
.push(laser);
}
}
pub fn finish_session(&self) -> (Vec<InteractionEvent>, Vec<Stroke>) {
self.active.store(false, Ordering::Release);
self.selecting.store(false, Ordering::Release);
*self
.drag_start
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner) = None;
*self
.shift_press
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner) = None;
self.finish_active_laser(None);
if let Some(ink) = self
.live_ink
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner)
.take()
&& ink.pts.len() >= 3
{
self.strokes
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner)
.push(ink);
}
let mut events = std::mem::take(
&mut *self
.events
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner),
);
let mut strokes = std::mem::take(
&mut *self
.strokes
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner),
);
if self.agent_context.swap(false, Ordering::AcqRel) {
let hits = crate::ax::Prober::shared().finish();
crate::ax::apply(&hits, &mut events, &mut strokes);
if self.capture_mode.load(Ordering::Acquire) != 3 {
let end = crate::pipeline::now_epoch_ms();
let start = end.saturating_sub(self.elapsed_ms());
let page = crate::page::between(start, end);
crate::page::apply(&page, start, &mut events, &mut strokes);
}
}
(events, strokes)
}
pub fn handle_input(&self, event: InputEvent) {
self.handle_input_with(event, |x, y| self.on_scrawl_window(x, y));
}
pub fn handle_input_with(&self, event: InputEvent, on_own_window: impl Fn(f64, f64) -> bool) {
match event {
// A ⇧-press is never a click: ⇧ is drawing.
InputEvent::ButtonDown {
t_ms,
x,
y,
right,
shift_only,
} => {
if !on_own_window(x, y) {
if !shift_only {
self.pointer_down(t_ms, x, y);
} else if !right {
self.shift_down(t_ms, x, y);
}
}
}
InputEvent::ButtonUp {
t_ms,
x,
y,
right,
shift_only,
} => {
let shift_press = !right && self.shift_up();
if !shift_press && !shift_only {
self.pointer_up(t_ms, x, y, right);
}
}
InputEvent::Scroll { t_ms, x, y, dx, dy } => {
if (dx.abs() > 0.0 || dy.abs() > 0.0) && !on_own_window(x, y) {
self.scroll_at(t_ms, x, y, dx as f32, dy as f32);
}
}
InputEvent::Move {
t_ms,
x,
y,
shift_only,
} => {
*self
.pointer_screen
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner) = [x as f32, y as f32];
self.track_ink_at(t_ms, (x, y), shift_only);
}
InputEvent::Laser { t_ms, x, y } => {
self.laser_at(t_ms, x, y);
}
InputEvent::KeyChord { t_ms, shortcut } => {
self.key_chord(t_ms, shortcut);
}
InputEvent::Flag { t_ms } => {
self.flag_at(t_ms);
}
}
}
pub fn key_chord(&self, t_ms: u64, shortcut: String) {
self.window_log_request.store(true, Ordering::Release);
self.events
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner)
.push(InteractionEvent::Key { t: t_ms, shortcut });
}
pub fn scroll_at(&self, t_ms: u64, screen_x: f64, screen_y: f64, dx: f32, dy: f32) {
if (dx.abs() > 0.0 || dy.abs() > 0.0) && self.contains_screen_point(screen_x, screen_y) {
let (nx, ny, _, _) = self.normalize_point(screen_x, screen_y);
self.events
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner)
.push(InteractionEvent::Scroll {
t: t_ms,
x: nx,
y: ny,
dx,
dy,
label: None,
});
}
}
pub fn track_ink_at(&self, t_ms: u64, (x, y): (f64, f64), shift_held: bool) {
let prev = self.shift_held.load(Ordering::Acquire)
|| self
.live_ink
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner)
.is_some();
track_shift_ink_at(self, t_ms, (x, y), shift_held, prev);
}
pub fn laser_at(&self, t_ms: u64, screen_x: f64, screen_y: f64) {
if !self.contains_screen_point(screen_x, screen_y) {
return;
}
let (nx, ny, fw, fh) = self.normalize_point(screen_x, screen_y);
let sx = (nx * SCALE as f32).round() as i32;
let sy = (ny * SCALE as f32).round() as i32;
let mut guard = self
.active_laser
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner);
let stroke = guard.get_or_insert_with(|| Stroke {
t0: t_ms,
t1: t_ms,
fw,
fh,
on: None,
pin: None,
laser: true,
pts: Vec::new(),
});
let dt = t_ms.saturating_sub(stroke.t0) as i32;
if stroke.pts.len() / 3 < 2_400 {
stroke.pts.extend_from_slice(&[sx, sy, dt]);
stroke.t1 = t_ms;
}
}
pub fn flag_at(&self, t_ms: u64) {
self.flags
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner)
.push(t_ms);
let _ = self.ui_sender.try_send(Event::RequestSnapshot);
}
pub fn take_flags(&self) -> Vec<u64> {
std::mem::take(
&mut *self
.flags
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner),
)
}
pub fn ink_snapshot(&self) -> InkSnapshot {
let target = *self
.target_frame
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner);
let strokes = self
.strokes
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner)
.clone();
let live = self
.live_ink
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner)
.clone();
let laser = self
.active_laser
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner)
.clone();
let glow = self.start_glow.load(Ordering::Acquire);
InkSnapshot {
target,
strokes,
live,
laser,
glow,
}
}
}
/// Clicks from the mouse buttons' polled state, while the mouse monitor (or hook) is missing: a
/// press starts a click (unless it lands on Scrawl's own window), a release ends it. A ⇧-press may
/// be a pin, as in the mouse monitor's handler.
#[cfg_attr(target_os = "linux", allow(dead_code))]
fn poll_clicks(
telemetry: &SharedCaptureTelemetry,
prev: &mut [bool; 2],
now: [bool; 2],
(x, y): (f64, f64),
pure_shift: bool,
on_scrawl: impl Fn(f64, f64) -> bool,
) {
let was = std::mem::replace(prev, now);
if telemetry.mouse_live.load(Ordering::Acquire) {
return;
}
let t_ms = telemetry.elapsed_ms();
for (button, (&down, &before)) in now.iter().zip(&was).enumerate() {
let right = button != 0;
if down && !before {
telemetry.handle_input_with(
InputEvent::ButtonDown {
t_ms,
x,
y,
right,
shift_only: pure_shift,
},
&on_scrawl,
);
} else if !down && before {
telemetry.handle_input_with(
InputEvent::ButtonUp {
t_ms,
x,
y,
right,
shift_only: pure_shift,
},
&on_scrawl,
);
}
}
}
/// Whether the topmost window under global point `at` (in `windows`, front to back, looking
/// through `passthrough` ones) is the host's own: this process's chrome
/// ([`is_own_chrome`](crate::windows::is_own_chrome), with the host's `excluded` windows), or a
/// helper's other than the app being captured (`source`). Clicks and scrolls there aren't part
/// of what is being shown; on the host's normal windows (Hex's Settings) they are.
pub fn on_own_window(
windows: &[crate::windows::RawWindow],
(x, y): (f64, f64),
passthrough: &[u32],
excluded: &[u32],
source: i32,
is_helper: impl Fn(i32) -> bool,
) -> bool {
let own = std::process::id() as i32;
crate::windows::owned_window_at(windows, x, y, passthrough, |w| {
crate::windows::is_own_chrome(w, own, excluded) || (w.pid != source && is_helper(w.pid))
})
}
/// A press that moves less than this (points) before it's let go is a click, not a drag.
pub const CLICK_SLOP_PT: f32 = 12.0;
/// Ink spanning less than this (points) is a stationary ⇧ tap, not a stroke.
pub const MIN_INK_SPAN_PT: f32 = 6.0;
/// Whether `stroke`'s points before `before_ms` reach at least `MIN_INK_SPAN_PT` from its start.
pub fn drew_before(stroke: &Stroke, before_ms: u64) -> bool {
let Some(first) = stroke.pts.get(0..2) else {
return false;
};
let mut points = 0_usize;
let mut span = 0.0_f32;
for p in stroke
.pts
.as_chunks::<3>()
.0
.iter()
.take_while(|p| stroke.t0.saturating_add(p[2].max(0) as u64) < before_ms)
{
points += 1;
let dx = (p[0] - first[0]) as f32 / SCALE as f32 * stroke.fw as f32;
let dy = (p[1] - first[1]) as f32 / SCALE as f32 * stroke.fh as f32;
span = span.max(dx.hypot(dy));
}
points >= 2 && span >= MIN_INK_SPAN_PT
}
/// Whether finished ink is worth keeping as a stroke.
pub fn is_real_ink(stroke: &Stroke) -> bool {
drew_before(stroke, u64::MAX)
}
fn same_target_frame(a: TargetFrame, b: TargetFrame) -> bool {
(a.x - b.x).abs() <= 0.5
&& (a.y - b.y).abs() <= 0.5
&& (a.width - b.width).abs() <= 0.5
&& (a.height - b.height).abs() <= 0.5
}
/// One poll of the Shift ink: holding Shift alone and moving the cursor inside the capture
/// target draws a persistent stroke; letting go of Shift commits it. `pure_shift` is Shift
/// without other modifiers (and not left over from a chord such as the capture shortcut).
#[cfg_attr(target_os = "linux", allow(dead_code))]
fn track_shift_ink(
telemetry: &SharedCaptureTelemetry,
(x, y): (f64, f64),
pure_shift: bool,
prev_shift_ink: bool,
) {
track_shift_ink_at(
telemetry,
telemetry.elapsed_ms(),
(x, y),
pure_shift,
prev_shift_ink,
);
}
fn track_shift_ink_at(
telemetry: &SharedCaptureTelemetry,
t_ms: u64,
(x, y): (f64, f64),
pure_shift: bool,
prev_shift_ink: bool,
) {
let inside_target = telemetry.contains_screen_point(x, y);
if pure_shift && inside_target {
telemetry.shift_held.store(true, Ordering::Release);
let (nx, ny, fw, fh) = telemetry.normalize_point(x, y);
let sx = (nx * SCALE as f32).round() as i32;
let sy = (ny * SCALE as f32).round() as i32;
let mut guard = telemetry
.live_ink
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner);
if guard.is_none() {
telemetry.window_log_request.store(true, Ordering::Release);
}
let stroke = guard.get_or_insert_with(|| Stroke {
t0: t_ms,
t1: t_ms,
fw,
fh,
on: None,
pin: None,
laser: false,
pts: Vec::new(),
});
let dt = t_ms.saturating_sub(stroke.t0) as i32;
let should_push = stroke.pts.as_chunks::<3>().0.last().is_none_or(|prev| {
let dx = (sx - prev[0]) as f32 / SCALE as f32 * fw as f32;
let dy = (sy - prev[1]) as f32 / SCALE as f32 * fh as f32;
dx.hypot(dy) >= 1.5
});
if should_push && stroke.pts.len() / 3 < 2_400 {
stroke.pts.extend_from_slice(&[sx, sy, dt]);
stroke.t1 = t_ms;
}
} else if !pure_shift && prev_shift_ink {
telemetry.shift_held.store(false, Ordering::Release);
if let Some(stroke) = telemetry
.live_ink
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner)
.take()
// Ignore stationary Shift taps (e.g. typing a capital letter without moving the mouse).
&& is_real_ink(&stroke)
{
if telemetry.probing() {
let frame = *telemetry
.target_frame
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner);
if let Some((x, y)) = crate::ax::stroke_probe_point(&stroke, frame) {
crate::ax::Prober::shared().probe(crate::ax::Target::Stroke(stroke.t0), x, y);
}
}
telemetry
.strokes
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner)
.push(stroke);
let _ = telemetry.ui_sender.try_send(Event::RequestSnapshot);
}
}
}
pub fn is_frontmost_terminal() -> bool {
let Some((_pid, app, bundle_id)) = frontmost_app_info() else {
return false;
};
let a = app.to_ascii_lowercase();
let b = bundle_id.unwrap_or_default().to_ascii_lowercase();
[
"ghostty",
"iterm",
"terminal",
"kitty",
"wezterm",
"alacritty",
"warp",
// Windows Terminal, the console host and the shells it runs.
"windowsterminal",
"conhost",
"powershell",
"pwsh",
"cmd.exe",
"mintty",
]
.iter()
.any(|needle| a.contains(needle) || b.contains(needle))
}
/// Writes all bundle artifacts (`scrawl.json`, `summary.md`, `contact-sheet.png`, `frames/frame-XX.png`)
/// into a new or specified `.scrawl` bundle directory. The manifest comes last, whole, so a
/// bundle with one is complete.
pub fn write_bundle_artifacts(
out_dir: &Path,
bundle: &crate::model::ScrawlBundle,
annotated_frames: &[(crate::model::Keyframe, RgbaFrame)],
contact_sheet: &RgbaFrame,
summary_md: &str,
) -> Result<()> {
write_bundle_files(out_dir, bundle, annotated_frames, contact_sheet, summary_md)?;
publish_manifest(out_dir, bundle)
}
/// Writes `scrawl.json` atomically (a temporary file renamed into place): the last step of a
/// build, which marks the bundle finished.
pub fn publish_manifest(out_dir: &Path, bundle: &crate::model::ScrawlBundle) -> Result<()> {
let manifest_bytes = serde_json::to_vec_pretty(bundle)?;
let tmp = out_dir.join(".scrawl.json.tmp");
crate::routing::undo::write_private(&tmp, &manifest_bytes)
.with_context(|| format!("could not write {}", tmp.display()))?;
if let Err(error) = std::fs::rename(&tmp, out_dir.join("scrawl.json")) {
let _ = std::fs::remove_file(&tmp);
return Err(error).context("could not publish scrawl.json");
}
Ok(())
}
/// Everything [`write_bundle_artifacts`] writes except the manifest.
pub fn write_bundle_files(
out_dir: &Path,
bundle: &crate::model::ScrawlBundle,
annotated_frames: &[(crate::model::Keyframe, RgbaFrame)],
contact_sheet: &RgbaFrame,
summary_md: &str,
) -> Result<()> {
std::fs::create_dir_all(out_dir.join("frames"))
.with_context(|| format!("could not create {}", out_dir.display()))?;
#[cfg(unix)]
{
use std::os::unix::fs::PermissionsExt;
let _ = std::fs::set_permissions(out_dir, std::fs::Permissions::from_mode(0o700));
}
thread::scope(|scope| -> Result<()> {
let workers: Vec<_> = annotated_frames
.iter()
.map(|(kf, frame)| {
scope.spawn(move || -> Result<()> {
crate::render::save_png(frame, &out_dir.join(&kf.file))
})
})
.collect();
crate::render::save_png(contact_sheet, &out_dir.join(&bundle.contact_sheet))?;
for worker in workers {
worker
.join()
.map_err(|_| anyhow::anyhow!("PNG encoding worker panicked"))??;
}
Ok(())
})?;
crate::routing::undo::write_private(&out_dir.join("summary.md"), summary_md.as_bytes())?;
Ok(())
}
pub const SCRAWL_BUNDLE_ID: &str = "dev.kitlangton.scrawl";
/// Returns `true` when another process with bundle identifier `dev.kitlangton.scrawl` is running.
pub fn is_scrawl_app_running() -> bool {
#[cfg(target_os = "macos")]
{
use objc2_app_kit::NSRunningApplication;
use objc2_foundation::NSString;
let current_pid = std::process::id() as i32;
let apps = NSRunningApplication::runningApplicationsWithBundleIdentifier(
&NSString::from_str(SCRAWL_BUNDLE_ID),
);
apps.iter()
.any(|app| app.processIdentifier() != current_pid && !app.isTerminated())
}
#[cfg(not(target_os = "macos"))]
{
false
}
}
/// Explicit ownership switches for machine-wide shared resources so an embedded host (such as
/// Hex) can yield while Scrawl (`dev.kitlangton.scrawl`) runs instead of colliding on port 47777,
/// racing `.deliver.json` retries, or registering conflicting global shortcuts.
///
/// Defaults to `false` for all resources so [`Engine::new`] never claims machine-wide resources
/// unless the host explicitly opts in.
#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)]
pub struct SharedResources {
/// Bind the ScrawlKit HTTP ingest server (`DEFAULT_INGEST_PORT`, 47777) and USB poller.
pub ingest_server: bool,
/// Run background `.deliver.json` OpenCode delivery retries.
pub delivery_retries: bool,
/// Register the global capture and paste shortcuts (`⌥⇧S`, `⌥⇧V`).
pub global_shortcuts: bool,
}
impl SharedResources {
pub const fn none() -> Self {
Self {
ingest_server: false,
delivery_retries: false,
global_shortcuts: false,
}
}
pub const fn all() -> Self {
Self {
ingest_server: true,
delivery_retries: true,
global_shortcuts: true,
}
}
/// Claims the shared resources only when `peer_running` is `false`.
pub const fn when_peer_absent(peer_running: bool) -> Self {
if peer_running {
Self::none()
} else {
Self::all()
}
}
/// Claims the shared resources only when no standalone Scrawl app (`dev.kitlangton.scrawl`)
/// is running.
pub fn when_scrawl_absent() -> Self {
Self::when_peer_absent(is_scrawl_app_running())
}
pub const fn with_ingest_server(mut self, own: bool) -> Self {
self.ingest_server = own;
self
}
pub const fn with_delivery_retries(mut self, own: bool) -> Self {
self.delivery_retries = own;
self
}
pub const fn with_global_shortcuts(mut self, own: bool) -> Self {
self.global_shortcuts = own;
self
}
}
fn start_audio_for<T>(
audio: AudioIn,
host: &dyn Host,
start_mic: impl FnOnce() -> Result<T>,
) -> Option<T> {
match audio {
AudioIn::Microphone => {
let stage = Instant::now();
let recorder = start_mic().ok();
host.metric("audio_start_ms", stage.elapsed().as_secs_f64() * 1_000.0);
recorder
}
AudioIn::External | AudioIn::None => None,
}
}
enum InnerSource {
Screen {
session: ScreenCaptureSession,
audio: Option<crate::audio::AudioRecorder>,
windows: Option<crate::windows::WindowLogger>,
},
Phone(crate::phone::PhoneSession),
}
enum SealedSource {
Screen {
recording: ScreenRecording,
recorded_audio: Option<crate::audio::RecordedAudio>,
activity: crate::windows::ActivityLog,
},
Phone(Result<crate::phone::PhoneRecording>),
}
pub struct Engine {
host: Arc<dyn Host>,
config: Arc<dyn ConfigSource>,
transcriber: Option<Arc<dyn crate::transcribe::Transcriber>>,
clipboard: Option<Arc<dyn crate::handoff::Clipboard>>,
telemetry: Arc<SharedCaptureTelemetry>,
custom_telemetry: bool,
resources: SharedResources,
}
struct HostLogSink(Arc<dyn Host>);
impl crate::log::Sink for HostLogSink {
fn log(&self, line: &str) {
self.0.log(line);
}
fn metric(&self, name: &str, ms: f64) {
self.0.metric(name, ms);
}
}
impl Engine {
pub fn new(host: Arc<dyn Host>, config: Arc<dyn ConfigSource>) -> Self {
crate::log::set_sink_if_unset(Arc::new(HostLogSink(Arc::clone(&host))));
let (tx, rx) = flume::unbounded();
// Without an app's own telemetry, the one event a host must act on (the iPhone mirror
// ending on its own) reaches it through `Host::emit`; the rest are for Scrawl's UI.
let relay = Arc::clone(&host);
let _ = std::thread::Builder::new()
.name("scrawl-ui-events".into())
.spawn(move || {
for event in rx {
if matches!(event, Event::PhoneEnded) {
relay.emit(event);
}
}
});
Self {
host,
config,
transcriber: None,
clipboard: None,
telemetry: SharedCaptureTelemetry::new(tx, ShortcutConfig::ansi_defaults()),
custom_telemetry: false,
resources: SharedResources::none(),
}
}
pub fn default_engine() -> Self {
let (tx, _rx) = flume::unbounded();
Self::new(
Arc::new(ChannelHost::new(tx)),
Arc::new(DefaultConfigSource),
)
.with_clipboard(Arc::new(crate::handoff::SystemClipboard))
}
pub fn with_telemetry(mut self, telemetry: Arc<SharedCaptureTelemetry>) -> Self {
self.telemetry = telemetry;
self.custom_telemetry = true;
self
}
pub fn with_transcriber(
mut self,
transcriber: Arc<dyn crate::transcribe::Transcriber>,
) -> Self {
self.transcriber = Some(transcriber);
self
}
pub fn with_clipboard(mut self, clipboard: Arc<dyn crate::handoff::Clipboard>) -> Self {
self.clipboard = Some(clipboard);
self
}
pub fn with_resources(mut self, resources: SharedResources) -> Self {
self.resources = resources;
self
}
pub fn resources(&self) -> SharedResources {
self.resources
}
/// Starts whichever machine-wide shared resources (`ingest_server`, `delivery_retries`,
/// `global_shortcuts`) are enabled in [`Self::resources`]. Does nothing when all switches are
/// `false`.
pub fn install_shared_resources(&self) {
if self.resources.ingest_server || self.resources.delivery_retries {
let cfg = self.config.engine_config().ok();
let captures = cfg.as_ref().map(|c| c.resolved_captures_dir());
let token_file = cfg
.as_ref()
.and_then(|c| c.ingest_token_path().map(Path::to_path_buf));
crate::phone::start_ingest_server_for_resources(
self.resources,
self.telemetry.ui_sender.clone(),
self.clipboard.clone(),
captures,
token_file,
);
}
if self.resources.global_shortcuts {
let (rec, paste) = self.telemetry.shortcuts();
if rec.is_default_for(ShortcutKind::Record) && paste.is_default_for(ShortcutKind::Paste)
{
*self
.telemetry
.shortcut
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner) =
ShortcutConfig::default_record();
*self
.telemetry
.paste_shortcut
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner) =
ShortcutConfig::default_paste();
}
install_global_hotkeys(&self.telemetry);
}
}
pub fn telemetry(&self) -> &Arc<SharedCaptureTelemetry> {
&self.telemetry
}
pub fn discover_target(&self, window: Option<&str>, at: Option<[f64; 2]>) -> TargetInfo {
discover_target_at(window, at)
}
pub fn selectable_windows(&self) -> Vec<TargetInfo> {
list_selectable_windows()
}
pub fn phones(&self) -> Result<Vec<crate::helper::PhoneDevice>> {
crate::helper::devices()
}
/// Blocking: performs synchronous target discovery (when `req.target` is
/// [`Target::ActiveDisplay`]), spawns the recording helper, and waits for stream readiness
/// (up to ~1.2 s). Call this on a background worker thread rather than the UI main thread,
/// or call [`Self::discover_target`] first and pass [`Start::screen`] to a worker thread.
pub fn start(&self, req: Start) -> Result<Capture> {
self.start_with_preview(req, None)
}
/// Blocking: see [`Self::start`].
pub fn start_with_preview(
&self,
req: Start,
existing_preview: Option<crate::phone::PhoneSession>,
) -> Result<Capture> {
let engine_cfg = self.config.engine_config()?;
let agent_context = engine_cfg.experiment(crate::handoff::AGENT_CONTEXT);
let dir = match req.bundle_dir {
Some(dir) => {
std::fs::create_dir_all(&dir)
.with_context(|| format!("could not create {}", dir.display()))?;
#[cfg(unix)]
{
use std::os::unix::fs::PermissionsExt;
let _ = std::fs::set_permissions(&dir, std::fs::Permissions::from_mode(0o700));
}
dir
}
None => {
let root = match std::env::var_os("SCRAWL_BENCH_START") {
Some(dir) => PathBuf::from(dir),
None => engine_cfg.resolved_captures_dir(),
};
let id = crate::pipeline::make_bundle_id();
let dir = root.join(format!("{id}.scrawl"));
std::fs::create_dir_all(&dir)
.with_context(|| format!("could not create {}", dir.display()))?;
#[cfg(unix)]
{
use std::os::unix::fs::PermissionsExt;
let _ = std::fs::set_permissions(&dir, std::fs::Permissions::from_mode(0o700));
}
dir
}
};
let saved_excluded = self
.telemetry
.excluded_window_ids
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner)
.clone();
let saved_passthrough = self
.telemetry
.passthrough_window_ids
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner)
.clone();
let own = self.host.own_windows();
let telemetry = if self.custom_telemetry || self.resources.global_shortcuts {
Arc::clone(&self.telemetry)
} else {
let isolated = SharedCaptureTelemetry::new(
self.telemetry.ui_sender.clone(),
ShortcutConfig::ansi_defaults(),
);
*isolated
.passthrough_window_ids
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner) = saved_passthrough;
isolated
};
let (target, source, session_id) = match req.target {
Target::Phone { device, name } => {
let movie_path = dir.join("recording.mov");
let dev_name = name.as_deref().unwrap_or("iPhone");
let target = crate::phone::phone_target(dev_name, None, None, 0, 0);
let initial_frame = existing_preview
.as_ref()
.and_then(crate::phone::PhoneSession::rect)
.map_or(
TargetFrame {
x: -100_000.0,
y: -100_000.0,
width: 1.0,
height: 1.0,
scale: 2.0,
},
|[rx, ry, rw, rh]| TargetFrame {
x: f64::from(rx),
y: f64::from(ry),
width: f64::from(rw),
height: f64::from(rh),
scale: 2.0,
},
);
telemetry.set_capture_mode(CaptureMode::Device);
let session_id = telemetry.reset_for_session_at_with_agent_context(
initial_frame,
Some(req.t0_boot_ns),
agent_context,
);
Self::restore_own_windows(&telemetry, &saved_excluded, &own);
// Hold interaction telemetry inactive until the helper's `recording` event arrives.
telemetry.active.store(false, Ordering::Release);
// The movie's offset is measured from `t0`, the clock input and audio share.
let since_t0_ms = boot_time_ns().saturating_sub(req.t0_boot_ns) / 1_000_000;
let origin_epoch_ms = crate::pipeline::now_epoch_ms().saturating_sub(since_t0_ms);
let mut session = if let Some(mut preview) = existing_preview {
preview.begin_recording(&movie_path);
preview
} else {
match crate::phone::PhoneSession::start(
&movie_path,
device.as_deref(),
Some(Arc::clone(&telemetry)),
req.audio == AudioIn::External,
) {
Ok(session) => session,
Err(error) => {
let _ = telemetry.finish_session();
let _ = telemetry.take_flags();
let _ = std::fs::remove_dir_all(&dir);
return Err(error);
}
}
};
session.clock_from(origin_epoch_ms);
(target, InnerSource::Phone(session), session_id)
}
Target::ActiveDisplay | Target::Screen(_) => {
let target = match req.target {
Target::Screen(t) => t,
Target::ActiveDisplay => discover_target_at(None, crate::screens::pointer()),
Target::Phone { .. } => unreachable!(),
};
let video_path = dir.join("recording.mp4");
telemetry.set_capture_mode(target.mode);
let session_id = telemetry.reset_for_session_at_with_agent_context(
target.frame,
Some(req.t0_boot_ns),
agent_context,
);
Self::restore_own_windows(&telemetry, &saved_excluded, &own);
let audio = start_audio_for(
req.audio,
self.host.as_ref(),
crate::audio::AudioRecorder::start,
);
let stage = Instant::now();
let session = ScreenCaptureSession::start_with_budget(
target.clone(),
Some(video_path),
Arc::clone(&telemetry.excluded_window_ids),
Arc::clone(&own.recorded),
engine_cfg.candidate_budget_bytes(),
req.movie,
req.portal_token,
);
self.host
.metric("sck_start_ms", stage.elapsed().as_secs_f64() * 1_000.0);
let windows = (req.window_log && target.mode == CaptureMode::Display).then(|| {
let started_at = *telemetry
.started_at
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner);
crate::windows::WindowLogger::start(
target.frame,
started_at,
Arc::clone(&telemetry.window_log_request),
Arc::clone(&telemetry.excluded_window_ids),
)
});
(
target,
InnerSource::Screen {
session,
audio,
windows,
},
session_id,
)
}
};
self.host.emit(Event::Progress {
bundle: dir.clone(),
stage: Stage::Recording,
});
Ok(Capture {
host: Arc::clone(&self.host),
transcriber: self.transcriber.clone(),
engine_cfg,
input: InputSink::new(Arc::clone(&telemetry)),
telemetry,
session_id,
dir,
target,
t0_boot_ns: req.t0_boot_ns,
note: req.note,
host_narration: req.audio == AudioIn::External,
source,
stopped_input: None,
})
}
fn restore_own_windows(
telemetry: &SharedCaptureTelemetry,
saved_excluded: &[u32],
own: &OwnWindows,
) {
let mut excluded = telemetry
.excluded_window_ids
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner);
for &id in saved_excluded.iter().chain(&own.excluded) {
if !excluded.contains(&id) {
excluded.push(id);
}
}
drop(excluded);
let mut passthrough = telemetry
.passthrough_window_ids
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner);
for &id in &own.passthrough {
if !passthrough.contains(&id) {
passthrough.push(id);
}
}
}
}
pub struct Capture {
host: Arc<dyn Host>,
transcriber: Option<Arc<dyn crate::transcribe::Transcriber>>,
engine_cfg: crate::handoff::EngineConfig,
telemetry: Arc<SharedCaptureTelemetry>,
session_id: u64,
input: InputSink,
dir: PathBuf,
target: TargetInfo,
t0_boot_ns: u64,
note: Option<String>,
/// A phone capture's narration is `finish`'s audio, not the movie's.
host_narration: bool,
source: InnerSource,
stopped_input: Option<(Vec<InteractionEvent>, Vec<Stroke>, Vec<u64>)>,
}
/// A capture whose screen/phone recording, window logger, and input collection have already
/// stopped, ready for [`SealedCapture::finish`] on a background worker.
pub struct SealedCapture {
host: Arc<dyn Host>,
transcriber: Option<Arc<dyn crate::transcribe::Transcriber>>,
engine_cfg: crate::handoff::EngineConfig,
dir: PathBuf,
target: TargetInfo,
t0_boot_ns: u64,
note: Option<String>,
host_narration: bool,
route_hint: RouteContext,
events: Vec<InteractionEvent>,
strokes: Vec<Stroke>,
flags: Vec<u64>,
source: SealedSource,
}
impl Capture {
pub fn dir(&self) -> &Path {
&self.dir
}
pub fn target(&self) -> &TargetInfo {
&self.target
}
pub fn is_screen(&self) -> bool {
matches!(self.source, InnerSource::Screen { .. })
}
pub fn is_phone(&self) -> bool {
matches!(self.source, InnerSource::Phone(_))
}
pub fn has_phone_recording(&self) -> bool {
match &self.source {
InnerSource::Phone(session) => session.has_recording(),
InnerSource::Screen { .. } => false,
}
}
pub fn phone_has_ended(&self) -> bool {
match &self.source {
InnerSource::Phone(session) => session.has_ended(),
InnerSource::Screen { .. } => false,
}
}
/// Why the screen source gave up before the stop, if it did (Linux: the portal's picker
/// was cancelled): the capture has nothing more to record.
pub fn screen_failure(&self) -> Option<String> {
match &self.source {
InnerSource::Screen { session, .. } => session.failure(),
InnerSource::Phone(_) => None,
}
}
pub fn started_at_boot_ns(&self) -> u64 {
self.t0_boot_ns
}
pub fn input(&self) -> &InputSink {
&self.input
}
pub fn ink(&self) -> InkSnapshot {
self.telemetry.ink_snapshot()
}
pub fn elapsed_ms(&self) -> u64 {
match &self.source {
InnerSource::Screen { session, .. } => session.elapsed_ms(),
InnerSource::Phone(session) => session.elapsed_ms(),
}
}
pub fn request_frame(&self) {
if let InnerSource::Screen { session, .. } = &self.source {
session.request_immediate_frame();
}
}
pub fn route_context(&self) -> RouteContext {
if !self.is_screen() {
return RouteContext::default();
}
let front_app = (self.target.mode != CaptureMode::Window)
.then(frontmost_app_info)
.flatten();
let app_pid = match (&front_app, self.target.window_id) {
(Some((pid, ..)), _) => Some(*pid),
(None, Some(window)) if crate::routing::jev::enabled() => crate::windows::raw_windows()
.into_iter()
.find(|w| w.id == window)
.map(|w| w.pid),
_ => None,
};
RouteContext {
front_bundle_id: front_app.and_then(|(_, _, id)| id),
captured_bundle_id: self.target.bundle_id.clone(),
app_pid,
}
}
/// Stops collecting pointer/ink input on the calling thread before moving `self` to a
/// background worker to run `finish`.
pub fn stop_input(&mut self) {
if self.stopped_input.is_none() {
if self.telemetry.session_id.load(Ordering::Acquire) == self.session_id {
let (events, strokes) = self.telemetry.finish_session();
let flags = self.telemetry.take_flags();
self.stopped_input = Some((events, strokes, flags));
} else {
self.stopped_input = Some(Default::default());
}
}
}
pub fn stop_raw_screen(
mut self,
) -> Option<(ScreenRecording, Option<crate::audio::RecordedAudio>)> {
self.stop_input();
match self.source {
InnerSource::Screen {
session,
audio,
windows,
} => {
if let Some(windows) = windows {
let _ = windows.stop();
}
let (recording, recorded_audio) = std::thread::scope(|scope| {
let screen = scope.spawn(|| session.stop());
let audio = audio.and_then(|r| r.stop().ok());
(screen.join().unwrap(), audio)
});
Some((recording, recorded_audio))
}
InnerSource::Phone(_) => None,
}
}
pub fn cancel(mut self) {
self.stop_input();
match self.source {
InnerSource::Screen {
session,
audio,
windows,
} => {
let recording = session.stop();
if let Some(audio) = audio {
let _ = audio.stop();
}
if let Some(windows) = windows {
let _ = windows.stop();
}
recording.discard();
}
InnerSource::Phone(session) => session.cancel(),
}
if let Err(error) = std::fs::remove_dir_all(&self.dir) {
self.host
.log(&format!("could not remove {}: {error}", self.dir.display()));
}
self.host.emit(Event::Progress {
bundle: self.dir,
stage: Stage::Cancelled,
});
}
/// Ends screen/phone recording, window logging, and input collection immediately and returns a
/// [`SealedCapture`] whose [`SealedCapture::finish`] can run later on a worker thread.
pub fn stop(mut self) -> SealedCapture {
self.stop_input();
let (events, strokes, flags) = self.stopped_input.take().unwrap_or_default();
let route_hint = self.route_context();
let bundle_dir = self.dir.clone();
self.host.emit(Event::Progress {
bundle: bundle_dir,
stage: Stage::Stopping,
});
let source = match self.source {
InnerSource::Phone(session) => SealedSource::Phone(session.stop()),
InnerSource::Screen {
session,
audio,
windows,
} => {
let stop_started = Instant::now();
let (recording, recorded_audio, mut activity) = std::thread::scope(|s| {
let screen_handle = s.spawn(|| session.stop());
let win_handle = s.spawn(|| {
windows
.map(crate::windows::WindowLogger::stop)
.unwrap_or_default()
});
let audio = audio.and_then(|r| r.stop().ok());
(
screen_handle.join().unwrap(),
audio,
win_handle.join().unwrap_or_default(),
)
});
activity.stop_started = Some(stop_started);
activity.stop_ms = Some(stop_started.elapsed().as_millis() as u64);
SealedSource::Screen {
recording,
recorded_audio,
activity,
}
}
};
SealedCapture {
host: self.host,
transcriber: self.transcriber,
engine_cfg: self.engine_cfg,
dir: self.dir,
target: self.target,
t0_boot_ns: self.t0_boot_ns,
note: self.note,
host_narration: self.host_narration,
route_hint,
events,
strokes,
flags,
source,
}
}
/// Alias for [`Self::stop`].
pub fn seal(self) -> SealedCapture {
self.stop()
}
/// Blocking: stops streams, transcribes audio, renders keyframes and writes the bundle.
pub fn finish(self, external_audio: Option<crate::audio::RecordedAudio>) -> Result<Finished> {
self.stop().finish(external_audio)
}
}
impl SealedCapture {
pub fn dir(&self) -> &Path {
&self.dir
}
pub fn target(&self) -> &TargetInfo {
&self.target
}
pub fn started_at_boot_ns(&self) -> u64 {
self.t0_boot_ns
}
pub fn route_context(&self) -> &RouteContext {
&self.route_hint
}
pub fn cancel(self) {
if let SealedSource::Screen { recording, .. } = self.source {
recording.discard();
}
if let Err(error) = std::fs::remove_dir_all(&self.dir) {
self.host
.log(&format!("could not remove {}: {error}", self.dir.display()));
}
self.host.emit(Event::Progress {
bundle: self.dir,
stage: Stage::Cancelled,
});
}
/// Blocking: transcribes audio, renders keyframes and writes the bundle for a stopped capture.
pub fn finish(self, external_audio: Option<crate::audio::RecordedAudio>) -> Result<Finished> {
let bundle_dir = self.dir.clone();
let route_hint = self.route_hint;
let flags = self.flags;
let events = self.events;
let strokes = self.strokes;
let result = match self.source {
SealedSource::Phone(stopped) => {
let finalized = stopped.and_then(|mut recording| {
for flag in flags {
if flag >= recording.offset_ms {
recording.flags.push(flag - recording.offset_ms);
}
}
let narration = if self.host_narration {
self.host.emit(Event::Progress {
bundle: bundle_dir.clone(),
stage: Stage::Transcribing,
});
Some(crate::phone::HostNarration {
audio: external_audio
.map(|audio| audio.since(recording.offset_ms))
.transpose()?,
transcriber: self.transcriber.as_deref(),
})
} else {
None
};
crate::phone::finalize_live_capture_for_config(
&bundle_dir,
recording,
events,
strokes,
self.note,
Some(&self.engine_cfg),
narration,
)
});
if let Err(error) = &finalized {
self.host.log(&format!("iPhone capture failed: {error:#}"));
}
finalized.map(|(dir, bundle, summary)| {
let words = crate::transcribe::word_count(&bundle.transcript);
let duration_ms = bundle.duration_ms;
let missing_transcript = bundle
.transcript
.missing
.as_ref()
.map(|m| m.reason.clone())
.or_else(|| {
(bundle.transcript.text.trim().is_empty() && bundle.audio.is_none())
.then(|| "no_audio".to_owned())
});
Finished {
dir,
bundle,
summary,
words,
duration_ms,
route_hint,
gaps: 0,
missing_transcript,
}
})
}
SealedSource::Screen {
recording,
recorded_audio,
activity,
} => {
let recorded_audio = external_audio.or(recorded_audio);
let gaps = recorded_audio.as_ref().map_or(0, |a| a.gaps.len());
let had_audio_samples = recorded_audio
.as_ref()
.is_some_and(|a| a.sample_count() > 0 || !a.wav_bytes.is_empty());
let duration_ms = recording.duration_ms;
self.host.emit(Event::Progress {
bundle: bundle_dir.clone(),
stage: Stage::Transcribing,
});
crate::pipeline::finalize_capture_bundle_for_config(
&bundle_dir,
self.target,
recording,
strokes,
events,
flags,
recorded_audio,
self.note,
activity,
self.transcriber.as_deref(),
Some(&self.engine_cfg),
)
.map(|(dir, bundle, summary)| {
let words = crate::transcribe::word_count(&bundle.transcript);
let missing_transcript = bundle
.transcript
.missing
.as_ref()
.map(|m| m.reason.clone())
.or_else(|| {
(bundle.transcript.text.trim().is_empty()
&& bundle.audio.is_none()
&& !had_audio_samples)
.then(|| "no_audio".to_owned())
});
Finished {
dir,
bundle,
summary,
words,
duration_ms,
route_hint,
gaps,
missing_transcript,
}
})
}
};
self.host.emit(Event::Progress {
bundle: bundle_dir,
stage: if result.is_ok() {
Stage::Done
} else {
Stage::Failed
},
});
result
}
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn screen_stop_joins_worker_before_collecting_its_final_media() {
let stopping = Arc::new(AtomicBool::new(false));
let frames = Arc::new(Mutex::new(CandidateBuffer::new(1 << 20)));
let worker_stopping = Arc::clone(&stopping);
let worker_frames = Arc::clone(&frames);
let path = std::env::temp_dir().join(crate::pipeline::make_bundle_id());
let worker_path = path.clone();
let worker = thread::spawn(move || {
let deadline = Instant::now() + Duration::from_secs(2);
while !worker_stopping.load(Ordering::Acquire) && Instant::now() < deadline {
thread::yield_now();
}
assert!(worker_stopping.load(Ordering::Acquire));
worker_frames
.lock()
.unwrap()
.push(RgbaFrame::new(42, 1, 1, vec![1, 2, 3, 255]), false);
std::fs::write(&worker_path, b"finalized media").unwrap();
Some(Movie::written(worker_path, 120))
});
let session = ScreenCaptureSession {
failure: Arc::default(),
started_at: Instant::now(),
stopping,
urgent_snapshot: Arc::new(AtomicBool::new(false)),
frames,
sampler_thread: Some(worker),
_recording: crate::retention::Recording::start(),
};
let recording = session.stop();
let frames = recording.frames;
assert_eq!(frames.len(), 1);
assert_eq!(frames[0].t_ms, 42);
assert_eq!(frames[0].rgba, vec![1, 2, 3, 255]);
assert_eq!(
recording.video.and_then(Movie::wait),
Some((path.clone(), 120))
);
std::fs::remove_file(path).unwrap();
}
/// A capture stopped with no picture names the screen source's failure when it gave one, and
/// otherwise the platform's reason, never that failure string.
#[test]
fn a_capture_with_no_frames_says_why_the_screen_gave_nothing() {
let stop_with = |failure: Option<&str>| {
ScreenCaptureSession {
failure: Arc::new(Mutex::new(failure.map(str::to_owned))),
started_at: Instant::now(),
stopping: Arc::new(AtomicBool::new(false)),
urgent_snapshot: Arc::new(AtomicBool::new(false)),
frames: Arc::new(Mutex::new(CandidateBuffer::new(1 << 20))),
sampler_thread: None,
_recording: crate::retention::Recording::start(),
}
.stop()
.no_screen
};
let failure = "the screen sharing stream sent no frames";
assert_eq!(stop_with(Some(failure)).as_deref(), Some(failure));
let reason = stop_with(None).expect("a stopped capture with no frame still says why");
assert_ne!(reason, failure);
#[cfg(target_os = "linux")]
assert_eq!(reason, "the screen recorded no frames");
}
/// Clicks on `scrawl-helper`'s windows (the SwiftUI capsule, Settings, onboarding) are
/// Scrawl's own, except on the iPhone mirror, which is what's being recorded. The host's
/// chrome takes clicks unless it lets them through: Hex's capsule is a 640×424 panel, named
/// and click-through (only its pill, which Hex checks itself, takes clicks).
#[test]
fn helper_windows_are_scrawl_s_except_the_capture_source() {
let (sender, _receiver) = flume::unbounded();
let telemetry = SharedCaptureTelemetry::new(sender, ShortcutConfig::ansi_defaults());
let window = |id, pid, bounds| crate::windows::RawWindow {
id,
pid,
owner: String::new(),
title: String::new(),
bounds,
layer: 0,
alpha: 1.0,
};
let own = std::process::id() as i32;
let (settings, mirror, app) = (own + 1, own + 2, own + 3);
let windows = [
crate::windows::RawWindow {
layer: 26,
..window(1, own, [0.0, 0.0, 1728.0, 1117.0]) // the ink overlay, click-through
},
crate::windows::RawWindow {
layer: 27,
..window(5, own, [544.0, 0.0, 640.0, 424.0]) // the host's capsule panel
},
crate::windows::RawWindow {
layer: 25,
..window(7, own, [700.0, 1000.0, 320.0, 44.0]) // a HUD of the host's
},
window(6, own, [1300.0, 900.0, 400.0, 200.0]), // the host's own Settings window
window(2, settings, [100.0, 100.0, 600.0, 400.0]),
window(3, mirror, [1300.0, 100.0, 300.0, 650.0]),
window(4, app, [0.0, 0.0, 1728.0, 1117.0]),
];
*telemetry.excluded_window_ids.lock().unwrap() = vec![5, 1];
*telemetry.passthrough_window_ids.lock().unwrap() = vec![1, 5];
telemetry.set_capture_source_pid(mirror as u32);
let helper = |pid| pid == settings || pid == mirror;
let at = |x, y| telemetry.scrawl_window_in(&windows, x, y, helper);
assert!(at(300.0, 300.0), "Settings");
assert!(!at(1400.0, 400.0), "the iPhone mirror");
assert!(!at(1000.0, 900.0), "the app under the ink overlay");
assert!(!at(1100.0, 300.0), "the app under the capsule's panel");
assert!(at(800.0, 1020.0), "the host's HUD, unnamed");
telemetry
.passthrough_window_ids
.lock()
.unwrap()
.retain(|id| *id != 5);
assert!(at(1100.0, 300.0), "a named window that takes clicks");
assert!(
!at(1500.0, 1000.0),
"the host's Settings, which can be what's shown"
);
telemetry.set_capture_source_pid(0);
assert!(at(1400.0, 400.0), "a helper that isn't the source");
}
/// A screen capture on a laptop: the old listen-only event tap was off (no Input
/// Monitoring), so his clicks on OpenCode's composer never reached the bundle. Clicks now
/// come from the mouse monitor; polled button state stands in if it couldn't be installed.
#[test]
fn clicks_are_recorded_from_polled_buttons_while_the_mouse_monitor_is_missing() {
let (sender, _receiver) = flume::unbounded();
let telemetry = SharedCaptureTelemetry::new(sender, ShortcutConfig::ansi_defaults());
telemetry.reset_for_session(TargetFrame {
x: 0.0,
y: 0.0,
width: 1728.0,
height: 1117.0,
scale: 2.0,
});
assert!(!telemetry.mouse_live.load(Ordering::Acquire));
let mut prev = [false; 2];
let not_scrawl = |_: f64, _: f64| false;
let composer = (778.0, 1050.0);
// A ⇧-click is a pin, not a click.
poll_clicks(
&telemetry,
&mut prev,
[true, false],
composer,
true,
not_scrawl,
);
poll_clicks(
&telemetry,
&mut prev,
[false, false],
composer,
true,
not_scrawl,
);
// A click on the composer.
poll_clicks(
&telemetry,
&mut prev,
[true, false],
composer,
false,
not_scrawl,
);
poll_clicks(
&telemetry,
&mut prev,
[true, false],
composer,
false,
not_scrawl,
);
poll_clicks(
&telemetry,
&mut prev,
[false, false],
composer,
false,
not_scrawl,
);
// Then the stop button, which is Scrawl's own.
let stop = (860.0, 20.0);
poll_clicks(&telemetry, &mut prev, [true, false], stop, false, |_, _| {
true
});
poll_clicks(
&telemetry,
&mut prev,
[false, false],
stop,
false,
|_, _| true,
);
// While the mouse monitor is installed it reports clicks itself.
telemetry.mouse_live.store(true, Ordering::Release);
poll_clicks(
&telemetry,
&mut prev,
[true, false],
composer,
false,
not_scrawl,
);
poll_clicks(
&telemetry,
&mut prev,
[false, false],
composer,
false,
not_scrawl,
);
let (events, _) = telemetry.finish_session();
assert_eq!(events.len(), 1, "{events:?}");
let InteractionEvent::Click { x, y, button, .. } = &events[0] else {
panic!("{events:?}");
};
assert!(
(x - 0.45).abs() < 0.01 && (y - 0.94).abs() < 0.01,
"{x}, {y}"
);
assert_eq!(button.as_deref(), Some("left"));
}
#[test]
fn the_capture_shortcut_matches_its_aliases_and_a_custom_one() {
let s = 1_u16; // the S key on this layout
let default = ShortcutConfig {
key_code: u32::from(s),
carbon_modifiers: CARBON_OPTION_KEY | CARBON_SHIFT_KEY,
label: "⌥⇧S".into(),
};
// (cmd, ctrl, alt, shift)
let hit = |cfg: &ShortcutConfig, key, (cmd, ctrl, alt, shift)| {
is_capture_shortcut(cfg, s, key, cmd, ctrl, alt, shift)
};
assert!(hit(&default, s, (false, false, true, true)), "⌥⇧S");
assert!(hit(&default, s, (true, false, true, true)), "⌥⇧⌘S");
assert!(hit(&default, s, (false, true, true, true)), "⌃⌥⇧S");
assert!(hit(&default, s, (true, true, true, false)), "⌃⌥⌘S");
assert!(hit(&default, s, (true, true, true, true)), "hyper S");
assert!(hit(&default, KEY_S, (false, false, true, true)), "ANSI S");
assert!(!hit(&default, s, (false, false, false, true)), "⇧S");
assert!(!hit(&default, s, (true, false, false, false)), "⌘S");
assert!(!hit(&default, 2, (false, false, true, true)), "⌥⇧D");
let custom = ShortcutConfig {
key_code: 15,
carbon_modifiers: CARBON_CONTROL_KEY | CARBON_OPTION_KEY,
label: "⌃⌥R".into(),
};
assert!(hit(&custom, 15, (false, true, true, false)));
assert!(!hit(&custom, 15, (true, true, true, false)));
assert!(!hit(&custom, 15, (false, false, true, false)));
// Once changed, ⌥⇧S and its aliases are free for other apps.
assert!(!hit(&custom, s, (false, false, true, true)), "⌥⇧S");
assert!(!hit(&custom, KEY_S, (true, true, true, true)), "hyper S");
assert_eq!(
hotkey_combos(ShortcutKind::Record, &custom),
[(15, custom.carbon_modifiers)]
);
assert_eq!(
hotkey_combos(ShortcutKind::Record, &default).len(),
if u32::from(KEY_S) == default.key_code {
4
} else {
8
},
"⌥⇧S and three aliases, on the layout's key and the ANSI one"
);
// The paste shortcut works the same way.
let v = 9_u16;
let paste = ShortcutConfig {
key_code: u32::from(v),
carbon_modifiers: CARBON_OPTION_KEY | CARBON_SHIFT_KEY,
label: "⌥⇧V".into(),
};
let paste_hit = |cfg: &ShortcutConfig, key, (cmd, ctrl, alt, shift)| {
shortcut_matches(ShortcutKind::Paste, cfg, v, key, cmd, ctrl, alt, shift)
};
assert!(paste_hit(&paste, v, (false, false, true, true)), "⌥⇧V");
assert!(paste_hit(&paste, v, (true, true, true, false)), "⌃⌥⌘V");
assert!(
!paste_hit(&paste, s, (false, false, true, true)),
"⌥⇧S isn't paste"
);
let paste_custom = ShortcutConfig {
key_code: 35,
carbon_modifiers: CARBON_CONTROL_KEY | CARBON_OPTION_KEY,
label: "⌃⌥P".into(),
};
assert!(paste_hit(&paste_custom, 35, (false, true, true, false)));
assert!(!paste_hit(&paste_custom, v, (false, false, true, true)));
assert!(
paste.is_default_for(ShortcutKind::Paste)
&& !paste.is_default_for(ShortcutKind::Record)
);
#[cfg(target_os = "macos")]
assert_eq!(
modifier_symbols(CARBON_CMD_KEY | CARBON_CONTROL_KEY | CARBON_SHIFT_KEY),
"⌃⇧⌘"
);
#[cfg(target_os = "windows")]
assert_eq!(
modifier_symbols(CARBON_CMD_KEY | CARBON_CONTROL_KEY | CARBON_SHIFT_KEY),
"Ctrl+Win+Shift+"
);
assert_eq!(shortcut_parts("⌥⇧S"), ("⌥⇧", "S"));
assert_eq!(shortcut_parts("⌃⌥⌘R"), ("⌃⌥⌘", "R"));
assert_eq!(shortcut_parts("⌥⇧É"), ("⌥⇧", "É"));
assert_eq!(shortcut_parts("Alt+Shift+S"), ("Alt+Shift", "S"));
assert_eq!(shortcut_parts("Ctrl+Alt+Ä"), ("Ctrl+Alt", "Ä"));
assert_eq!(shortcut_parts("Ctrl++"), ("Ctrl", "+"));
assert_eq!(shortcut_parts(""), ("", ""));
}
#[test]
fn recordings_use_the_display_under_the_cursor() {
let displays = [[0.0, 0.0, 1728.0, 1117.0], [1728.0, -200.0, 2560.0, 1440.0]];
assert_eq!(pick_display(&displays, Some([2000.0, 300.0]), None), 1);
assert_eq!(pick_display(&displays, Some([100.0, 100.0]), None), 0);
// No cursor: the display holding the focused window's centre.
let focused = Some([1900.0, 0.0, 800.0, 600.0]);
assert_eq!(pick_display(&displays, None, focused), 1);
assert_eq!(pick_display(&displays, Some([-50.0, 0.0]), focused), 1);
assert_eq!(pick_display(&displays, None, None), 0);
// A display left of the laptop, at negative x.
let left = [[0.0, 0.0, 1728.0, 1117.0], [-1920.0, 37.0, 1920.0, 1080.0]];
assert_eq!(pick_display(&left, Some([-5.0, 600.0]), None), 1);
// One above it, at negative y.
let above = [[0.0, 0.0, 1728.0, 1117.0], [0.0, -1440.0, 2560.0, 1440.0]];
assert_eq!(pick_display(&above, Some([10.0, -1.0]), None), 1);
assert_eq!(pick_display(&above, Some([10.0, 0.0]), None), 0);
}
/// Clicks and ink on a display left of, right of or above the laptop normalize into that
/// display, and land on the recording's pixels at its own scale.
#[test]
fn input_on_an_external_display_normalizes_into_it() {
let (sender, _receiver) = flume::unbounded();
let telemetry = SharedCaptureTelemetry::new(sender, ShortcutConfig::ansi_defaults());
let left = [-1920.0, 37.0, 1920.0, 1080.0];
let right = [1728.0, 0.0, 1920.0, 1080.0];
let above = [0.0, -1440.0, 2560.0, 1440.0];
let small_left = [-1280.0, -100.0, 1280.0, 800.0];
let below = [0.0, 1117.0, 1920.0, 1080.0];
// (display, scale, global point, expected normalized point, recording size)
let cases = [
// 1x displays are recorded at their own pixels
(left, 1.0, [-960.0, 577.0], (0.5, 0.5), (1920, 1080)),
(right, 1.0, [2208.0, 270.0], (0.25, 0.25), (1920, 1080)),
(below, 1.0, [480.0, 1387.0], (0.25, 0.25), (1920, 1080)),
// 2x ones at the 2560 px long edge
(above, 2.0, [640.0, -360.0], (0.25, 0.75), (2560, 1440)),
(small_left, 2.0, [-320.0, 500.0], (0.75, 0.75), (2560, 1600)),
];
for (rect, scale, point, expected, size) in cases {
let frame = crate::screens::with_scale(rect, scale);
let target = TargetInfo {
mode: CaptureMode::Display,
app: None,
bundle_id: None,
window_title: None,
window_id: None,
display_id: 2,
frame,
};
// Windows frames are pixels already and never upscaled.
if cfg!(target_os = "macos") {
assert_eq!(capture_size(&target, 2560.0), size, "{rect:?}");
}
telemetry.reset_for_session(frame);
assert!(telemetry.contains_screen_point(point[0], point[1]));
// The laptop's centre is outside this target.
assert!(!telemetry.contains_screen_point(864.0, 558.0));
let (nx, ny, fw, fh) = telemetry.normalize_point(point[0], point[1]);
assert_eq!((nx, ny), expected, "{rect:?}");
assert_eq!((fw, fh), (rect[2] as u32, rect[3] as u32));
// A click there is recorded at those coordinates.
telemetry.pointer_down(10, point[0], point[1]);
telemetry.pointer_up(20, point[0], point[1], false);
let (events, _) = telemetry.finish_session();
assert!(
matches!(events[..], [InteractionEvent::Click { x, y, .. }] if (x, y) == (nx, ny)),
"{events:?}"
);
}
// In recording pixels: the left 1x display's centre is (960, 540).
let frame = crate::screens::with_scale([-1920.0, 37.0, 1920.0, 1080.0], 1.0);
telemetry.reset_for_session(frame);
let (nx, ny, _, _) = telemetry.normalize_point(-960.0, 577.0);
assert_eq!((nx * 1920.0, ny * 1080.0), (960.0, 540.0));
// Above 2x: (640, -360) is a quarter in → (640, 1080) px of 2560×1440.
let frame = crate::screens::with_scale([0.0, -1440.0, 2560.0, 1440.0], 2.0);
telemetry.reset_for_session(frame);
let (nx, ny, _, _) = telemetry.normalize_point(640.0, -360.0);
assert_eq!((nx * 2560.0, ny * 1440.0), (640.0, 1080.0));
telemetry.finish_session();
}
fn session(width: f64, height: f64) -> Arc<SharedCaptureTelemetry> {
let (sender, _receiver) = flume::unbounded();
let telemetry = SharedCaptureTelemetry::new(sender, ShortcutConfig::ansi_defaults());
telemetry.reset_for_session(TargetFrame {
x: 0.0,
y: 0.0,
width,
height,
scale: 2.0,
});
telemetry
}
/// The poller draws ink while ⇧ is held. A ⇧-click is part of the drawing, never a click:
/// the stroke carries on whole, and a ⇧-click with nothing drawn (only its own jitter)
/// leaves no stroke once ⇧ comes up.
#[test]
fn a_shift_click_is_part_of_the_drawing() {
let telemetry = session(1000.0, 1000.0);
let ink = |pts: &[(i32, i32, i32)]| Stroke {
t0: 0,
t1: 0,
fw: 1000,
fh: 1000,
on: None,
pin: None,
laser: false,
pts: pts.iter().flat_map(|&(x, y, t)| [x, y, t]).collect(),
};
let press = telemetry.elapsed_ms() + 50;
let p = press as i32;
let s = SCALE;
// A line drawn while ⇧ was held, then a click at its end.
let line = ink(&[
(s / 10, s / 2, 0),
(s / 5, s / 2, p - 20),
(s / 5 + s / 500, s / 2, p + 5),
]);
*telemetry.live_ink.lock().unwrap() = Some(line.clone());
telemetry.shift_down(press, 200.0, 500.0);
assert!(telemetry.shift_up());
assert_eq!(telemetry.live_ink.lock().unwrap().as_ref(), Some(&line));
// Letting go of ⇧ commits it as one stroke.
let at = (200.0, 500.0);
track_shift_ink(&telemetry, at, false, true);
// Drawn long before the press still counts: the hold is a drawing.
*telemetry.live_ink.lock().unwrap() = Some(ink(&[(s / 10, s / 10, 0), (s / 5, s / 10, 1)]));
let t = telemetry.elapsed_ms() + 5_000;
telemetry.shift_down(t, 200.0, 100.0);
assert!(telemetry.shift_up());
*telemetry.live_ink.lock().unwrap() = None;
// A ⇧-click with only its own few points of jitter leaves nothing.
let jitter = ink(&[(s / 2, s / 2, 0), (s / 2 + 2, s / 2, 30)]);
*telemetry.live_ink.lock().unwrap() = Some(jitter);
let t = telemetry.elapsed_ms();
telemetry.shift_down(t, 500.0, 500.0);
assert!(telemetry.shift_up());
assert!(!telemetry.shift_up(), "one press, one release");
track_shift_ink(&telemetry, (500.0, 500.0), false, true);
let (events, strokes) = telemetry.finish_session();
assert!(events.is_empty(), "{events:?}");
let kinds: Vec<_> = strokes.iter().map(|s| (s.pin, s.pts.len() / 3)).collect();
assert_eq!(kinds, vec![(None, 3)]);
}
#[test]
fn shift_clicks_from_polled_buttons_are_neither_clicks_nor_pins() {
let telemetry = session(1000.0, 1000.0);
let mut prev = [false; 2];
// ⇧-clicks at a point, the second on Scrawl's own window.
for (at, on_scrawl) in [((250.0, 750.0), false), ((400.0, 400.0), true)] {
for buttons in [[true, false], [false, false]] {
poll_clicks(&telemetry, &mut prev, buttons, at, true, |_, _| on_scrawl);
}
}
let (events, strokes) = telemetry.finish_session();
assert!(events.is_empty(), "{events:?}");
assert!(strokes.is_empty(), "{strokes:?}");
}
#[test]
fn input_sink_routes_clicks_ink_and_chord_release() {
let telemetry = session(1000.0, 1000.0);
let sink = InputSink::new(Arc::clone(&telemetry));
// Residual Shift from a multi-modifier chord (⌥⇧S) is ignored until Shift is released.
let mut await_release = false;
assert!(!pure_shift_state(true, true, &mut await_release));
assert!(!pure_shift_state(true, false, &mut await_release));
assert!(!pure_shift_state(false, false, &mut await_release));
assert!(pure_shift_state(true, false, &mut await_release));
// Shift-ink stroke.
sink.ink(100, 100.0, 200.0, true);
sink.ink(120, 140.0, 200.0, true);
sink.ink(140, 180.0, 200.0, true);
assert!(sink.snapshot().live.is_some());
sink.ink(160, 180.0, 200.0, false);
assert_eq!(sink.snapshot().strokes.len(), 1);
// A Shift-click (pointer_down/up while shift_held) is neither a click nor a pin.
sink.ink(200, 500.0, 500.0, true);
sink.pointer_down(210, 500.0, 500.0);
sink.pointer_up(240, 500.0, 500.0, false);
sink.ink(250, 500.0, 500.0, false);
assert_eq!(sink.snapshot().strokes.len(), 1);
// Key chord and flag.
sink.key_chord(300, "⌘K");
sink.flag(350);
let flags = telemetry.take_flags();
let (events, strokes) = telemetry.finish_session();
assert_eq!(flags, vec![350]);
assert_eq!(strokes.len(), 1);
assert_eq!(strokes[0].pin, None);
assert!(matches!(
&events[..],
[InteractionEvent::Key {
t: 300,
shortcut
}] if shortcut == "⌘K"
));
}
#[test]
fn capture_finish_writes_bundle_and_emits_progress_with_host_and_transcriber() {
struct TestHost {
stages: Mutex<Vec<Stage>>,
}
impl Host for TestHost {
fn emit(&self, event: Event) {
if let Event::Progress { stage, .. } = event {
self.stages.lock().unwrap().push(stage);
}
}
}
struct TestTranscriber;
impl crate::transcribe::Transcriber for TestTranscriber {
fn transcribe(
&self,
_audio: &crate::audio::RecordedAudio,
) -> Result<Option<crate::model::TranscriptData>> {
Ok(Some(crate::model::TranscriptData {
source: "test".into(),
model: Some("parakeet".into()),
text: "hello world".into(),
words: vec![
crate::model::WordTiming {
t0: 0,
t1: 300,
text: "hello".into(),
},
crate::model::WordTiming {
t0: 300,
t1: 600,
text: "world".into(),
},
],
segments: Vec::new(),
estimated: false,
missing: None,
}))
}
}
let dir = std::env::temp_dir().join(format!(
"scrawl-capture-test-{}.scrawl",
crate::pipeline::make_bundle_id()
));
std::fs::create_dir_all(&dir).unwrap();
let host = Arc::new(TestHost {
stages: Mutex::new(Vec::new()),
});
let telemetry = session(200.0, 100.0);
let mut buf = CandidateBuffer::new(1 << 20);
buf.push(RgbaFrame::new(0, 20, 10, vec![40; 20 * 10 * 4]), false);
buf.push(RgbaFrame::new(500, 20, 10, vec![80; 20 * 10 * 4]), true);
let screen_session = ScreenCaptureSession {
failure: Arc::default(),
started_at: Instant::now(),
stopping: Arc::new(AtomicBool::new(false)),
urgent_snapshot: Arc::new(AtomicBool::new(false)),
frames: Arc::new(Mutex::new(buf)),
sampler_thread: None,
_recording: crate::retention::Recording::start(),
};
let target = TargetInfo {
mode: CaptureMode::Window,
app: Some("TestApp".into()),
bundle_id: Some("dev.test.app".into()),
window_title: Some("Main".into()),
window_id: Some(1),
display_id: 1,
frame: TargetFrame {
x: 0.0,
y: 0.0,
width: 200.0,
height: 100.0,
scale: 1.0,
},
};
let session_id = telemetry.session_id.load(Ordering::Acquire);
let capture = Capture {
host: host.clone(),
transcriber: Some(Arc::new(TestTranscriber)),
engine_cfg: crate::handoff::EngineConfig::default(),
input: InputSink::new(Arc::clone(&telemetry)),
telemetry,
session_id,
dir: dir.clone(),
target,
t0_boot_ns: 123_456_789,
note: Some("unit test".into()),
host_narration: false,
source: InnerSource::Screen {
session: screen_session,
audio: None,
windows: None,
},
stopped_input: None,
};
assert_eq!(capture.started_at_boot_ns(), 123_456_789);
capture.input().pointer_down(100, 50.0, 50.0);
capture.input().pointer_up(140, 50.0, 50.0, false);
let audio = crate::audio::RecordedAudio::new(vec![0.0; 16_000], 16_000, 200, Vec::new());
let finished = capture.finish(Some(audio)).unwrap();
assert_eq!(finished.dir, dir);
assert_eq!(finished.words, 2);
assert_eq!(
finished.route_hint.captured_bundle_id.as_deref(),
Some("dev.test.app")
);
assert!(dir.join("scrawl.json").is_file());
assert!(dir.join("summary.md").is_file());
assert!(dir.join("audio.wav").is_file());
assert_eq!(
*host.stages.lock().unwrap(),
vec![Stage::Stopping, Stage::Transcribing, Stage::Done]
);
let _ = std::fs::remove_dir_all(&dir);
}
#[test]
fn seal_stops_recording_immediately_and_captures_do_not_clobber_newer_telemetry() {
let dir1 = std::env::temp_dir().join(format!(
"scrawl-seal-1-{}.scrawl",
crate::pipeline::make_bundle_id()
));
let dir2 = std::env::temp_dir().join(format!(
"scrawl-seal-2-{}.scrawl",
crate::pipeline::make_bundle_id()
));
std::fs::create_dir_all(&dir1).unwrap();
std::fs::create_dir_all(&dir2).unwrap();
// Building an Engine on a background worker thread must not touch TIS/TSM.
let engine = std::thread::spawn(Engine::default_engine).join().unwrap();
assert_eq!(
engine.telemetry().shortcuts(),
ShortcutConfig::ansi_defaults()
);
let shared = session(200.0, 100.0);
let s1_id = shared.session_id.load(Ordering::Acquire);
let stopping1 = Arc::new(AtomicBool::new(false));
let mut buf1 = CandidateBuffer::new(1 << 20);
buf1.push(RgbaFrame::new(0, 20, 10, vec![40; 20 * 10 * 4]), false);
let target = TargetInfo {
mode: CaptureMode::Window,
app: Some("TestApp".into()),
bundle_id: Some("dev.test.app".into()),
window_title: Some("Main".into()),
window_id: Some(1),
display_id: 1,
frame: TargetFrame {
x: 0.0,
y: 0.0,
width: 200.0,
height: 100.0,
scale: 1.0,
},
};
let (tx, _rx) = flume::unbounded();
let host: Arc<dyn Host> = Arc::new(ChannelHost::new(tx));
let mut cfg1 = crate::handoff::EngineConfig::default();
cfg1.experiments
.insert(crate::handoff::AGENT_CONTEXT.into(), true);
let c1 = Capture {
host: Arc::clone(&host),
transcriber: None,
engine_cfg: cfg1,
input: InputSink::new(Arc::clone(&shared)),
telemetry: Arc::clone(&shared),
session_id: s1_id,
dir: dir1.clone(),
target: target.clone(),
t0_boot_ns: 100,
note: None,
host_narration: false,
source: InnerSource::Screen {
session: ScreenCaptureSession {
failure: Arc::default(),
started_at: Instant::now(),
stopping: Arc::clone(&stopping1),
urgent_snapshot: Arc::new(AtomicBool::new(false)),
frames: Arc::new(Mutex::new(buf1)),
sampler_thread: None,
_recording: crate::retention::Recording::start(),
},
audio: None,
windows: None,
},
stopped_input: None,
};
c1.input().pointer_down(50, 10.0, 10.0);
c1.input().pointer_up(80, 10.0, 10.0, false);
// Seal c1: stops its ScreenCaptureSession and input immediately.
let sealed1 = c1.seal();
assert!(stopping1.load(Ordering::Acquire));
// Start a second session on the same telemetry before sealed1 finishes (or if an unsealed
// capture is cancelled after session 2 starts).
let s2_id = shared.reset_for_session_at_with_agent_context(target.frame, Some(200), false);
assert!(s2_id > s1_id);
shared.flag_at(333);
let stale = Capture {
host: Arc::clone(&host),
transcriber: None,
engine_cfg: crate::handoff::EngineConfig::default(),
input: InputSink::new(Arc::clone(&shared)),
telemetry: Arc::clone(&shared),
session_id: s1_id,
dir: dir2.clone(),
target,
t0_boot_ns: 100,
note: None,
host_narration: false,
source: InnerSource::Screen {
session: ScreenCaptureSession {
failure: Arc::default(),
started_at: Instant::now(),
stopping: Arc::new(AtomicBool::new(false)),
urgent_snapshot: Arc::new(AtomicBool::new(false)),
frames: Arc::new(Mutex::new(CandidateBuffer::new(1 << 20))),
sampler_thread: None,
_recording: crate::retention::Recording::start(),
},
audio: None,
windows: None,
},
stopped_input: None,
};
// Cancelling the stale capture must not clear session 2's active state or flags.
stale.cancel();
assert!(shared.active.load(Ordering::Acquire));
assert_eq!(shared.take_flags(), vec![333]);
// Finishing sealed1 later produces its bundle with the click recorded before seal() and
// honors c1's ConfigSource-supplied agentContext setting rather than global handoff state.
assert!(!crate::handoff::experiment(crate::handoff::AGENT_CONTEXT));
let finished1 = sealed1.finish(None).unwrap();
assert_eq!(finished1.bundle.events.len(), 1);
assert!(
finished1.summary.contains("· 0:00 clicked"),
"summary: {}",
finished1.summary
);
assert_eq!(finished1.gaps, 0);
assert_eq!(finished1.missing_transcript.as_deref(), Some("no_audio"));
let _ = std::fs::remove_dir_all(&dir1);
let _ = std::fs::remove_dir_all(&dir2);
}
#[test]
fn candidate_buffer_stays_bounded_and_keeps_the_whole_timeline() {
// A 1 MB budget of 4 KB frames: a 256-frame cap.
let mut buffer = CandidateBuffer::new(1 << 20);
let frame = |t| RgbaFrame::new(t, 32, 32, vec![0; 32 * 32 * 4]);
let mut t = 0;
let mut peak = 0;
// Three hours at the sampler's pace, with a click frame every 10 s.
while t < 3 * 3_600_000 {
buffer.push(frame(t), t % 10_000 == 0);
peak = peak.max(buffer.len());
t += buffer.interval().as_millis() as u64;
}
assert!(peak <= 256, "peak {peak}");
assert!(buffer.interval() >= FRAME_SAMPLE_INTERVAL * 64);
let frames = buffer.into_frames();
assert_eq!(frames[0].t_ms, 0, "the start is kept");
assert!(frames.windows(2).all(|w| w[0].t_ms < w[1].t_ms));
// Spread over the whole capture, not bunched at the end.
let first_half = frames.iter().filter(|f| f.t_ms < 3 * 1_800_000).count();
assert!(
first_half * 3 > frames.len(),
"{first_half} of {}",
frames.len()
);
}
#[test]
fn candidate_buffer_keeps_urgent_frames_when_thinning() {
let mut buffer = CandidateBuffer::new(40 * 4096);
let frame = |t| RgbaFrame::new(t, 32, 32, vec![0; 32 * 32 * 4]);
for t in 0..60 {
buffer.push(frame(t * 220), t == 33);
}
assert!(buffer.len() < 60);
assert_eq!(buffer.interval(), FRAME_SAMPLE_INTERVAL * 2);
assert!(buffer.into_frames().iter().any(|f| f.t_ms == 33 * 220));
}
#[test]
fn frames_that_may_show_a_system_prompt_are_dropped_but_never_the_first() {
let mut buffer = CandidateBuffer::new(1 << 20);
for t in [0, 220, 440, 660, 880] {
buffer.push(RgbaFrame::new(t, 4, 4, vec![0; 64]), false);
}
buffer.drop_since(440);
let times: Vec<u64> = buffer.into_frames().iter().map(|f| f.t_ms).collect();
assert_eq!(times, [0, 220, 440]);
let mut buffer = CandidateBuffer::new(1 << 20);
buffer.push(RgbaFrame::new(500, 4, 4, vec![0; 64]), false);
buffer.drop_since(0);
assert_eq!(buffer.len(), 1);
}
#[test]
fn candidates_are_small_and_recordings_keep_full_size() {
let target = |mode, width, height| TargetInfo {
mode,
app: None,
bundle_id: None,
window_title: None,
window_id: None,
display_id: 1,
frame: TargetFrame {
x: 0.0,
y: 0.0,
width,
height,
scale: 2.0,
},
};
let display = target(CaptureMode::Display, 1728.0, 1117.0);
assert_eq!(capture_size(&display, CANDIDATE_EDGE), (480, 310));
let region = target(CaptureMode::Region, 480.0, 320.0);
assert_eq!(capture_size(&region, CANDIDATE_EDGE), (480, 320));
if cfg!(any(target_os = "windows", target_os = "linux")) {
// Frames are in pixels: never upscaled past the screen's own.
assert_eq!(capture_size(&display, 2560.0), (1728, 1116));
assert_eq!(capture_size(&region, 1280.0), (480, 320));
let wide = target(CaptureMode::Display, 3440.0, 1440.0);
assert_eq!(capture_size(&wide, 2560.0), (2560, 1072));
} else {
assert_eq!(capture_size(&display, 2560.0), (2560, 1654));
assert_eq!(capture_size(&region, 1280.0), (960, 640));
}
// A 1x external display is recorded at its own pixels, never upscaled.
let external = |mode, width, height| TargetInfo {
frame: crate::screens::with_scale([-1920.0, 37.0, width, height], 1.0),
..target(mode, width, height)
};
let screen = external(CaptureMode::Display, 1920.0, 1080.0);
assert_eq!(capture_size(&screen, 2560.0), (1920, 1080));
assert_eq!(capture_size(&screen, CANDIDATE_EDGE), (480, 270));
let region = external(CaptureMode::Region, 480.0, 320.0);
assert_eq!(capture_size(&region, 1280.0), (480, 320));
// A 2x one above the laptop keeps the 2560 px long edge.
let above = TargetInfo {
frame: crate::screens::with_scale([0.0, -1440.0, 2560.0, 1440.0], 2.0),
..target(CaptureMode::Display, 2560.0, 1440.0)
};
assert_eq!(capture_size(&above, 2560.0), (2560, 1440));
}
#[test]
fn cached_window_hit_preserves_front_to_back_order_and_screen_coordinates() {
let window = |id, x, y| TargetInfo {
mode: CaptureMode::Window,
app: Some(format!("App {id}")),
bundle_id: None,
window_title: Some(format!("Window {id}")),
window_id: Some(id),
display_id: 1,
frame: TargetFrame {
x,
y,
width: 200.0,
height: 100.0,
scale: 2.0,
},
};
let windows = [window(1, -100.0, 20.0), window(2, 0.0, 0.0)];
assert_eq!(
selectable_window_at(&windows, 50.0, 50.0)
.unwrap()
.window_id,
Some(1)
);
assert_eq!(
selectable_window_at(&windows, 150.0, 50.0)
.unwrap()
.window_id,
Some(2)
);
assert_eq!(
selectable_window_at(&windows, -100.0, 20.0)
.unwrap()
.window_id,
Some(1)
);
assert!(selectable_window_at(&windows, -101.0, 50.0).is_none());
assert!(selectable_window_at(&[], 0.0, 0.0).is_none());
}
#[test]
fn shared_resources_default_to_none_and_yield_when_peer_runs() {
let engine = Engine::default_engine();
assert_eq!(engine.resources(), SharedResources::none());
assert_eq!(
SharedResources::when_peer_absent(true),
SharedResources::none()
);
assert_eq!(
SharedResources::when_peer_absent(false),
SharedResources::all()
);
let custom = SharedResources::none()
.with_ingest_server(true)
.with_delivery_retries(false)
.with_global_shortcuts(false);
let engine = engine.with_resources(custom);
assert_eq!(engine.resources(), custom);
// `install_shared_resources` on `SharedResources::none()` is a no-op.
Engine::default_engine().install_shared_resources();
}
#[test]
fn start_defaults_to_external_audio_and_never_opens_microphone() {
let target = TargetInfo {
mode: CaptureMode::Display,
app: None,
bundle_id: None,
window_title: None,
window_id: None,
display_id: 1,
frame: TargetFrame {
x: 0.0,
y: 0.0,
width: 100.0,
height: 100.0,
scale: 1.0,
},
};
assert_eq!(Start::default().audio, AudioIn::External);
assert_eq!(Start::screen(target.clone()).audio, AudioIn::External);
assert_eq!(Start::screen_with_mic(target).audio, AudioIn::Microphone);
let (tx, _rx) = flume::unbounded();
let host = ChannelHost::new(tx);
let mic_opened = AtomicBool::new(false);
assert!(
start_audio_for(AudioIn::External, &host, || -> Result<()> {
mic_opened.store(true, Ordering::Release);
Ok(())
})
.is_none()
);
assert!(
start_audio_for(AudioIn::None, &host, || -> Result<()> {
mic_opened.store(true, Ordering::Release);
Ok(())
})
.is_none()
);
assert!(!mic_opened.load(Ordering::Acquire));
assert!(
start_audio_for(AudioIn::Microphone, &host, || -> Result<u32> {
mic_opened.store(true, Ordering::Release);
Ok(42)
}) == Some(42)
);
assert!(mic_opened.load(Ordering::Acquire));
}
}