mirror of
https://github.com/anomalyco/hex.git
synced 2026-11-22 21:23:02 +01:00
* Screenshots recordings: quality settings, copy as a file, export size (t1607) Labs › Screenshots gets a Recording group (Scale Retina to 1x, Max resolution, Frame rate, Quality), each with a hex settings key; defaults 1x, Original, 30 fps, High (HEVC 0.50). The editor copies a recording's movie file, and its export-size pop-up (Original, 1080p, 720p) shows an estimate for the span kept, re-encoding off the UI thread with progress on the pressed button. (t1607) * Screenshots: ⌘⇧5 recording toolbar with microphone and computer audio mixed into the movie (t1607) * Screenshots: compact recording toolbar, editor playback and tidier toolbar (t1607) * Screenshots: simplify the toolbar still, export-size save and playhead tick (t1607) * docs: the compact recording toolbar and editor playback (t1607) * Screenshots: filmstrip cells keep the movie's shape; mic-off icon is macOS-only (t1607) * Screenshots: trim editor goldens for the transport and filmstrip (t1607) * Screenshots: a trimmed export keeps a whole video track; reencode has a deadline (t1607) * Screenshots: recording sound isn't silenced; toolbar choices apply at once and save in order (t1607) * Screenshots: Return and Escape reach an open toolbar menu (t1607) * Screenshots: a menu-only open of the chosen microphone; no microphone keeps no audio; Annotate may take the record key while the Lab is off (t1607) * Screenshots: an export lands whole or not at all, and reports only to the recording it came from (t1607) * Screenshots: keep the exact-device open and partial exports macOS-only (t1607)
4210 lines
149 KiB
Rust
4210 lines
149 KiB
Rust
use std::path::{Path, PathBuf};
|
||
use std::sync::atomic::{AtomicBool, AtomicI32, AtomicU64, Ordering};
|
||
use std::sync::{Arc, Mutex};
|
||
use std::thread::{self, JoinHandle};
|
||
use std::time::{Duration, Instant};
|
||
|
||
use anyhow::{Context, Result};
|
||
|
||
use crate::log::scrawl_log;
|
||
use crate::model::{
|
||
CaptureMode, InteractionEvent, RgbaFrame, SCALE, Stroke, TargetFrame, TargetInfo,
|
||
};
|
||
|
||
// The operating system side of capture: global shortcuts, input monitoring, the frontmost app,
|
||
// displays and windows, and the screen recorder. Each platform module provides the same items.
|
||
#[cfg_attr(target_os = "macos", path = "capture/macos.rs")]
|
||
#[cfg_attr(target_os = "windows", path = "capture/windows.rs")]
|
||
#[cfg_attr(target_os = "linux", path = "capture/linux.rs")]
|
||
mod platform;
|
||
pub use platform::*;
|
||
|
||
pub const FRAME_SAMPLE_INTERVAL: Duration = Duration::from_millis(220);
|
||
/// Candidate frames only feed scene diffs and keyframe selection; the keyframes themselves are
|
||
/// decoded from `recording.mp4` at full size afterwards.
|
||
pub const CANDIDATE_EDGE: f64 = 480.0;
|
||
/// Without a recording to decode from, the candidates are the keyframes, so keep them sharper.
|
||
#[cfg_attr(target_os = "linux", allow(dead_code))]
|
||
pub(crate) const CANDIDATE_EDGE_NO_VIDEO: f64 = 1280.0;
|
||
/// The candidates' memory ceiling, however long the capture runs.
|
||
pub const CANDIDATE_BUDGET_BYTES: usize = 384 << 20;
|
||
|
||
/// The candidate frames of a capture, bounded in memory. When full it drops every other frame
|
||
/// (keeping the first and the ones taken for a click, stroke or shortcut) and samples half as
|
||
/// often from then on, so a long capture keeps its whole timeline, evenly, at a coarser step.
|
||
pub struct CandidateBuffer {
|
||
frames: Vec<(RgbaFrame, bool)>,
|
||
cap: usize,
|
||
budget: usize,
|
||
interval: Duration,
|
||
}
|
||
|
||
impl CandidateBuffer {
|
||
pub fn new(budget_bytes: usize) -> Self {
|
||
Self {
|
||
frames: Vec::new(),
|
||
cap: 0,
|
||
budget: budget_bytes,
|
||
interval: FRAME_SAMPLE_INTERVAL,
|
||
}
|
||
}
|
||
|
||
/// How often to sample now.
|
||
pub fn interval(&self) -> Duration {
|
||
self.interval
|
||
}
|
||
|
||
#[cfg(test)]
|
||
fn len(&self) -> usize {
|
||
self.frames.len()
|
||
}
|
||
|
||
pub fn push(&mut self, frame: RgbaFrame, urgent: bool) {
|
||
if self.cap == 0 {
|
||
self.cap = (self.budget / frame.rgba.len().max(1)).clamp(60, 1_200);
|
||
}
|
||
self.frames.push((frame, urgent));
|
||
if self.frames.len() >= self.cap {
|
||
self.thin();
|
||
}
|
||
}
|
||
|
||
fn thin(&mut self) {
|
||
let mut regular = 0_usize;
|
||
let mut kept: Vec<(RgbaFrame, bool)> = std::mem::take(&mut self.frames)
|
||
.into_iter()
|
||
.enumerate()
|
||
.filter(|(i, (_, urgent))| {
|
||
let keep = *i == 0 || *urgent || regular.is_multiple_of(2);
|
||
if !urgent {
|
||
regular += 1;
|
||
}
|
||
keep
|
||
})
|
||
.map(|(_, f)| f)
|
||
.collect();
|
||
// Mostly urgent frames (a storm of clicks): thin those too.
|
||
if kept.len() > self.cap * 3 / 4 {
|
||
kept = kept
|
||
.into_iter()
|
||
.enumerate()
|
||
.filter(|(i, _)| i.is_multiple_of(2))
|
||
.map(|(_, f)| f)
|
||
.collect();
|
||
}
|
||
self.frames = kept;
|
||
self.interval *= 2;
|
||
}
|
||
|
||
/// Forgets frames taken after `t_ms`, except the first.
|
||
#[cfg_attr(not(target_os = "macos"), allow(dead_code))]
|
||
pub fn drop_since(&mut self, t_ms: u64) {
|
||
let mut index = 0;
|
||
self.frames.retain(|(frame, _)| {
|
||
index += 1;
|
||
index == 1 || frame.t_ms <= t_ms
|
||
});
|
||
}
|
||
|
||
pub fn into_frames(self) -> Vec<RgbaFrame> {
|
||
self.frames.into_iter().map(|(f, _)| f).collect()
|
||
}
|
||
}
|
||
|
||
/// What a finished screen capture leaves behind.
|
||
pub struct ScreenRecording {
|
||
/// Small candidate frames for keyframe selection (see `CANDIDATE_EDGE`).
|
||
pub frames: Vec<RgbaFrame>,
|
||
/// The movie, which may still be finishing: only what reads it waits ([`Movie::wait`]).
|
||
pub video: Option<Movie>,
|
||
pub duration_ms: u64,
|
||
/// Why there is no picture of the screen, when no frame was recorded: the summary says so
|
||
/// instead of showing anything.
|
||
pub no_screen: Option<String>,
|
||
}
|
||
|
||
/// Why a screen capture recorded no frame: what the screen source said, or the platform's reason.
|
||
pub fn no_screen_reason(failure: Option<String>) -> String {
|
||
match failure {
|
||
Some(why) => why,
|
||
None if screen_capture_usable(false) => "the screen recorded no frames".into(),
|
||
None => format!("Screen Recording is off for {}", host_app_name()),
|
||
}
|
||
}
|
||
|
||
/// The app this process runs as: its `.app` bundle's name, else its executable's.
|
||
fn host_app_name() -> String {
|
||
let exe = std::env::current_exe().unwrap_or_default();
|
||
exe.ancestors()
|
||
.find(|dir| dir.extension().is_some_and(|ext| ext == "app"))
|
||
.unwrap_or(&exe)
|
||
.file_stem()
|
||
.map(|name| name.to_string_lossy().into_owned())
|
||
.unwrap_or_else(|| "this app".into())
|
||
}
|
||
|
||
impl ScreenRecording {
|
||
/// Blocks until nothing writes the movie any more, so its bundle can be removed.
|
||
pub fn discard(self) {
|
||
if let Some(movie) = self.video {
|
||
let _ = movie.wait();
|
||
}
|
||
}
|
||
}
|
||
|
||
/// A screen capture's movie. On macOS its file is still being finished when the capture stops:
|
||
/// the recording output closes it and its final frame is held to the stop.
|
||
pub struct Movie {
|
||
path: PathBuf,
|
||
state: MovieState,
|
||
}
|
||
|
||
enum MovieState {
|
||
Written {
|
||
offset_ms: u64,
|
||
},
|
||
#[cfg(target_os = "macos")]
|
||
Finishing(platform::FinishingMovie),
|
||
}
|
||
|
||
impl Movie {
|
||
/// A movie already written, starting `offset_ms` into the capture (on the frames' clock).
|
||
pub fn written(path: PathBuf, offset_ms: u64) -> Self {
|
||
Self {
|
||
path,
|
||
state: MovieState::Written { offset_ms },
|
||
}
|
||
}
|
||
|
||
#[cfg(target_os = "macos")]
|
||
pub(crate) fn finishing(path: PathBuf, finishing: platform::FinishingMovie) -> Self {
|
||
Self {
|
||
path,
|
||
state: MovieState::Finishing(finishing),
|
||
}
|
||
}
|
||
|
||
/// Where the movie is, finished or not.
|
||
pub fn path(&self) -> &Path {
|
||
&self.path
|
||
}
|
||
|
||
/// When the movie starts on the frames' clock, before it finishes: exact once written, an
|
||
/// estimate from when its recording started while it finishes.
|
||
pub fn offset_hint_ms(&self) -> u64 {
|
||
match &self.state {
|
||
MovieState::Written { offset_ms } => *offset_ms,
|
||
#[cfg(target_os = "macos")]
|
||
MovieState::Finishing(finishing) => finishing.offset_hint_ms(),
|
||
}
|
||
}
|
||
|
||
/// Blocks until the movie is finished. Returns its path and when it starts on the frames'
|
||
/// clock, or `None` when nothing playable was left.
|
||
pub fn wait(self) -> Option<(PathBuf, u64)> {
|
||
let offset_ms = match self.state {
|
||
MovieState::Written { offset_ms } => Some(offset_ms),
|
||
#[cfg(target_os = "macos")]
|
||
MovieState::Finishing(finishing) => finishing.wait(),
|
||
}?;
|
||
let written = self
|
||
.path
|
||
.metadata()
|
||
.is_ok_and(|m| m.is_file() && m.len() > 0);
|
||
written.then_some((self.path, offset_ms))
|
||
}
|
||
}
|
||
|
||
#[derive(Clone, Copy, Debug, Eq, PartialEq)]
|
||
pub enum Stage {
|
||
Recording,
|
||
Stopping,
|
||
Transcribing,
|
||
Rendering,
|
||
Writing,
|
||
Done,
|
||
Cancelled,
|
||
Failed,
|
||
}
|
||
|
||
impl Stage {
|
||
pub fn label(self) -> &'static str {
|
||
match self {
|
||
Self::Recording => "recording",
|
||
Self::Stopping => "stopping",
|
||
Self::Transcribing => "transcribing",
|
||
Self::Rendering => "rendering",
|
||
Self::Writing => "writing",
|
||
Self::Done => "done",
|
||
Self::Cancelled => "cancelled",
|
||
Self::Failed => "failed",
|
||
}
|
||
}
|
||
}
|
||
|
||
/// Live engine events emitted by the global input monitor, iPhone mirror and ingest server.
|
||
#[derive(Clone, Debug)]
|
||
pub enum Event {
|
||
/// ⌥⇧S (or the configured shortcut) went down; `shortcut::Shortcut` decides what it means.
|
||
CaptureShortcut,
|
||
PasteLatestShortcut,
|
||
RequestSnapshot,
|
||
/// The iPhone mirror ended on its own (unplugged, never found, or access denied).
|
||
PhoneEnded,
|
||
/// Lifecycle progress of an active or finishing capture.
|
||
Progress {
|
||
bundle: PathBuf,
|
||
stage: Stage,
|
||
},
|
||
/// A ScrawlKit recording was ingested from an iPhone and compiled into a `.scrawl` bundle.
|
||
IngestedBundle {
|
||
dir: PathBuf,
|
||
bundle: Box<crate::model::ScrawlBundle>,
|
||
status: Option<String>,
|
||
},
|
||
/// A capture bundle was delivered (or queued for retry) to a destination.
|
||
Delivered {
|
||
dir: PathBuf,
|
||
delivery: crate::delivery::Delivery,
|
||
},
|
||
}
|
||
|
||
#[derive(Clone, Debug, Default)]
|
||
pub struct OwnWindows {
|
||
/// Window IDs excluded from ScreenCaptureKit / Windows.Graphics.Capture recordings.
|
||
pub excluded: Vec<u32>,
|
||
/// Click-through window IDs (such as the ink overlay) ignored when checking whether a click
|
||
/// landed on the host's own UI.
|
||
pub passthrough: Vec<u32>,
|
||
/// Set, the recording shows the host's own windows: neither `excluded` nor the host's
|
||
/// other non-layer-0 windows are left out. Read while recording, so the host can flip it
|
||
/// mid-capture (a demo that shows the host's overlays).
|
||
pub recorded: Arc<AtomicBool>,
|
||
}
|
||
|
||
/// What the host provides. Every method has a default except `emit`; `emit` and `log` must return
|
||
/// without blocking.
|
||
pub trait Host: Send + Sync + 'static {
|
||
fn emit(&self, event: Event);
|
||
fn log(&self, line: &str) {
|
||
scrawl_log(line);
|
||
}
|
||
fn metric(&self, name: &str, ms: f64) {
|
||
crate::log::record_metric(name, ms);
|
||
}
|
||
fn clipboard(&self) -> Option<&dyn crate::handoff::Clipboard> {
|
||
None
|
||
}
|
||
/// Window IDs to leave out of recordings (overlay, capsule, host windows) and to let clicks
|
||
/// through (`excluded_window_ids` / `passthrough_window_ids`).
|
||
fn own_windows(&self) -> OwnWindows {
|
||
OwnWindows::default()
|
||
}
|
||
}
|
||
|
||
pub struct ChannelHost {
|
||
sender: flume::Sender<Event>,
|
||
}
|
||
|
||
impl ChannelHost {
|
||
pub fn new(sender: flume::Sender<Event>) -> Self {
|
||
Self { sender }
|
||
}
|
||
}
|
||
|
||
impl Host for ChannelHost {
|
||
fn emit(&self, event: Event) {
|
||
let _ = self.sender.try_send(event);
|
||
}
|
||
}
|
||
|
||
pub trait ConfigSource: Send + Sync {
|
||
fn engine_config(&self) -> Result<crate::handoff::EngineConfig>;
|
||
}
|
||
|
||
#[derive(Clone, Copy, Debug, Default)]
|
||
pub struct DefaultConfigSource;
|
||
|
||
impl ConfigSource for DefaultConfigSource {
|
||
fn engine_config(&self) -> Result<crate::handoff::EngineConfig> {
|
||
Ok(crate::handoff::engine_config())
|
||
}
|
||
}
|
||
|
||
#[derive(Clone, Debug, Default)]
|
||
pub enum Target {
|
||
/// Discover the display under the cursor (or focused window) when starting.
|
||
#[default]
|
||
ActiveDisplay,
|
||
/// Record a specific screen, window or region target.
|
||
Screen(TargetInfo),
|
||
/// Record a cabled iPhone (either a new session for `device` or an existing preview). With
|
||
/// [`AudioIn::External`] the mirror opens no microphone and `finish`'s audio is the
|
||
/// narration; otherwise the mirror records the Mac's microphone into the movie.
|
||
Phone {
|
||
device: Option<String>,
|
||
name: Option<String>,
|
||
},
|
||
}
|
||
|
||
#[derive(Clone, Copy, Debug, Default, Eq, PartialEq)]
|
||
pub enum AudioIn {
|
||
/// No engine-managed microphone stream; the host passes `Option<RecordedAudio>` to `finish`.
|
||
#[default]
|
||
External,
|
||
/// No audio at all.
|
||
None,
|
||
/// Open Scrawl's built-in `AudioRecorder` stream.
|
||
Microphone,
|
||
}
|
||
|
||
/// How a screen capture's movie is recorded (macOS; elsewhere always compact).
|
||
#[derive(Clone, Copy, Debug, Default, Eq, PartialEq)]
|
||
pub enum MovieQuality {
|
||
/// Sized for agents: H.264, 1280 px long edge (2560 for a whole display, for crops).
|
||
#[default]
|
||
Compact,
|
||
/// For people: hardware HEVC (`capture_core::Encoder::Hevc`) at the size, frame rate, and
|
||
/// quality asked.
|
||
Full(MovieSize),
|
||
}
|
||
|
||
/// A [`MovieQuality::Full`] movie's size, frame rate, and quality.
|
||
#[derive(Clone, Copy, Debug, Eq, PartialEq)]
|
||
pub struct MovieSize {
|
||
/// At most one pixel a point, so a Retina display records at its size in points.
|
||
pub one_x: bool,
|
||
/// The longest side in pixels; `None` keeps the source's (or one pixel a point's).
|
||
pub long_edge: Option<u32>,
|
||
pub fps: u32,
|
||
/// HEVC's constant quality, in hundredths.
|
||
pub quality: u8,
|
||
/// Also records other apps' audio (never Hex's) into `system.m4a`.
|
||
pub system_audio: bool,
|
||
}
|
||
|
||
impl Default for MovieSize {
|
||
/// Every pixel at up to 60 fps, at `capture_core`'s quality for Retina text.
|
||
fn default() -> Self {
|
||
Self {
|
||
one_x: false,
|
||
long_edge: None,
|
||
fps: 60,
|
||
quality: 70,
|
||
system_audio: false,
|
||
}
|
||
}
|
||
}
|
||
|
||
impl MovieSize {
|
||
/// The long edge a source `points` large records at: one pixel a point with
|
||
/// [`Self::one_x`], capped at [`Self::long_edge`]. `None` keeps every pixel.
|
||
pub fn long_edge_for(self, points: (f64, f64)) -> Option<u32> {
|
||
let longest = points.0.max(points.1);
|
||
let one_x =
|
||
(self.one_x && longest.is_finite() && longest >= 1.0).then(|| longest.round() as u32);
|
||
match (one_x, self.long_edge) {
|
||
(Some(a), Some(b)) => Some(a.min(b)),
|
||
(a, b) => a.or(b),
|
||
}
|
||
}
|
||
}
|
||
|
||
mod start;
|
||
mod token;
|
||
pub use start::Start;
|
||
pub use token::{OutputBounds, PortalToken};
|
||
|
||
#[derive(Clone, Debug, Default)]
|
||
pub struct RouteContext {
|
||
pub front_bundle_id: Option<String>,
|
||
pub captured_bundle_id: Option<String>,
|
||
pub app_pid: Option<i32>,
|
||
}
|
||
|
||
#[derive(Clone, Debug)]
|
||
pub struct Finished {
|
||
pub dir: PathBuf,
|
||
pub bundle: crate::model::ScrawlBundle,
|
||
pub summary: String,
|
||
pub words: usize,
|
||
pub duration_ms: u64,
|
||
pub route_hint: RouteContext,
|
||
/// Number of audio gap spans where microphone samples were missing or dropped (`RecordedAudio::gaps`).
|
||
pub gaps: usize,
|
||
/// Structured reason when no transcript was produced (`"no_audio"` when no audio was
|
||
/// captured, or `bundle.transcript.missing.reason` when transcription failed or is pending).
|
||
pub missing_transcript: Option<String>,
|
||
}
|
||
|
||
// Carbon HIToolbox modifier bits. The shortcut is stored with them on every platform.
|
||
pub const CARBON_CMD_KEY: u32 = 1 << 8;
|
||
pub const CARBON_SHIFT_KEY: u32 = 1 << 9;
|
||
pub const CARBON_OPTION_KEY: u32 = 1 << 11;
|
||
pub const CARBON_CONTROL_KEY: u32 = 1 << 12;
|
||
|
||
#[derive(Clone, Debug, PartialEq, Eq, serde::Serialize, serde::Deserialize)]
|
||
pub struct ShortcutConfig {
|
||
pub key_code: u32,
|
||
pub carbon_modifiers: u32,
|
||
pub label: String,
|
||
}
|
||
|
||
impl Default for ShortcutConfig {
|
||
fn default() -> Self {
|
||
Self::default_record()
|
||
}
|
||
}
|
||
|
||
/// Scrawl's two global shortcuts.
|
||
#[derive(Clone, Copy, Debug, PartialEq, Eq, serde::Serialize, serde::Deserialize)]
|
||
#[serde(rename_all = "kebab-case")]
|
||
pub enum ShortcutKind {
|
||
/// Start, switch or finish a recording (⌥⇧S).
|
||
Record,
|
||
/// Paste the latest capture into the app in front (⌥⇧V).
|
||
Paste,
|
||
}
|
||
|
||
impl ShortcutKind {
|
||
pub const ALL: [Self; 2] = [Self::Record, Self::Paste];
|
||
|
||
pub fn default_config(self) -> ShortcutConfig {
|
||
match self {
|
||
Self::Record => ShortcutConfig::default_record(),
|
||
Self::Paste => ShortcutConfig::default_paste(),
|
||
}
|
||
}
|
||
|
||
/// The letter of the default, on the active layout (Dvorak's S is not ANSI's).
|
||
fn default_char(self) -> char {
|
||
match self {
|
||
Self::Record => 's',
|
||
Self::Paste => 'v',
|
||
}
|
||
}
|
||
|
||
pub(crate) fn ansi_code(self) -> u16 {
|
||
match self {
|
||
Self::Record => KEY_S,
|
||
Self::Paste => KEY_V,
|
||
}
|
||
}
|
||
|
||
pub fn name(self) -> &'static str {
|
||
match self {
|
||
Self::Record => "Record",
|
||
Self::Paste => "Paste last capture",
|
||
}
|
||
}
|
||
|
||
/// Whether it can be turned off. Record can't: it's the way into Scrawl.
|
||
pub fn can_unset(self) -> bool {
|
||
self == Self::Paste
|
||
}
|
||
}
|
||
|
||
impl ShortcutConfig {
|
||
pub fn default_record() -> Self {
|
||
Self {
|
||
key_code: u32::from(layout_s_keycode()),
|
||
carbon_modifiers: CARBON_OPTION_KEY | CARBON_SHIFT_KEY,
|
||
label: shortcut_label(CARBON_OPTION_KEY | CARBON_SHIFT_KEY, "S"),
|
||
}
|
||
}
|
||
|
||
pub fn default_paste() -> Self {
|
||
Self {
|
||
key_code: u32::from(layout_v_keycode()),
|
||
carbon_modifiers: CARBON_OPTION_KEY | CARBON_SHIFT_KEY,
|
||
label: shortcut_label(CARBON_OPTION_KEY | CARBON_SHIFT_KEY, "V"),
|
||
}
|
||
}
|
||
|
||
pub fn defaults() -> (Self, Self) {
|
||
(Self::default_record(), Self::default_paste())
|
||
}
|
||
|
||
/// Default shortcuts using ANSI key codes (`KEY_S`, `KEY_V`) without querying macOS TIS/TSM,
|
||
/// safe to call off the main thread.
|
||
pub fn ansi_defaults() -> (Self, Self) {
|
||
(
|
||
Self {
|
||
key_code: u32::from(KEY_S),
|
||
carbon_modifiers: CARBON_OPTION_KEY | CARBON_SHIFT_KEY,
|
||
label: shortcut_label(CARBON_OPTION_KEY | CARBON_SHIFT_KEY, "S"),
|
||
},
|
||
Self {
|
||
key_code: u32::from(KEY_V),
|
||
carbon_modifiers: CARBON_OPTION_KEY | CARBON_SHIFT_KEY,
|
||
label: shortcut_label(CARBON_OPTION_KEY | CARBON_SHIFT_KEY, "V"),
|
||
},
|
||
)
|
||
}
|
||
|
||
/// No shortcut: nothing is registered and no key press matches. Only the paste shortcut can
|
||
/// be turned off.
|
||
pub fn unset() -> Self {
|
||
Self {
|
||
key_code: u32::MAX,
|
||
carbon_modifiers: 0,
|
||
label: String::new(),
|
||
}
|
||
}
|
||
|
||
pub fn is_unset(&self) -> bool {
|
||
self.label.is_empty()
|
||
}
|
||
|
||
/// Whether this is `kind`'s default, which also answers to its aliases (⌃⌥⇧, ⌘⌥⇧, ⌃⌥⌘
|
||
/// and the ANSI key).
|
||
pub fn is_default_for(&self, kind: ShortcutKind) -> bool {
|
||
self.carbon_modifiers == CARBON_OPTION_KEY | CARBON_SHIFT_KEY
|
||
&& shortcut_parts(&self.label).1 == kind.default_char().to_ascii_uppercase().to_string()
|
||
}
|
||
|
||
pub fn key_label(&self) -> &str {
|
||
shortcut_parts(&self.label).1
|
||
}
|
||
}
|
||
|
||
/// `⌃⌥⇧⌘` in the order macOS writes them, for Carbon modifier bits; on Windows the same
|
||
/// modifiers as `Ctrl+Win+Alt+Shift+` (ready to take a key).
|
||
pub fn modifier_symbols(carbon_modifiers: u32) -> String {
|
||
if cfg!(target_os = "windows") {
|
||
let label = shortcut_label(carbon_modifiers, "");
|
||
return label;
|
||
}
|
||
[
|
||
(CARBON_CONTROL_KEY, '⌃'),
|
||
(CARBON_OPTION_KEY, '⌥'),
|
||
(CARBON_SHIFT_KEY, '⇧'),
|
||
(CARBON_CMD_KEY, '⌘'),
|
||
]
|
||
.into_iter()
|
||
.filter(|(bit, _)| carbon_modifiers & bit != 0)
|
||
.map(|(_, symbol)| symbol)
|
||
.collect()
|
||
}
|
||
|
||
/// A shortcut as this platform writes it: `⌥⇧S` on macOS, `Alt+Shift+S` on Windows.
|
||
pub fn shortcut_label(carbon_modifiers: u32, key: &str) -> String {
|
||
let held = |bit: u32| carbon_modifiers & bit != 0;
|
||
if cfg!(target_os = "windows") {
|
||
let mut parts = Vec::new();
|
||
for (bit, name) in [
|
||
(CARBON_CONTROL_KEY, "Ctrl"),
|
||
(CARBON_CMD_KEY, "Win"),
|
||
(CARBON_OPTION_KEY, "Alt"),
|
||
(CARBON_SHIFT_KEY, "Shift"),
|
||
] {
|
||
if held(bit) {
|
||
parts.push(name);
|
||
}
|
||
}
|
||
parts.push(key);
|
||
parts.join("+")
|
||
} else {
|
||
let mut label = String::new();
|
||
for (bit, glyph) in [
|
||
(CARBON_CONTROL_KEY, '⌃'),
|
||
(CARBON_OPTION_KEY, '⌥'),
|
||
(CARBON_SHIFT_KEY, '⇧'),
|
||
(CARBON_CMD_KEY, '⌘'),
|
||
] {
|
||
if held(bit) {
|
||
label.push(glyph);
|
||
}
|
||
}
|
||
label.push_str(key);
|
||
label
|
||
}
|
||
}
|
||
|
||
/// The system paste keystroke: `⌘V` on macOS, `Ctrl+V` on Windows.
|
||
pub fn system_paste_label() -> &'static str {
|
||
if cfg!(target_os = "windows") {
|
||
"Ctrl+V"
|
||
} else {
|
||
"⌘V"
|
||
}
|
||
}
|
||
|
||
/// The keys of a shortcut label, one per keycap: `⌥⇧S` → `⌥`, `⇧`, `S`; `Alt+Shift+S` →
|
||
/// `Alt`, `Shift`, `S`.
|
||
pub fn shortcut_keys(label: &str) -> Vec<String> {
|
||
if label.contains('+') && label.len() > 1 {
|
||
let mut keys: Vec<String> = label
|
||
.split('+')
|
||
.filter(|key| !key.is_empty())
|
||
.map(str::to_owned)
|
||
.collect();
|
||
// `Ctrl++` binds the plus key itself.
|
||
if label.ends_with("++") {
|
||
keys.push("+".into());
|
||
}
|
||
keys
|
||
} else {
|
||
label.chars().map(String::from).collect()
|
||
}
|
||
}
|
||
|
||
/// A shortcut label split into its modifiers and key: `⌥⇧S` → (`⌥⇧`, `S`), and on Windows
|
||
/// `Alt+Shift+S` → (`Alt+Shift`, `S`).
|
||
pub fn shortcut_parts(label: &str) -> (&str, &str) {
|
||
// The key itself may be "+" (`Ctrl++`), or a letter outside ASCII.
|
||
let before_key = label.char_indices().last().map_or("", |(i, _)| &label[..i]);
|
||
if let Some(plus) = before_key.rfind('+') {
|
||
return (&label[..plus], &label[plus + 1..]);
|
||
}
|
||
let split = label
|
||
.char_indices()
|
||
.find(|(_, c)| !matches!(c, '⌃' | '⌥' | '⇧' | '⌘'))
|
||
.map_or(label.len(), |(i, _)| i);
|
||
label.split_at(split)
|
||
}
|
||
|
||
/// Whether a key press is the record shortcut. A custom one matches exactly; the default ⌥⇧S
|
||
/// also answers with ⌃ or ⌘ added and as ⌃⌥⌘S, on the active layout's S key (`s_code`) or the
|
||
/// ANSI one.
|
||
pub fn is_capture_shortcut(
|
||
cfg: &ShortcutConfig,
|
||
s_code: u16,
|
||
keycode: u16,
|
||
cmd: bool,
|
||
ctrl: bool,
|
||
alt: bool,
|
||
shift: bool,
|
||
) -> bool {
|
||
shortcut_matches(
|
||
ShortcutKind::Record,
|
||
cfg,
|
||
s_code,
|
||
keycode,
|
||
cmd,
|
||
ctrl,
|
||
alt,
|
||
shift,
|
||
)
|
||
}
|
||
|
||
/// `is_capture_shortcut` for any of Scrawl's shortcuts; `layout_code` is the default key on the
|
||
/// active layout.
|
||
#[allow(clippy::too_many_arguments)]
|
||
pub fn shortcut_matches(
|
||
kind: ShortcutKind,
|
||
cfg: &ShortcutConfig,
|
||
layout_code: u16,
|
||
keycode: u16,
|
||
cmd: bool,
|
||
ctrl: bool,
|
||
alt: bool,
|
||
shift: bool,
|
||
) -> bool {
|
||
if cfg.is_unset() {
|
||
return false;
|
||
}
|
||
let want_cmd = (cfg.carbon_modifiers & CARBON_CMD_KEY) != 0;
|
||
let want_ctrl = (cfg.carbon_modifiers & CARBON_CONTROL_KEY) != 0;
|
||
let want_alt = (cfg.carbon_modifiers & CARBON_OPTION_KEY) != 0;
|
||
let want_shift = (cfg.carbon_modifiers & CARBON_SHIFT_KEY) != 0;
|
||
let exact = u32::from(keycode) == cfg.key_code
|
||
&& cmd == want_cmd
|
||
&& ctrl == want_ctrl
|
||
&& alt == want_alt
|
||
&& shift == want_shift;
|
||
if exact {
|
||
return true;
|
||
}
|
||
let is_key = keycode == layout_code || keycode == kind.ansi_code();
|
||
cfg.is_default_for(kind) && is_key && alt && (shift || (ctrl && cmd))
|
||
}
|
||
|
||
/// The active layout's key for `kind`'s default letter (Windows' hooks and poller match on it).
|
||
#[cfg_attr(target_os = "macos", allow(dead_code))]
|
||
pub fn layout_default_keycode(kind: ShortcutKind) -> u16 {
|
||
key_code_for_char(kind.default_char()).unwrap_or(kind.ansi_code())
|
||
}
|
||
|
||
/// The key codes and Carbon modifiers `cfg` registers: itself, plus for a default its aliases
|
||
/// (⌃⌥⇧, ⌘⌥⇧, ⌃⌥⌘) on the active layout's key and the ANSI one.
|
||
pub fn hotkey_combos(kind: ShortcutKind, cfg: &ShortcutConfig) -> Vec<(u32, u32)> {
|
||
if cfg.is_unset() {
|
||
return Vec::new();
|
||
}
|
||
let mut combos = vec![(cfg.key_code, cfg.carbon_modifiers)];
|
||
if !cfg.is_default_for(kind) {
|
||
return combos;
|
||
}
|
||
let mut codes = vec![cfg.key_code];
|
||
if cfg.key_code != u32::from(kind.ansi_code()) {
|
||
codes.push(u32::from(kind.ansi_code()));
|
||
}
|
||
for code in codes {
|
||
for mods in [
|
||
CARBON_OPTION_KEY | CARBON_SHIFT_KEY,
|
||
CARBON_CONTROL_KEY | CARBON_OPTION_KEY | CARBON_SHIFT_KEY,
|
||
CARBON_CMD_KEY | CARBON_OPTION_KEY | CARBON_SHIFT_KEY,
|
||
CARBON_CONTROL_KEY | CARBON_OPTION_KEY | CARBON_CMD_KEY,
|
||
] {
|
||
if !combos.contains(&(code, mods)) {
|
||
combos.push((code, mods));
|
||
}
|
||
}
|
||
}
|
||
combos
|
||
}
|
||
|
||
#[cfg_attr(target_os = "linux", allow(dead_code))]
|
||
fn clean_label(input: &str) -> String {
|
||
let squashed = input.split_whitespace().collect::<Vec<_>>().join(" ");
|
||
let trimmed = squashed.trim();
|
||
if trimmed.chars().count() > 44 {
|
||
let prefix: String = trimmed.chars().take(43).collect();
|
||
format!("{}…", prefix.trim_end())
|
||
} else {
|
||
trimmed.to_string()
|
||
}
|
||
}
|
||
|
||
/// Which display to record: the one under `point`, else the one holding the focused window's
|
||
/// centre, else the first. Frames and points are global, top-left origin.
|
||
pub fn pick_display(
|
||
displays: &[[f64; 4]],
|
||
point: Option<[f64; 2]>,
|
||
focused: Option<[f64; 4]>,
|
||
) -> usize {
|
||
let inside = |p: [f64; 2]| {
|
||
displays
|
||
.iter()
|
||
.position(|&[x, y, w, h]| p[0] >= x && p[0] < x + w && p[1] >= y && p[1] < y + h)
|
||
};
|
||
point
|
||
.and_then(inside)
|
||
.or_else(|| focused.and_then(|[x, y, w, h]| inside([x + w / 2.0, y + h / 2.0])))
|
||
.unwrap_or(0)
|
||
}
|
||
|
||
pub fn selectable_window_at(windows: &[TargetInfo], x: f64, y: f64) -> Option<&TargetInfo> {
|
||
windows.iter().find(|window| {
|
||
let f = window.frame;
|
||
x >= f.x && x <= f.x + f.width && y >= f.y && y <= f.y + f.height
|
||
})
|
||
}
|
||
|
||
/// Resolves region metadata from the picker's snapshot, without another SCK query.
|
||
pub fn region_target(
|
||
x: f64,
|
||
y: f64,
|
||
width: f64,
|
||
height: f64,
|
||
windows: &[TargetInfo],
|
||
) -> TargetInfo {
|
||
let mid_x = x + width * 0.5;
|
||
let mid_y = y + height * 0.5;
|
||
let hit = selectable_window_at(windows, mid_x, mid_y);
|
||
let frontmost = frontmost_app_info();
|
||
// One display per recording: the area's (by its centre), trimmed to that display and at
|
||
// least `MIN_AREA` a side inside it, so this frame is exactly the recorded `sourceRect`.
|
||
let screens = crate::screens::displays();
|
||
let (display_id, scale, [x, y, width, height]) =
|
||
crate::screens::area_on_display(&screens, [x, y, width, height]).map_or_else(
|
||
|| {
|
||
let min = crate::screens::MIN_AREA;
|
||
let rect = [x, y, width.max(min), height.max(min)];
|
||
(main_display_id(), MAX_CAPTURE_SCALE, rect)
|
||
},
|
||
|(i, rect)| (screens[i].id, screens[i].scale, rect),
|
||
);
|
||
TargetInfo {
|
||
mode: CaptureMode::Region,
|
||
app: hit
|
||
.and_then(|window| window.app.clone())
|
||
.or_else(|| frontmost.as_ref().map(|(_, name, _)| name.clone())),
|
||
bundle_id: hit
|
||
.and_then(|window| window.bundle_id.clone())
|
||
.or_else(|| frontmost.and_then(|(_, _, bid)| bid)),
|
||
window_title: hit.and_then(|window| window.window_title.clone()),
|
||
window_id: None,
|
||
display_id,
|
||
frame: TargetFrame {
|
||
x,
|
||
y,
|
||
width,
|
||
height,
|
||
scale,
|
||
},
|
||
}
|
||
}
|
||
|
||
/// The capture size for `target` with its long edge at most `long_edge` px (and at most its
|
||
/// display's pixels: a 1x display isn't upscaled, and on Windows frames are pixels already,
|
||
/// `MAX_CAPTURE_SCALE`).
|
||
pub fn capture_size(target: &TargetInfo, long_edge: f64) -> (u32, u32) {
|
||
let max_dim = target.frame.width.max(target.frame.height).max(1.0);
|
||
let native = target.frame.scale.clamp(1.0, MAX_CAPTURE_SCALE);
|
||
let scale = (long_edge / max_dim).min(native);
|
||
(
|
||
((target.frame.width * scale).round() as u32).max(48) & !1,
|
||
((target.frame.height * scale).round() as u32).max(48) & !1,
|
||
)
|
||
}
|
||
|
||
/// What the platform's sampler thread records into. It runs until `stopping`, pushes candidate
|
||
/// frames (at once when `urgent`), and returns the movie it recorded.
|
||
pub(crate) struct Sampler {
|
||
pub target: TargetInfo,
|
||
pub video_output_path: Option<PathBuf>,
|
||
#[cfg_attr(not(target_os = "macos"), allow(dead_code))]
|
||
pub movie_quality: MovieQuality,
|
||
/// The screen-cast portal's token choice; ignored off Linux.
|
||
#[cfg_attr(not(target_os = "linux"), allow(dead_code))]
|
||
pub portal_token: PortalToken,
|
||
/// Scrawl's own windows to leave out of the recording.
|
||
#[cfg_attr(any(target_os = "windows", target_os = "linux"), allow(dead_code))]
|
||
pub excluded_window_ids: Arc<Mutex<Vec<u32>>>,
|
||
/// [`OwnWindows::recorded`]: while set, nothing of the host's is left out.
|
||
#[cfg_attr(any(target_os = "windows", target_os = "linux"), allow(dead_code))]
|
||
pub own_windows_recorded: Arc<AtomicBool>,
|
||
pub started_at: Instant,
|
||
/// `started_at` on the host clock, which places the movie's start on the frames' clock.
|
||
#[cfg(target_os = "macos")]
|
||
pub started_host: capture_core::HostTime,
|
||
pub stopping: Arc<AtomicBool>,
|
||
pub urgent: Arc<AtomicBool>,
|
||
pub frames: Arc<Mutex<CandidateBuffer>>,
|
||
/// Why the screen source gave up before the stop (Linux: the portal's picker was cancelled
|
||
/// or the stream never started), so the host can end the capture.
|
||
#[cfg_attr(not(target_os = "linux"), allow(dead_code))]
|
||
pub failure: Arc<Mutex<Option<String>>>,
|
||
/// Where the Windows recorder stores when its movie started (ms since `started_at`).
|
||
#[cfg(target_os = "windows")]
|
||
pub video_offset_ms: Arc<AtomicU64>,
|
||
}
|
||
|
||
/// Active screen + optional MP4 recorder (ScreenCaptureKit on macOS, Windows.Graphics.Capture
|
||
/// and Media Foundation on Windows).
|
||
pub struct ScreenCaptureSession {
|
||
started_at: Instant,
|
||
stopping: Arc<AtomicBool>,
|
||
urgent_snapshot: Arc<AtomicBool>,
|
||
frames: Arc<Mutex<CandidateBuffer>>,
|
||
failure: Arc<Mutex<Option<String>>>,
|
||
sampler_thread: Option<JoinHandle<Option<Movie>>>,
|
||
_recording: crate::retention::Recording,
|
||
}
|
||
|
||
impl ScreenCaptureSession {
|
||
pub fn start(
|
||
target: TargetInfo,
|
||
video_output_path: Option<PathBuf>,
|
||
excluded_window_ids: Arc<Mutex<Vec<u32>>>,
|
||
) -> Self {
|
||
Self::start_with_budget(
|
||
target,
|
||
video_output_path,
|
||
excluded_window_ids,
|
||
Arc::default(),
|
||
CANDIDATE_BUDGET_BYTES,
|
||
MovieQuality::Compact,
|
||
PortalToken::None,
|
||
)
|
||
}
|
||
|
||
pub fn start_with_budget(
|
||
target: TargetInfo,
|
||
video_output_path: Option<PathBuf>,
|
||
excluded_window_ids: Arc<Mutex<Vec<u32>>>,
|
||
own_windows_recorded: Arc<AtomicBool>,
|
||
budget_bytes: usize,
|
||
movie_quality: MovieQuality,
|
||
portal_token: PortalToken,
|
||
) -> Self {
|
||
let started_at = Instant::now();
|
||
#[cfg(target_os = "macos")]
|
||
let started_host = capture_core::HostTime::now();
|
||
let stopping = Arc::new(AtomicBool::new(false));
|
||
let urgent_snapshot = Arc::new(AtomicBool::new(true));
|
||
let frames = Arc::new(Mutex::new(CandidateBuffer::new(budget_bytes)));
|
||
|
||
let thread_stopping = Arc::clone(&stopping);
|
||
let thread_urgent = Arc::clone(&urgent_snapshot);
|
||
let thread_frames = Arc::clone(&frames);
|
||
let failure = Arc::new(Mutex::new(None));
|
||
let thread_failure = Arc::clone(&failure);
|
||
let sampler_target = target;
|
||
let sampler_thread = thread::Builder::new()
|
||
.name("scrawl-frame-sampler".into())
|
||
.spawn(move || {
|
||
platform::run_sampler(Sampler {
|
||
target: sampler_target,
|
||
video_output_path,
|
||
movie_quality,
|
||
portal_token,
|
||
excluded_window_ids,
|
||
own_windows_recorded,
|
||
started_at,
|
||
#[cfg(target_os = "macos")]
|
||
started_host,
|
||
stopping: thread_stopping,
|
||
urgent: thread_urgent,
|
||
frames: thread_frames,
|
||
failure: thread_failure,
|
||
#[cfg(target_os = "windows")]
|
||
video_offset_ms: Arc::new(AtomicU64::new(0)),
|
||
})
|
||
})
|
||
.ok();
|
||
|
||
Self {
|
||
started_at,
|
||
stopping,
|
||
urgent_snapshot,
|
||
frames,
|
||
failure,
|
||
sampler_thread,
|
||
_recording: crate::retention::Recording::start(),
|
||
}
|
||
}
|
||
|
||
/// Why the screen source gave up before the stop, if it did: nothing more of the screen
|
||
/// will be recorded.
|
||
pub fn failure(&self) -> Option<String> {
|
||
self.failure
|
||
.lock()
|
||
.unwrap_or_else(std::sync::PoisonError::into_inner)
|
||
.clone()
|
||
}
|
||
|
||
pub fn request_immediate_frame(&self) {
|
||
self.urgent_snapshot.store(true, Ordering::Release);
|
||
if let Some(ref handle) = self.sampler_thread {
|
||
handle.thread().unpark();
|
||
}
|
||
}
|
||
|
||
pub fn elapsed_ms(&self) -> u64 {
|
||
self.started_at.elapsed().as_millis() as u64
|
||
}
|
||
|
||
/// Stops at once: the movie keeps finishing on its own (see [`Movie`]).
|
||
pub fn stop(mut self) -> ScreenRecording {
|
||
let duration_ms = self.elapsed_ms();
|
||
self.stopping.store(true, Ordering::Release);
|
||
let video = self
|
||
.sampler_thread
|
||
.take()
|
||
.and_then(|handle| {
|
||
handle.thread().unpark();
|
||
handle.join().ok()
|
||
})
|
||
.flatten();
|
||
let buffer = std::mem::replace(
|
||
&mut *self
|
||
.frames
|
||
.lock()
|
||
.unwrap_or_else(std::sync::PoisonError::into_inner),
|
||
CandidateBuffer::new(0),
|
||
);
|
||
let frames = buffer.into_frames();
|
||
// Never a made-up frame: the capture goes with its voice and marks alone, and says why.
|
||
let no_screen = frames.is_empty().then(|| no_screen_reason(self.failure()));
|
||
ScreenRecording {
|
||
frames,
|
||
video,
|
||
duration_ms: duration_ms.max(100),
|
||
no_screen,
|
||
}
|
||
}
|
||
}
|
||
|
||
impl Drop for ScreenCaptureSession {
|
||
fn drop(&mut self) {
|
||
self.stopping.store(true, Ordering::Release);
|
||
if let Some(handle) = self.sampler_thread.take() {
|
||
handle.thread().unpark();
|
||
let _ = handle.join();
|
||
}
|
||
}
|
||
}
|
||
|
||
/// Shared state collected by the global input monitor during an active capture session.
|
||
/// Monotonic boot-time nanoseconds (`mach_absolute_time` scaled by `mach_timebase_info` on macOS,
|
||
/// matching CoreAudio and CGEvent timestamps).
|
||
pub fn boot_time_ns() -> u64 {
|
||
#[cfg(target_os = "macos")]
|
||
{
|
||
#[repr(C)]
|
||
struct MachTimebaseInfo {
|
||
numer: u32,
|
||
denom: u32,
|
||
}
|
||
unsafe extern "C" {
|
||
fn mach_absolute_time() -> u64;
|
||
fn mach_timebase_info(info: *mut MachTimebaseInfo) -> i32;
|
||
}
|
||
static TIMEBASE: std::sync::OnceLock<(u32, u32)> = std::sync::OnceLock::new();
|
||
let &(numer, denom) = TIMEBASE.get_or_init(|| {
|
||
let mut info = MachTimebaseInfo { numer: 0, denom: 0 };
|
||
let status = unsafe { mach_timebase_info(&mut info) };
|
||
if status == 0 && info.denom != 0 {
|
||
(info.numer, info.denom)
|
||
} else {
|
||
(1, 1)
|
||
}
|
||
});
|
||
let ticks = unsafe { mach_absolute_time() };
|
||
let nanos = u128::from(ticks) * u128::from(numer) / u128::from(denom);
|
||
u64::try_from(nanos).unwrap_or(u64::MAX)
|
||
}
|
||
#[cfg(not(target_os = "macos"))]
|
||
{
|
||
static EPOCH: std::sync::OnceLock<Instant> = std::sync::OnceLock::new();
|
||
EPOCH.get_or_init(Instant::now).elapsed().as_nanos() as u64
|
||
}
|
||
}
|
||
|
||
#[derive(Clone, Debug)]
|
||
pub struct InkSnapshot {
|
||
pub target: TargetFrame,
|
||
pub strokes: Vec<Stroke>,
|
||
pub live: Option<Stroke>,
|
||
pub laser: Option<Stroke>,
|
||
pub glow: bool,
|
||
}
|
||
|
||
/// A raw input event on the capture's clock, in global top-left screen points.
|
||
#[derive(Clone, Debug, PartialEq)]
|
||
pub enum InputEvent {
|
||
ButtonDown {
|
||
t_ms: u64,
|
||
x: f64,
|
||
y: f64,
|
||
right: bool,
|
||
shift_only: bool,
|
||
},
|
||
ButtonUp {
|
||
t_ms: u64,
|
||
x: f64,
|
||
y: f64,
|
||
right: bool,
|
||
shift_only: bool,
|
||
},
|
||
Scroll {
|
||
t_ms: u64,
|
||
x: f64,
|
||
y: f64,
|
||
dx: f64,
|
||
dy: f64,
|
||
},
|
||
Move {
|
||
t_ms: u64,
|
||
x: f64,
|
||
y: f64,
|
||
shift_only: bool,
|
||
},
|
||
Laser {
|
||
t_ms: u64,
|
||
x: f64,
|
||
y: f64,
|
||
},
|
||
KeyChord {
|
||
t_ms: u64,
|
||
shortcut: String,
|
||
},
|
||
Flag {
|
||
t_ms: u64,
|
||
},
|
||
}
|
||
|
||
/// Whether `shift` is held alone without other modifiers and is not left over from a multi-modifier
|
||
/// chord (such as the capture shortcut `⌥⇧S`) before `shift` was released.
|
||
pub fn pure_shift_state(
|
||
shift: bool,
|
||
other_modifiers: bool,
|
||
await_shift_release: &mut bool,
|
||
) -> bool {
|
||
if !shift {
|
||
*await_shift_release = false;
|
||
} else if other_modifiers {
|
||
*await_shift_release = true;
|
||
}
|
||
shift && !other_modifiers && !*await_shift_release
|
||
}
|
||
|
||
/// Ink and pointer input in global top-left points, ms since capture start.
|
||
#[derive(Clone)]
|
||
pub struct InputSink {
|
||
telemetry: Arc<SharedCaptureTelemetry>,
|
||
}
|
||
|
||
impl Default for InputSink {
|
||
fn default() -> Self {
|
||
let (tx, _rx) = flume::unbounded();
|
||
Self {
|
||
telemetry: SharedCaptureTelemetry::new(tx, ShortcutConfig::ansi_defaults()),
|
||
}
|
||
}
|
||
}
|
||
|
||
impl InputSink {
|
||
pub fn new(telemetry: Arc<SharedCaptureTelemetry>) -> Self {
|
||
Self { telemetry }
|
||
}
|
||
|
||
pub fn telemetry(&self) -> &Arc<SharedCaptureTelemetry> {
|
||
&self.telemetry
|
||
}
|
||
|
||
pub fn send(&self, event: InputEvent) {
|
||
self.telemetry.handle_input(event);
|
||
}
|
||
|
||
pub fn button_down(&self, t_ms: u64, x: f64, y: f64, right: bool, shift_only: bool) {
|
||
self.send(InputEvent::ButtonDown {
|
||
t_ms,
|
||
x,
|
||
y,
|
||
right,
|
||
shift_only,
|
||
});
|
||
}
|
||
|
||
pub fn button_up(&self, t_ms: u64, x: f64, y: f64, right: bool, shift_only: bool) {
|
||
self.send(InputEvent::ButtonUp {
|
||
t_ms,
|
||
x,
|
||
y,
|
||
right,
|
||
shift_only,
|
||
});
|
||
}
|
||
|
||
pub fn pointer_down(&self, t_ms: u64, x: f64, y: f64) {
|
||
let shift_only = self.telemetry.shift_held.load(Ordering::Acquire);
|
||
self.button_down(t_ms, x, y, false, shift_only);
|
||
}
|
||
|
||
pub fn pointer_up(&self, t_ms: u64, x: f64, y: f64, right: bool) {
|
||
let shift_only = self.telemetry.shift_held.load(Ordering::Acquire);
|
||
self.button_up(t_ms, x, y, right, shift_only);
|
||
}
|
||
|
||
pub fn shift_down(&self, t_ms: u64, x: f64, y: f64) {
|
||
self.button_down(t_ms, x, y, false, true);
|
||
}
|
||
|
||
pub fn shift_up(&self) -> bool {
|
||
self.telemetry.shift_up()
|
||
}
|
||
|
||
pub fn scroll(&self, t_ms: u64, x: f64, y: f64, dx: f64, dy: f64) {
|
||
self.send(InputEvent::Scroll { t_ms, x, y, dx, dy });
|
||
}
|
||
|
||
pub fn ink(&self, t_ms: u64, x: f64, y: f64, shift_held: bool) {
|
||
self.send(InputEvent::Move {
|
||
t_ms,
|
||
x,
|
||
y,
|
||
shift_only: shift_held,
|
||
});
|
||
}
|
||
|
||
pub fn laser(&self, t_ms: u64, x: f64, y: f64) {
|
||
self.send(InputEvent::Laser { t_ms, x, y });
|
||
}
|
||
|
||
pub fn key_chord(&self, t_ms: u64, shortcut: impl Into<String>) {
|
||
self.send(InputEvent::KeyChord {
|
||
t_ms,
|
||
shortcut: shortcut.into(),
|
||
});
|
||
}
|
||
|
||
pub fn flag(&self, t_ms: u64) {
|
||
self.send(InputEvent::Flag { t_ms });
|
||
}
|
||
|
||
pub fn snapshot(&self) -> InkSnapshot {
|
||
self.telemetry.ink_snapshot()
|
||
}
|
||
}
|
||
|
||
/// A drag's start: when (ms), where (normalized), its Accessibility label, and the frame.
|
||
type DragStart = (u64, f32, f32, Option<String>, TargetFrame);
|
||
|
||
pub struct SharedCaptureTelemetry {
|
||
pub active: AtomicBool,
|
||
pub selecting: AtomicBool,
|
||
pub shift_held: AtomicBool,
|
||
pub capture_mode: AtomicU64,
|
||
/// The record shortcut (`crate::settings` loads and saves it).
|
||
pub shortcut: Mutex<ShortcutConfig>,
|
||
pub paste_shortcut: Mutex<ShortcutConfig>,
|
||
/// Settings is recording a new shortcut: none of Scrawl's fire (`pause_shortcuts`).
|
||
pub shortcuts_paused: AtomicBool,
|
||
pub started_at: Mutex<Instant>,
|
||
pub started_at_boot_ns: AtomicU64,
|
||
pub picker_started_at: Mutex<Instant>,
|
||
pub picker_rect: Mutex<Option<([f32; 4], u8)>>,
|
||
pub target_frame: Mutex<TargetFrame>,
|
||
pub pointer_screen: Mutex<[f32; 2]>,
|
||
pub events: Mutex<Vec<InteractionEvent>>,
|
||
pub strokes: Mutex<Vec<Stroke>>,
|
||
pub flags: Mutex<Vec<u64>>,
|
||
pub active_laser: Mutex<Option<Stroke>>,
|
||
pub live_ink: Mutex<Option<Stroke>>,
|
||
pub excluded_window_ids: Arc<Mutex<Vec<u32>>>,
|
||
/// Raised when a click, ink stroke or shortcut starts, so the Screen-mode window log takes
|
||
/// a snapshot right away.
|
||
pub window_log_request: Arc<AtomicBool>,
|
||
pub ui_sender: flume::Sender<Event>,
|
||
/// Scrawl's windows that let clicks through (the ink overlay), skipped when deciding whether
|
||
/// a click landed on Scrawl itself.
|
||
pub passthrough_window_ids: Mutex<Vec<u32>>,
|
||
/// The `scrawl-helper` whose window is what's being recorded (the iPhone mirror): its clicks
|
||
/// count, unlike those on other helper windows (Settings, onboarding). 0 when none.
|
||
capture_source_pid: AtomicI32,
|
||
/// The last `on_scrawl_window` answer (when, where, what), reused for a burst of scrolls.
|
||
scrawl_hit: Mutex<Option<(Instant, f64, f64, bool)>>,
|
||
/// macOS's key event tap (`CFMachPortRef`), made once Accessibility is on (`check_key_tap`).
|
||
#[cfg_attr(any(target_os = "windows", target_os = "linux"), allow(dead_code))]
|
||
tap_port: AtomicU64,
|
||
/// The key event tap thread's run loop (retained), so `check_key_tap` can stop it.
|
||
#[cfg_attr(any(target_os = "windows", target_os = "linux"), allow(dead_code))]
|
||
tap_run_loop: AtomicU64,
|
||
/// A key tap thread is being started.
|
||
#[cfg_attr(any(target_os = "windows", target_os = "linux"), allow(dead_code))]
|
||
tap_starting: AtomicBool,
|
||
/// When the last key tap was tried (ms since `UNIX_EPOCH`), to space out retries.
|
||
#[cfg_attr(any(target_os = "windows", target_os = "linux"), allow(dead_code))]
|
||
tap_tried_ms: AtomicU64,
|
||
/// Whether key presses are arriving (macOS's key tap, Windows' keyboard hook).
|
||
tap_live: AtomicBool,
|
||
/// Esc went down since the UI loop last looked (`take_escape`).
|
||
escape_pressed: AtomicBool,
|
||
/// A Screen recording started straight from the shortcut: the overlay flashes an edge glow.
|
||
pub start_glow: AtomicBool,
|
||
/// Clicks are arriving (macOS's global mouse monitor, Windows' mouse hook). Until then the
|
||
/// poller records clicks from polled button state.
|
||
mouse_live: AtomicBool,
|
||
last_shortcut_ms: AtomicU64,
|
||
drag_start: Mutex<Option<DragStart>>,
|
||
/// The `agentContext` experiment was on when this capture started: clicks and strokes are
|
||
/// looked up in the Accessibility tree (`crate::ax`, macOS only).
|
||
agent_context: AtomicBool,
|
||
/// A ⇧-press inside the target: when, where (global points) and on which frame.
|
||
shift_press: Mutex<Option<(u64, f64, f64, TargetFrame)>>,
|
||
/// Monotonic session counter incremented each time a capture starts on this telemetry.
|
||
session_id: AtomicU64,
|
||
}
|
||
|
||
impl SharedCaptureTelemetry {
|
||
pub fn new(
|
||
ui_sender: flume::Sender<Event>,
|
||
shortcuts: (ShortcutConfig, ShortcutConfig),
|
||
) -> Arc<Self> {
|
||
let (record, paste) = shortcuts;
|
||
Arc::new(Self {
|
||
active: AtomicBool::new(false),
|
||
selecting: AtomicBool::new(false),
|
||
shift_held: AtomicBool::new(false),
|
||
capture_mode: AtomicU64::new(0),
|
||
shortcut: Mutex::new(record),
|
||
paste_shortcut: Mutex::new(paste),
|
||
shortcuts_paused: AtomicBool::new(false),
|
||
started_at: Mutex::new(Instant::now()),
|
||
started_at_boot_ns: AtomicU64::new(boot_time_ns()),
|
||
picker_started_at: Mutex::new(Instant::now()),
|
||
picker_rect: Mutex::new(None),
|
||
target_frame: Mutex::new(TargetFrame {
|
||
x: 0.0,
|
||
y: 0.0,
|
||
width: 1440.0,
|
||
height: 900.0,
|
||
scale: 2.0,
|
||
}),
|
||
pointer_screen: Mutex::new([0.0, 0.0]),
|
||
events: Mutex::new(Vec::new()),
|
||
strokes: Mutex::new(Vec::new()),
|
||
flags: Mutex::new(Vec::new()),
|
||
active_laser: Mutex::new(None),
|
||
live_ink: Mutex::new(None),
|
||
excluded_window_ids: Arc::new(Mutex::new(Vec::new())),
|
||
window_log_request: Arc::new(AtomicBool::new(false)),
|
||
ui_sender,
|
||
passthrough_window_ids: Mutex::new(Vec::new()),
|
||
capture_source_pid: AtomicI32::new(0),
|
||
scrawl_hit: Mutex::new(None),
|
||
tap_port: AtomicU64::new(0),
|
||
tap_run_loop: AtomicU64::new(0),
|
||
tap_starting: AtomicBool::new(false),
|
||
tap_tried_ms: AtomicU64::new(0),
|
||
tap_live: AtomicBool::new(false),
|
||
escape_pressed: AtomicBool::new(false),
|
||
start_glow: AtomicBool::new(false),
|
||
mouse_live: AtomicBool::new(false),
|
||
last_shortcut_ms: AtomicU64::new(0),
|
||
drag_start: Mutex::new(None),
|
||
agent_context: AtomicBool::new(false),
|
||
shift_press: Mutex::new(None),
|
||
session_id: AtomicU64::new(0),
|
||
})
|
||
}
|
||
|
||
/// Each source (Carbon or `RegisterHotKey`, the key tap or hook) reports the same press within
|
||
/// a few ms; a press after that is a new one, as when S is pressed again to switch modes.
|
||
pub fn emit_capture_shortcut(&self) {
|
||
if self.shortcuts_paused.load(Ordering::Acquire) {
|
||
return;
|
||
}
|
||
let now_ms = std::time::SystemTime::now()
|
||
.duration_since(std::time::UNIX_EPOCH)
|
||
.unwrap_or_default()
|
||
.as_millis() as u64;
|
||
let prev = self.last_shortcut_ms.load(Ordering::Acquire);
|
||
if now_ms.saturating_sub(prev) >= 90
|
||
&& self
|
||
.last_shortcut_ms
|
||
.compare_exchange(prev, now_ms, Ordering::AcqRel, Ordering::Acquire)
|
||
.is_ok()
|
||
{
|
||
scrawl_log("emit_capture_shortcut");
|
||
let _ = self.ui_sender.try_send(Event::CaptureShortcut);
|
||
}
|
||
}
|
||
|
||
pub fn emit_paste_shortcut(&self) {
|
||
if self.shortcuts_paused.load(Ordering::Acquire) {
|
||
return;
|
||
}
|
||
let now_ms = std::time::SystemTime::now()
|
||
.duration_since(std::time::UNIX_EPOCH)
|
||
.unwrap_or_default()
|
||
.as_millis() as u64;
|
||
let prev = self.last_shortcut_ms.load(Ordering::Acquire);
|
||
if now_ms.saturating_sub(prev) >= 400
|
||
&& self
|
||
.last_shortcut_ms
|
||
.compare_exchange(prev, now_ms, Ordering::AcqRel, Ordering::Acquire)
|
||
.is_ok()
|
||
{
|
||
scrawl_log("emit_paste_shortcut -> sending PasteLatestShortcut");
|
||
let _ = self.ui_sender.try_send(Event::PasteLatestShortcut);
|
||
}
|
||
}
|
||
|
||
pub fn matches_shortcut(
|
||
&self,
|
||
keycode: u16,
|
||
cmd: bool,
|
||
ctrl: bool,
|
||
alt: bool,
|
||
shift: bool,
|
||
) -> bool {
|
||
let cfg = self
|
||
.shortcut
|
||
.lock()
|
||
.unwrap_or_else(std::sync::PoisonError::into_inner)
|
||
.clone();
|
||
is_capture_shortcut(&cfg, layout_s_keycode(), keycode, cmd, ctrl, alt, shift)
|
||
}
|
||
|
||
/// The record and paste shortcuts.
|
||
pub fn shortcuts(&self) -> (ShortcutConfig, ShortcutConfig) {
|
||
let lock = |m: &Mutex<ShortcutConfig>| {
|
||
m.lock()
|
||
.unwrap_or_else(std::sync::PoisonError::into_inner)
|
||
.clone()
|
||
};
|
||
(lock(&self.shortcut), lock(&self.paste_shortcut))
|
||
}
|
||
|
||
pub fn set_capture_mode(&self, mode: CaptureMode) {
|
||
let code = match mode {
|
||
CaptureMode::Display => 0,
|
||
CaptureMode::Region => 1,
|
||
CaptureMode::Window => 2,
|
||
CaptureMode::Device => 3,
|
||
};
|
||
self.capture_mode.store(code, Ordering::Release);
|
||
}
|
||
|
||
pub fn begin_picker_session(&self) {
|
||
*self
|
||
.picker_started_at
|
||
.lock()
|
||
.unwrap_or_else(std::sync::PoisonError::into_inner) = Instant::now();
|
||
*self
|
||
.picker_rect
|
||
.lock()
|
||
.unwrap_or_else(std::sync::PoisonError::into_inner) = None;
|
||
self.active.store(false, Ordering::Release);
|
||
self.selecting.store(true, Ordering::Release);
|
||
}
|
||
|
||
pub fn set_picker_rect(&self, rect: Option<([f32; 4], u8)>) {
|
||
*self
|
||
.picker_rect
|
||
.lock()
|
||
.unwrap_or_else(std::sync::PoisonError::into_inner) = rect;
|
||
}
|
||
|
||
pub fn started_at_boot_ns(&self) -> u64 {
|
||
self.started_at_boot_ns.load(Ordering::Acquire)
|
||
}
|
||
|
||
pub fn reset_for_session(&self, frame: TargetFrame) {
|
||
self.reset_for_session_at(frame, None);
|
||
}
|
||
|
||
pub fn reset_for_session_at(&self, frame: TargetFrame, t0_boot_ns: Option<u64>) {
|
||
let agent_context = crate::handoff::experiment(crate::handoff::AGENT_CONTEXT);
|
||
let _ = self.reset_for_session_at_with_agent_context(frame, t0_boot_ns, agent_context);
|
||
}
|
||
|
||
pub fn reset_for_session_at_with_agent_context(
|
||
&self,
|
||
frame: TargetFrame,
|
||
t0_boot_ns: Option<u64>,
|
||
agent_context: bool,
|
||
) -> u64 {
|
||
let session_id = self.session_id.fetch_add(1, Ordering::AcqRel) + 1;
|
||
self.selecting.store(false, Ordering::Release);
|
||
*self
|
||
.picker_rect
|
||
.lock()
|
||
.unwrap_or_else(std::sync::PoisonError::into_inner) = None;
|
||
let now_boot_ns = boot_time_ns();
|
||
let boot_ns = t0_boot_ns.unwrap_or(now_boot_ns);
|
||
self.started_at_boot_ns.store(boot_ns, Ordering::Release);
|
||
let started_instant = Instant::now()
|
||
.checked_sub(Duration::from_nanos(now_boot_ns.saturating_sub(boot_ns)))
|
||
.unwrap_or_else(Instant::now);
|
||
*self
|
||
.started_at
|
||
.lock()
|
||
.unwrap_or_else(std::sync::PoisonError::into_inner) = started_instant;
|
||
*self
|
||
.target_frame
|
||
.lock()
|
||
.unwrap_or_else(std::sync::PoisonError::into_inner) = frame;
|
||
self.events
|
||
.lock()
|
||
.unwrap_or_else(std::sync::PoisonError::into_inner)
|
||
.clear();
|
||
self.strokes
|
||
.lock()
|
||
.unwrap_or_else(std::sync::PoisonError::into_inner)
|
||
.clear();
|
||
self.flags
|
||
.lock()
|
||
.unwrap_or_else(std::sync::PoisonError::into_inner)
|
||
.clear();
|
||
*self
|
||
.active_laser
|
||
.lock()
|
||
.unwrap_or_else(std::sync::PoisonError::into_inner) = None;
|
||
*self
|
||
.live_ink
|
||
.lock()
|
||
.unwrap_or_else(std::sync::PoisonError::into_inner) = None;
|
||
*self
|
||
.drag_start
|
||
.lock()
|
||
.unwrap_or_else(std::sync::PoisonError::into_inner) = None;
|
||
*self
|
||
.shift_press
|
||
.lock()
|
||
.unwrap_or_else(std::sync::PoisonError::into_inner) = None;
|
||
self.excluded_window_ids
|
||
.lock()
|
||
.unwrap_or_else(std::sync::PoisonError::into_inner)
|
||
.clear();
|
||
self.shift_held.store(false, Ordering::Release);
|
||
self.window_log_request.store(false, Ordering::Release);
|
||
self.agent_context.store(agent_context, Ordering::Release);
|
||
if agent_context {
|
||
crate::ax::Prober::shared().begin();
|
||
}
|
||
self.active.store(true, Ordering::Release);
|
||
let clicks = if self.mouse_live.load(Ordering::Acquire) {
|
||
platform::MOUSE_SOURCE
|
||
} else {
|
||
"polled mouse buttons"
|
||
};
|
||
let keys = if self.tap_live.load(Ordering::Acquire) {
|
||
platform::KEY_SOURCE
|
||
} else {
|
||
platform::NO_KEY_SOURCE
|
||
};
|
||
scrawl_log(format!("session clicks from {clicks}, keys from {keys}"));
|
||
session_id
|
||
}
|
||
|
||
/// Whether Esc went down since the last call (a pending shortcut or an open picker cancels on it).
|
||
pub fn take_escape(&self) -> bool {
|
||
self.escape_pressed.swap(false, Ordering::AcqRel)
|
||
}
|
||
|
||
pub fn elapsed_ms(&self) -> u64 {
|
||
self.started_at
|
||
.lock()
|
||
.unwrap_or_else(std::sync::PoisonError::into_inner)
|
||
.elapsed()
|
||
.as_millis() as u64
|
||
}
|
||
|
||
pub fn contains_screen_point(&self, screen_x: f64, screen_y: f64) -> bool {
|
||
let frame = *self
|
||
.target_frame
|
||
.lock()
|
||
.unwrap_or_else(std::sync::PoisonError::into_inner);
|
||
frame.width >= 16.0
|
||
&& frame.height >= 16.0
|
||
&& frame.x > -10_000.0
|
||
&& screen_x >= frame.x
|
||
&& screen_x <= frame.x + frame.width
|
||
&& screen_y >= frame.y
|
||
&& screen_y <= frame.y + frame.height
|
||
}
|
||
|
||
/// Whether to look up what clicks and strokes land on: `agentContext` is on, the target is
|
||
/// this Mac's screen (not the iPhone mirror) and Accessibility is granted.
|
||
fn probing(&self) -> bool {
|
||
self.agent_context.load(Ordering::Acquire)
|
||
&& self.capture_mode.load(Ordering::Acquire) != 3
|
||
&& accessibility_granted()
|
||
}
|
||
|
||
/// A mouse button went down at a global point.
|
||
pub fn pointer_down(&self, t_ms: u64, screen_x: f64, screen_y: f64) {
|
||
if !self.contains_screen_point(screen_x, screen_y) {
|
||
return;
|
||
}
|
||
#[cfg(target_os = "macos")]
|
||
{
|
||
let raw = objc2::rc::autoreleasepool(|_| crate::windows::raw_windows());
|
||
if crate::windows::system_prompt_at(&raw, screen_x, screen_y) {
|
||
return;
|
||
}
|
||
}
|
||
let (nx, ny, _, _) = self.normalize_point(screen_x, screen_y);
|
||
self.window_log_request.store(true, Ordering::Release);
|
||
let start_frame = *self
|
||
.target_frame
|
||
.lock()
|
||
.unwrap_or_else(std::sync::PoisonError::into_inner);
|
||
*self
|
||
.drag_start
|
||
.lock()
|
||
.unwrap_or_else(std::sync::PoisonError::into_inner) =
|
||
Some((t_ms, nx, ny, None, start_frame));
|
||
if self.probing() {
|
||
crate::ax::Prober::shared().probe(crate::ax::Target::Pointer(t_ms), screen_x, screen_y);
|
||
}
|
||
}
|
||
|
||
/// A mouse button came up: records the click or drag that `pointer_down` began.
|
||
pub fn pointer_up(&self, t_ms: u64, screen_x: f64, screen_y: f64, right: bool) {
|
||
let Some((t0, x0, y0, start_label, start_frame)) = self
|
||
.drag_start
|
||
.lock()
|
||
.unwrap_or_else(std::sync::PoisonError::into_inner)
|
||
.take()
|
||
else {
|
||
return;
|
||
};
|
||
let end_frame = *self
|
||
.target_frame
|
||
.lock()
|
||
.unwrap_or_else(std::sync::PoisonError::into_inner);
|
||
if !same_target_frame(start_frame, end_frame) {
|
||
return;
|
||
}
|
||
let (nx, ny, fw, fh) = self.normalize_point(screen_x, screen_y);
|
||
let dist_px = ((nx - x0) * fw as f32).hypot((ny - y0) * fh as f32);
|
||
let _ = self.ui_sender.try_send(Event::RequestSnapshot);
|
||
let event = if dist_px < CLICK_SLOP_PT {
|
||
InteractionEvent::Click {
|
||
t: t0,
|
||
x: x0,
|
||
y: y0,
|
||
button: Some(if right { "right" } else { "left" }.into()),
|
||
label: start_label,
|
||
role: None,
|
||
}
|
||
} else {
|
||
InteractionEvent::Drag {
|
||
t: t0,
|
||
t1: t_ms,
|
||
x0,
|
||
y0,
|
||
x1: nx,
|
||
y1: ny,
|
||
label: start_label,
|
||
button: None,
|
||
path: Vec::new(),
|
||
}
|
||
};
|
||
self.events
|
||
.lock()
|
||
.unwrap_or_else(std::sync::PoisonError::into_inner)
|
||
.push(event);
|
||
}
|
||
|
||
/// The left button went down with only ⇧ held: drawing, never a click (`shift_up`).
|
||
pub fn shift_down(&self, t_ms: u64, screen_x: f64, screen_y: f64) {
|
||
if !self.contains_screen_point(screen_x, screen_y) {
|
||
return;
|
||
}
|
||
let frame = *self
|
||
.target_frame
|
||
.lock()
|
||
.unwrap_or_else(std::sync::PoisonError::into_inner);
|
||
*self
|
||
.shift_press
|
||
.lock()
|
||
.unwrap_or_else(std::sync::PoisonError::into_inner) =
|
||
Some((t_ms, screen_x, screen_y, frame));
|
||
}
|
||
|
||
/// The button of a ⇧-press came up (with or without ⇧ still held). A ⇧-press is part of
|
||
/// the drawing, never a click: returns whether one was waiting, so the release isn't taken
|
||
/// as a click's.
|
||
pub fn shift_up(&self) -> bool {
|
||
self.shift_press
|
||
.lock()
|
||
.unwrap_or_else(std::sync::PoisonError::into_inner)
|
||
.take()
|
||
.is_some()
|
||
}
|
||
|
||
/// The helper process whose window is the capture target (the iPhone mirror).
|
||
#[cfg_attr(any(target_os = "windows", target_os = "linux"), allow(dead_code))]
|
||
pub fn set_capture_source_pid(&self, pid: u32) {
|
||
self.capture_source_pid
|
||
.store(i32::try_from(pid).unwrap_or(0), Ordering::Release);
|
||
}
|
||
|
||
/// Whether the topmost window at a global point is Scrawl's own or a `scrawl-helper`'s other
|
||
/// than the capture source (ignoring the click-through ink overlay). Asked once per press, and
|
||
/// for scrolls, which come in bursts at one point, at most every 250 ms.
|
||
fn on_scrawl_window(&self, x: f64, y: f64) -> bool {
|
||
let mut hit = self
|
||
.scrawl_hit
|
||
.lock()
|
||
.unwrap_or_else(std::sync::PoisonError::into_inner);
|
||
if let Some((at, hx, hy, on)) = *hit
|
||
&& at.elapsed() < Duration::from_millis(250)
|
||
&& (hx - x).abs() <= 2.0
|
||
&& (hy - y).abs() <= 2.0
|
||
{
|
||
return on;
|
||
}
|
||
let on = self.scrawl_window_in(
|
||
&platform::raw_windows_now(),
|
||
x,
|
||
y,
|
||
platform::is_helper_process,
|
||
);
|
||
*hit = Some((Instant::now(), x, y, on));
|
||
on
|
||
}
|
||
|
||
fn scrawl_window_in(
|
||
&self,
|
||
windows: &[crate::windows::RawWindow],
|
||
x: f64,
|
||
y: f64,
|
||
is_helper: impl Fn(i32) -> bool,
|
||
) -> bool {
|
||
let passthrough = self
|
||
.passthrough_window_ids
|
||
.lock()
|
||
.unwrap_or_else(std::sync::PoisonError::into_inner)
|
||
.clone();
|
||
let excluded = self
|
||
.excluded_window_ids
|
||
.lock()
|
||
.unwrap_or_else(std::sync::PoisonError::into_inner)
|
||
.clone();
|
||
let source = self.capture_source_pid.load(Ordering::Acquire);
|
||
on_own_window(windows, (x, y), &passthrough, &excluded, source, is_helper)
|
||
}
|
||
|
||
pub fn normalize_point(&self, screen_x: f64, screen_y: f64) -> (f32, f32, u32, u32) {
|
||
let frame = *self
|
||
.target_frame
|
||
.lock()
|
||
.unwrap_or_else(std::sync::PoisonError::into_inner);
|
||
let nx = ((screen_x - frame.x) / frame.width.max(1.0)).clamp(0.0, 1.0) as f32;
|
||
let ny = ((screen_y - frame.y) / frame.height.max(1.0)).clamp(0.0, 1.0) as f32;
|
||
(
|
||
nx,
|
||
ny,
|
||
frame.width.round().max(1.0) as u32,
|
||
frame.height.round().max(1.0) as u32,
|
||
)
|
||
}
|
||
|
||
pub fn finish_active_laser(&self, _ax_point: Option<(f64, f64)>) {
|
||
if let Some(mut laser) = self
|
||
.active_laser
|
||
.lock()
|
||
.unwrap_or_else(std::sync::PoisonError::into_inner)
|
||
.take()
|
||
&& !laser.pts.is_empty()
|
||
{
|
||
if let Some(last) = laser.pts.as_chunks::<3>().0.last() {
|
||
let lx = last[0] as f32 / SCALE as f32;
|
||
let ly = last[1] as f32 / SCALE as f32;
|
||
laser.t1 = laser.t0.saturating_add(last[2].max(0) as u64);
|
||
self.events
|
||
.lock()
|
||
.unwrap_or_else(std::sync::PoisonError::into_inner)
|
||
.push(InteractionEvent::Point {
|
||
t: laser.t1,
|
||
x: lx,
|
||
y: ly,
|
||
label: None,
|
||
});
|
||
}
|
||
self.strokes
|
||
.lock()
|
||
.unwrap_or_else(std::sync::PoisonError::into_inner)
|
||
.push(laser);
|
||
}
|
||
}
|
||
|
||
pub fn finish_session(&self) -> (Vec<InteractionEvent>, Vec<Stroke>) {
|
||
self.active.store(false, Ordering::Release);
|
||
self.selecting.store(false, Ordering::Release);
|
||
*self
|
||
.drag_start
|
||
.lock()
|
||
.unwrap_or_else(std::sync::PoisonError::into_inner) = None;
|
||
*self
|
||
.shift_press
|
||
.lock()
|
||
.unwrap_or_else(std::sync::PoisonError::into_inner) = None;
|
||
self.finish_active_laser(None);
|
||
if let Some(ink) = self
|
||
.live_ink
|
||
.lock()
|
||
.unwrap_or_else(std::sync::PoisonError::into_inner)
|
||
.take()
|
||
&& ink.pts.len() >= 3
|
||
{
|
||
self.strokes
|
||
.lock()
|
||
.unwrap_or_else(std::sync::PoisonError::into_inner)
|
||
.push(ink);
|
||
}
|
||
let mut events = std::mem::take(
|
||
&mut *self
|
||
.events
|
||
.lock()
|
||
.unwrap_or_else(std::sync::PoisonError::into_inner),
|
||
);
|
||
let mut strokes = std::mem::take(
|
||
&mut *self
|
||
.strokes
|
||
.lock()
|
||
.unwrap_or_else(std::sync::PoisonError::into_inner),
|
||
);
|
||
if self.agent_context.swap(false, Ordering::AcqRel) {
|
||
let hits = crate::ax::Prober::shared().finish();
|
||
crate::ax::apply(&hits, &mut events, &mut strokes);
|
||
if self.capture_mode.load(Ordering::Acquire) != 3 {
|
||
let end = crate::pipeline::now_epoch_ms();
|
||
let start = end.saturating_sub(self.elapsed_ms());
|
||
let page = crate::page::between(start, end);
|
||
crate::page::apply(&page, start, &mut events, &mut strokes);
|
||
}
|
||
}
|
||
(events, strokes)
|
||
}
|
||
|
||
pub fn handle_input(&self, event: InputEvent) {
|
||
self.handle_input_with(event, |x, y| self.on_scrawl_window(x, y));
|
||
}
|
||
|
||
pub fn handle_input_with(&self, event: InputEvent, on_own_window: impl Fn(f64, f64) -> bool) {
|
||
match event {
|
||
// A ⇧-press is never a click: ⇧ is drawing.
|
||
InputEvent::ButtonDown {
|
||
t_ms,
|
||
x,
|
||
y,
|
||
right,
|
||
shift_only,
|
||
} => {
|
||
if !on_own_window(x, y) {
|
||
if !shift_only {
|
||
self.pointer_down(t_ms, x, y);
|
||
} else if !right {
|
||
self.shift_down(t_ms, x, y);
|
||
}
|
||
}
|
||
}
|
||
InputEvent::ButtonUp {
|
||
t_ms,
|
||
x,
|
||
y,
|
||
right,
|
||
shift_only,
|
||
} => {
|
||
let shift_press = !right && self.shift_up();
|
||
if !shift_press && !shift_only {
|
||
self.pointer_up(t_ms, x, y, right);
|
||
}
|
||
}
|
||
InputEvent::Scroll { t_ms, x, y, dx, dy } => {
|
||
if (dx.abs() > 0.0 || dy.abs() > 0.0) && !on_own_window(x, y) {
|
||
self.scroll_at(t_ms, x, y, dx as f32, dy as f32);
|
||
}
|
||
}
|
||
InputEvent::Move {
|
||
t_ms,
|
||
x,
|
||
y,
|
||
shift_only,
|
||
} => {
|
||
*self
|
||
.pointer_screen
|
||
.lock()
|
||
.unwrap_or_else(std::sync::PoisonError::into_inner) = [x as f32, y as f32];
|
||
self.track_ink_at(t_ms, (x, y), shift_only);
|
||
}
|
||
InputEvent::Laser { t_ms, x, y } => {
|
||
self.laser_at(t_ms, x, y);
|
||
}
|
||
InputEvent::KeyChord { t_ms, shortcut } => {
|
||
self.key_chord(t_ms, shortcut);
|
||
}
|
||
InputEvent::Flag { t_ms } => {
|
||
self.flag_at(t_ms);
|
||
}
|
||
}
|
||
}
|
||
|
||
pub fn key_chord(&self, t_ms: u64, shortcut: String) {
|
||
self.window_log_request.store(true, Ordering::Release);
|
||
self.events
|
||
.lock()
|
||
.unwrap_or_else(std::sync::PoisonError::into_inner)
|
||
.push(InteractionEvent::Key { t: t_ms, shortcut });
|
||
}
|
||
|
||
pub fn scroll_at(&self, t_ms: u64, screen_x: f64, screen_y: f64, dx: f32, dy: f32) {
|
||
if (dx.abs() > 0.0 || dy.abs() > 0.0) && self.contains_screen_point(screen_x, screen_y) {
|
||
let (nx, ny, _, _) = self.normalize_point(screen_x, screen_y);
|
||
self.events
|
||
.lock()
|
||
.unwrap_or_else(std::sync::PoisonError::into_inner)
|
||
.push(InteractionEvent::Scroll {
|
||
t: t_ms,
|
||
x: nx,
|
||
y: ny,
|
||
dx,
|
||
dy,
|
||
label: None,
|
||
});
|
||
}
|
||
}
|
||
|
||
pub fn track_ink_at(&self, t_ms: u64, (x, y): (f64, f64), shift_held: bool) {
|
||
let prev = self.shift_held.load(Ordering::Acquire)
|
||
|| self
|
||
.live_ink
|
||
.lock()
|
||
.unwrap_or_else(std::sync::PoisonError::into_inner)
|
||
.is_some();
|
||
track_shift_ink_at(self, t_ms, (x, y), shift_held, prev);
|
||
}
|
||
|
||
pub fn laser_at(&self, t_ms: u64, screen_x: f64, screen_y: f64) {
|
||
if !self.contains_screen_point(screen_x, screen_y) {
|
||
return;
|
||
}
|
||
let (nx, ny, fw, fh) = self.normalize_point(screen_x, screen_y);
|
||
let sx = (nx * SCALE as f32).round() as i32;
|
||
let sy = (ny * SCALE as f32).round() as i32;
|
||
let mut guard = self
|
||
.active_laser
|
||
.lock()
|
||
.unwrap_or_else(std::sync::PoisonError::into_inner);
|
||
let stroke = guard.get_or_insert_with(|| Stroke {
|
||
t0: t_ms,
|
||
t1: t_ms,
|
||
fw,
|
||
fh,
|
||
on: None,
|
||
pin: None,
|
||
laser: true,
|
||
pts: Vec::new(),
|
||
});
|
||
let dt = t_ms.saturating_sub(stroke.t0) as i32;
|
||
if stroke.pts.len() / 3 < 2_400 {
|
||
stroke.pts.extend_from_slice(&[sx, sy, dt]);
|
||
stroke.t1 = t_ms;
|
||
}
|
||
}
|
||
|
||
pub fn flag_at(&self, t_ms: u64) {
|
||
self.flags
|
||
.lock()
|
||
.unwrap_or_else(std::sync::PoisonError::into_inner)
|
||
.push(t_ms);
|
||
let _ = self.ui_sender.try_send(Event::RequestSnapshot);
|
||
}
|
||
|
||
pub fn take_flags(&self) -> Vec<u64> {
|
||
std::mem::take(
|
||
&mut *self
|
||
.flags
|
||
.lock()
|
||
.unwrap_or_else(std::sync::PoisonError::into_inner),
|
||
)
|
||
}
|
||
|
||
pub fn ink_snapshot(&self) -> InkSnapshot {
|
||
let target = *self
|
||
.target_frame
|
||
.lock()
|
||
.unwrap_or_else(std::sync::PoisonError::into_inner);
|
||
let strokes = self
|
||
.strokes
|
||
.lock()
|
||
.unwrap_or_else(std::sync::PoisonError::into_inner)
|
||
.clone();
|
||
let live = self
|
||
.live_ink
|
||
.lock()
|
||
.unwrap_or_else(std::sync::PoisonError::into_inner)
|
||
.clone();
|
||
let laser = self
|
||
.active_laser
|
||
.lock()
|
||
.unwrap_or_else(std::sync::PoisonError::into_inner)
|
||
.clone();
|
||
let glow = self.start_glow.load(Ordering::Acquire);
|
||
InkSnapshot {
|
||
target,
|
||
strokes,
|
||
live,
|
||
laser,
|
||
glow,
|
||
}
|
||
}
|
||
}
|
||
|
||
/// Clicks from the mouse buttons' polled state, while the mouse monitor (or hook) is missing: a
|
||
/// press starts a click (unless it lands on Scrawl's own window), a release ends it. A ⇧-press may
|
||
/// be a pin, as in the mouse monitor's handler.
|
||
#[cfg_attr(target_os = "linux", allow(dead_code))]
|
||
fn poll_clicks(
|
||
telemetry: &SharedCaptureTelemetry,
|
||
prev: &mut [bool; 2],
|
||
now: [bool; 2],
|
||
(x, y): (f64, f64),
|
||
pure_shift: bool,
|
||
on_scrawl: impl Fn(f64, f64) -> bool,
|
||
) {
|
||
let was = std::mem::replace(prev, now);
|
||
if telemetry.mouse_live.load(Ordering::Acquire) {
|
||
return;
|
||
}
|
||
let t_ms = telemetry.elapsed_ms();
|
||
for (button, (&down, &before)) in now.iter().zip(&was).enumerate() {
|
||
let right = button != 0;
|
||
if down && !before {
|
||
telemetry.handle_input_with(
|
||
InputEvent::ButtonDown {
|
||
t_ms,
|
||
x,
|
||
y,
|
||
right,
|
||
shift_only: pure_shift,
|
||
},
|
||
&on_scrawl,
|
||
);
|
||
} else if !down && before {
|
||
telemetry.handle_input_with(
|
||
InputEvent::ButtonUp {
|
||
t_ms,
|
||
x,
|
||
y,
|
||
right,
|
||
shift_only: pure_shift,
|
||
},
|
||
&on_scrawl,
|
||
);
|
||
}
|
||
}
|
||
}
|
||
|
||
/// Whether the topmost window under global point `at` (in `windows`, front to back, looking
|
||
/// through `passthrough` ones) is the host's own: this process's chrome
|
||
/// ([`is_own_chrome`](crate::windows::is_own_chrome), with the host's `excluded` windows), or a
|
||
/// helper's other than the app being captured (`source`). Clicks and scrolls there aren't part
|
||
/// of what is being shown; on the host's normal windows (Hex's Settings) they are.
|
||
pub fn on_own_window(
|
||
windows: &[crate::windows::RawWindow],
|
||
(x, y): (f64, f64),
|
||
passthrough: &[u32],
|
||
excluded: &[u32],
|
||
source: i32,
|
||
is_helper: impl Fn(i32) -> bool,
|
||
) -> bool {
|
||
let own = std::process::id() as i32;
|
||
crate::windows::owned_window_at(windows, x, y, passthrough, |w| {
|
||
crate::windows::is_own_chrome(w, own, excluded) || (w.pid != source && is_helper(w.pid))
|
||
})
|
||
}
|
||
|
||
/// A press that moves less than this (points) before it's let go is a click, not a drag.
|
||
pub const CLICK_SLOP_PT: f32 = 12.0;
|
||
/// Ink spanning less than this (points) is a stationary ⇧ tap, not a stroke.
|
||
pub const MIN_INK_SPAN_PT: f32 = 6.0;
|
||
|
||
/// Whether `stroke`'s points before `before_ms` reach at least `MIN_INK_SPAN_PT` from its start.
|
||
pub fn drew_before(stroke: &Stroke, before_ms: u64) -> bool {
|
||
let Some(first) = stroke.pts.get(0..2) else {
|
||
return false;
|
||
};
|
||
let mut points = 0_usize;
|
||
let mut span = 0.0_f32;
|
||
for p in stroke
|
||
.pts
|
||
.as_chunks::<3>()
|
||
.0
|
||
.iter()
|
||
.take_while(|p| stroke.t0.saturating_add(p[2].max(0) as u64) < before_ms)
|
||
{
|
||
points += 1;
|
||
let dx = (p[0] - first[0]) as f32 / SCALE as f32 * stroke.fw as f32;
|
||
let dy = (p[1] - first[1]) as f32 / SCALE as f32 * stroke.fh as f32;
|
||
span = span.max(dx.hypot(dy));
|
||
}
|
||
points >= 2 && span >= MIN_INK_SPAN_PT
|
||
}
|
||
|
||
/// Whether finished ink is worth keeping as a stroke.
|
||
pub fn is_real_ink(stroke: &Stroke) -> bool {
|
||
drew_before(stroke, u64::MAX)
|
||
}
|
||
|
||
fn same_target_frame(a: TargetFrame, b: TargetFrame) -> bool {
|
||
(a.x - b.x).abs() <= 0.5
|
||
&& (a.y - b.y).abs() <= 0.5
|
||
&& (a.width - b.width).abs() <= 0.5
|
||
&& (a.height - b.height).abs() <= 0.5
|
||
}
|
||
|
||
/// One poll of the Shift ink: holding Shift alone and moving the cursor inside the capture
|
||
/// target draws a persistent stroke; letting go of Shift commits it. `pure_shift` is Shift
|
||
/// without other modifiers (and not left over from a chord such as the capture shortcut).
|
||
#[cfg_attr(target_os = "linux", allow(dead_code))]
|
||
fn track_shift_ink(
|
||
telemetry: &SharedCaptureTelemetry,
|
||
(x, y): (f64, f64),
|
||
pure_shift: bool,
|
||
prev_shift_ink: bool,
|
||
) {
|
||
track_shift_ink_at(
|
||
telemetry,
|
||
telemetry.elapsed_ms(),
|
||
(x, y),
|
||
pure_shift,
|
||
prev_shift_ink,
|
||
);
|
||
}
|
||
|
||
fn track_shift_ink_at(
|
||
telemetry: &SharedCaptureTelemetry,
|
||
t_ms: u64,
|
||
(x, y): (f64, f64),
|
||
pure_shift: bool,
|
||
prev_shift_ink: bool,
|
||
) {
|
||
let inside_target = telemetry.contains_screen_point(x, y);
|
||
if pure_shift && inside_target {
|
||
telemetry.shift_held.store(true, Ordering::Release);
|
||
let (nx, ny, fw, fh) = telemetry.normalize_point(x, y);
|
||
let sx = (nx * SCALE as f32).round() as i32;
|
||
let sy = (ny * SCALE as f32).round() as i32;
|
||
let mut guard = telemetry
|
||
.live_ink
|
||
.lock()
|
||
.unwrap_or_else(std::sync::PoisonError::into_inner);
|
||
if guard.is_none() {
|
||
telemetry.window_log_request.store(true, Ordering::Release);
|
||
}
|
||
let stroke = guard.get_or_insert_with(|| Stroke {
|
||
t0: t_ms,
|
||
t1: t_ms,
|
||
fw,
|
||
fh,
|
||
on: None,
|
||
pin: None,
|
||
laser: false,
|
||
pts: Vec::new(),
|
||
});
|
||
let dt = t_ms.saturating_sub(stroke.t0) as i32;
|
||
let should_push = stroke.pts.as_chunks::<3>().0.last().is_none_or(|prev| {
|
||
let dx = (sx - prev[0]) as f32 / SCALE as f32 * fw as f32;
|
||
let dy = (sy - prev[1]) as f32 / SCALE as f32 * fh as f32;
|
||
dx.hypot(dy) >= 1.5
|
||
});
|
||
if should_push && stroke.pts.len() / 3 < 2_400 {
|
||
stroke.pts.extend_from_slice(&[sx, sy, dt]);
|
||
stroke.t1 = t_ms;
|
||
}
|
||
} else if !pure_shift && prev_shift_ink {
|
||
telemetry.shift_held.store(false, Ordering::Release);
|
||
if let Some(stroke) = telemetry
|
||
.live_ink
|
||
.lock()
|
||
.unwrap_or_else(std::sync::PoisonError::into_inner)
|
||
.take()
|
||
// Ignore stationary Shift taps (e.g. typing a capital letter without moving the mouse).
|
||
&& is_real_ink(&stroke)
|
||
{
|
||
if telemetry.probing() {
|
||
let frame = *telemetry
|
||
.target_frame
|
||
.lock()
|
||
.unwrap_or_else(std::sync::PoisonError::into_inner);
|
||
if let Some((x, y)) = crate::ax::stroke_probe_point(&stroke, frame) {
|
||
crate::ax::Prober::shared().probe(crate::ax::Target::Stroke(stroke.t0), x, y);
|
||
}
|
||
}
|
||
telemetry
|
||
.strokes
|
||
.lock()
|
||
.unwrap_or_else(std::sync::PoisonError::into_inner)
|
||
.push(stroke);
|
||
let _ = telemetry.ui_sender.try_send(Event::RequestSnapshot);
|
||
}
|
||
}
|
||
}
|
||
|
||
pub fn is_frontmost_terminal() -> bool {
|
||
let Some((_pid, app, bundle_id)) = frontmost_app_info() else {
|
||
return false;
|
||
};
|
||
let a = app.to_ascii_lowercase();
|
||
let b = bundle_id.unwrap_or_default().to_ascii_lowercase();
|
||
[
|
||
"ghostty",
|
||
"iterm",
|
||
"terminal",
|
||
"kitty",
|
||
"wezterm",
|
||
"alacritty",
|
||
"warp",
|
||
// Windows Terminal, the console host and the shells it runs.
|
||
"windowsterminal",
|
||
"conhost",
|
||
"powershell",
|
||
"pwsh",
|
||
"cmd.exe",
|
||
"mintty",
|
||
]
|
||
.iter()
|
||
.any(|needle| a.contains(needle) || b.contains(needle))
|
||
}
|
||
|
||
/// Writes all bundle artifacts (`scrawl.json`, `summary.md`, `contact-sheet.png`, `frames/frame-XX.png`)
|
||
/// into a new or specified `.scrawl` bundle directory. The manifest comes last, whole, so a
|
||
/// bundle with one is complete.
|
||
pub fn write_bundle_artifacts(
|
||
out_dir: &Path,
|
||
bundle: &crate::model::ScrawlBundle,
|
||
annotated_frames: &[(crate::model::Keyframe, RgbaFrame)],
|
||
contact_sheet: &RgbaFrame,
|
||
summary_md: &str,
|
||
) -> Result<()> {
|
||
write_bundle_files(out_dir, bundle, annotated_frames, contact_sheet, summary_md)?;
|
||
publish_manifest(out_dir, bundle)
|
||
}
|
||
|
||
/// Writes `scrawl.json` atomically (a temporary file renamed into place): the last step of a
|
||
/// build, which marks the bundle finished.
|
||
pub fn publish_manifest(out_dir: &Path, bundle: &crate::model::ScrawlBundle) -> Result<()> {
|
||
let manifest_bytes = serde_json::to_vec_pretty(bundle)?;
|
||
let tmp = out_dir.join(".scrawl.json.tmp");
|
||
crate::routing::undo::write_private(&tmp, &manifest_bytes)
|
||
.with_context(|| format!("could not write {}", tmp.display()))?;
|
||
if let Err(error) = std::fs::rename(&tmp, out_dir.join("scrawl.json")) {
|
||
let _ = std::fs::remove_file(&tmp);
|
||
return Err(error).context("could not publish scrawl.json");
|
||
}
|
||
Ok(())
|
||
}
|
||
|
||
/// Everything [`write_bundle_artifacts`] writes except the manifest.
|
||
pub fn write_bundle_files(
|
||
out_dir: &Path,
|
||
bundle: &crate::model::ScrawlBundle,
|
||
annotated_frames: &[(crate::model::Keyframe, RgbaFrame)],
|
||
contact_sheet: &RgbaFrame,
|
||
summary_md: &str,
|
||
) -> Result<()> {
|
||
std::fs::create_dir_all(out_dir.join("frames"))
|
||
.with_context(|| format!("could not create {}", out_dir.display()))?;
|
||
#[cfg(unix)]
|
||
{
|
||
use std::os::unix::fs::PermissionsExt;
|
||
let _ = std::fs::set_permissions(out_dir, std::fs::Permissions::from_mode(0o700));
|
||
}
|
||
|
||
thread::scope(|scope| -> Result<()> {
|
||
let workers: Vec<_> = annotated_frames
|
||
.iter()
|
||
.map(|(kf, frame)| {
|
||
scope.spawn(move || -> Result<()> {
|
||
crate::render::save_png(frame, &out_dir.join(&kf.file))
|
||
})
|
||
})
|
||
.collect();
|
||
crate::render::save_png(contact_sheet, &out_dir.join(&bundle.contact_sheet))?;
|
||
for worker in workers {
|
||
worker
|
||
.join()
|
||
.map_err(|_| anyhow::anyhow!("PNG encoding worker panicked"))??;
|
||
}
|
||
Ok(())
|
||
})?;
|
||
|
||
crate::routing::undo::write_private(&out_dir.join("summary.md"), summary_md.as_bytes())?;
|
||
|
||
Ok(())
|
||
}
|
||
|
||
pub const SCRAWL_BUNDLE_ID: &str = "dev.kitlangton.scrawl";
|
||
|
||
/// Returns `true` when another process with bundle identifier `dev.kitlangton.scrawl` is running.
|
||
pub fn is_scrawl_app_running() -> bool {
|
||
#[cfg(target_os = "macos")]
|
||
{
|
||
use objc2_app_kit::NSRunningApplication;
|
||
use objc2_foundation::NSString;
|
||
let current_pid = std::process::id() as i32;
|
||
let apps = NSRunningApplication::runningApplicationsWithBundleIdentifier(
|
||
&NSString::from_str(SCRAWL_BUNDLE_ID),
|
||
);
|
||
apps.iter()
|
||
.any(|app| app.processIdentifier() != current_pid && !app.isTerminated())
|
||
}
|
||
#[cfg(not(target_os = "macos"))]
|
||
{
|
||
false
|
||
}
|
||
}
|
||
|
||
/// Explicit ownership switches for machine-wide shared resources so an embedded host (such as
|
||
/// Hex) can yield while Scrawl (`dev.kitlangton.scrawl`) runs instead of colliding on port 47777,
|
||
/// racing `.deliver.json` retries, or registering conflicting global shortcuts.
|
||
///
|
||
/// Defaults to `false` for all resources so [`Engine::new`] never claims machine-wide resources
|
||
/// unless the host explicitly opts in.
|
||
#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)]
|
||
pub struct SharedResources {
|
||
/// Bind the ScrawlKit HTTP ingest server (`DEFAULT_INGEST_PORT`, 47777) and USB poller.
|
||
pub ingest_server: bool,
|
||
/// Run background `.deliver.json` OpenCode delivery retries.
|
||
pub delivery_retries: bool,
|
||
/// Register the global capture and paste shortcuts (`⌥⇧S`, `⌥⇧V`).
|
||
pub global_shortcuts: bool,
|
||
}
|
||
|
||
impl SharedResources {
|
||
pub const fn none() -> Self {
|
||
Self {
|
||
ingest_server: false,
|
||
delivery_retries: false,
|
||
global_shortcuts: false,
|
||
}
|
||
}
|
||
|
||
pub const fn all() -> Self {
|
||
Self {
|
||
ingest_server: true,
|
||
delivery_retries: true,
|
||
global_shortcuts: true,
|
||
}
|
||
}
|
||
|
||
/// Claims the shared resources only when `peer_running` is `false`.
|
||
pub const fn when_peer_absent(peer_running: bool) -> Self {
|
||
if peer_running {
|
||
Self::none()
|
||
} else {
|
||
Self::all()
|
||
}
|
||
}
|
||
|
||
/// Claims the shared resources only when no standalone Scrawl app (`dev.kitlangton.scrawl`)
|
||
/// is running.
|
||
pub fn when_scrawl_absent() -> Self {
|
||
Self::when_peer_absent(is_scrawl_app_running())
|
||
}
|
||
|
||
pub const fn with_ingest_server(mut self, own: bool) -> Self {
|
||
self.ingest_server = own;
|
||
self
|
||
}
|
||
|
||
pub const fn with_delivery_retries(mut self, own: bool) -> Self {
|
||
self.delivery_retries = own;
|
||
self
|
||
}
|
||
|
||
pub const fn with_global_shortcuts(mut self, own: bool) -> Self {
|
||
self.global_shortcuts = own;
|
||
self
|
||
}
|
||
}
|
||
|
||
fn start_audio_for<T>(
|
||
audio: AudioIn,
|
||
host: &dyn Host,
|
||
start_mic: impl FnOnce() -> Result<T>,
|
||
) -> Option<T> {
|
||
match audio {
|
||
AudioIn::Microphone => {
|
||
let stage = Instant::now();
|
||
let recorder = start_mic().ok();
|
||
host.metric("audio_start_ms", stage.elapsed().as_secs_f64() * 1_000.0);
|
||
recorder
|
||
}
|
||
AudioIn::External | AudioIn::None => None,
|
||
}
|
||
}
|
||
|
||
enum InnerSource {
|
||
Screen {
|
||
session: ScreenCaptureSession,
|
||
audio: Option<crate::audio::AudioRecorder>,
|
||
windows: Option<crate::windows::WindowLogger>,
|
||
},
|
||
Phone(crate::phone::PhoneSession),
|
||
}
|
||
|
||
enum SealedSource {
|
||
Screen {
|
||
recording: ScreenRecording,
|
||
recorded_audio: Option<crate::audio::RecordedAudio>,
|
||
activity: crate::windows::ActivityLog,
|
||
},
|
||
Phone(Result<crate::phone::PhoneRecording>),
|
||
}
|
||
|
||
pub struct Engine {
|
||
host: Arc<dyn Host>,
|
||
config: Arc<dyn ConfigSource>,
|
||
transcriber: Option<Arc<dyn crate::transcribe::Transcriber>>,
|
||
clipboard: Option<Arc<dyn crate::handoff::Clipboard>>,
|
||
telemetry: Arc<SharedCaptureTelemetry>,
|
||
custom_telemetry: bool,
|
||
resources: SharedResources,
|
||
}
|
||
|
||
struct HostLogSink(Arc<dyn Host>);
|
||
|
||
impl crate::log::Sink for HostLogSink {
|
||
fn log(&self, line: &str) {
|
||
self.0.log(line);
|
||
}
|
||
|
||
fn metric(&self, name: &str, ms: f64) {
|
||
self.0.metric(name, ms);
|
||
}
|
||
}
|
||
|
||
impl Engine {
|
||
pub fn new(host: Arc<dyn Host>, config: Arc<dyn ConfigSource>) -> Self {
|
||
crate::log::set_sink_if_unset(Arc::new(HostLogSink(Arc::clone(&host))));
|
||
let (tx, rx) = flume::unbounded();
|
||
// Without an app's own telemetry, the one event a host must act on (the iPhone mirror
|
||
// ending on its own) reaches it through `Host::emit`; the rest are for Scrawl's UI.
|
||
let relay = Arc::clone(&host);
|
||
let _ = std::thread::Builder::new()
|
||
.name("scrawl-ui-events".into())
|
||
.spawn(move || {
|
||
for event in rx {
|
||
if matches!(event, Event::PhoneEnded) {
|
||
relay.emit(event);
|
||
}
|
||
}
|
||
});
|
||
Self {
|
||
host,
|
||
config,
|
||
transcriber: None,
|
||
clipboard: None,
|
||
telemetry: SharedCaptureTelemetry::new(tx, ShortcutConfig::ansi_defaults()),
|
||
custom_telemetry: false,
|
||
resources: SharedResources::none(),
|
||
}
|
||
}
|
||
|
||
pub fn default_engine() -> Self {
|
||
let (tx, _rx) = flume::unbounded();
|
||
Self::new(
|
||
Arc::new(ChannelHost::new(tx)),
|
||
Arc::new(DefaultConfigSource),
|
||
)
|
||
.with_clipboard(Arc::new(crate::handoff::SystemClipboard))
|
||
}
|
||
|
||
pub fn with_telemetry(mut self, telemetry: Arc<SharedCaptureTelemetry>) -> Self {
|
||
self.telemetry = telemetry;
|
||
self.custom_telemetry = true;
|
||
self
|
||
}
|
||
|
||
pub fn with_transcriber(
|
||
mut self,
|
||
transcriber: Arc<dyn crate::transcribe::Transcriber>,
|
||
) -> Self {
|
||
self.transcriber = Some(transcriber);
|
||
self
|
||
}
|
||
|
||
pub fn with_clipboard(mut self, clipboard: Arc<dyn crate::handoff::Clipboard>) -> Self {
|
||
self.clipboard = Some(clipboard);
|
||
self
|
||
}
|
||
|
||
pub fn with_resources(mut self, resources: SharedResources) -> Self {
|
||
self.resources = resources;
|
||
self
|
||
}
|
||
|
||
pub fn resources(&self) -> SharedResources {
|
||
self.resources
|
||
}
|
||
|
||
/// Starts whichever machine-wide shared resources (`ingest_server`, `delivery_retries`,
|
||
/// `global_shortcuts`) are enabled in [`Self::resources`]. Does nothing when all switches are
|
||
/// `false`.
|
||
pub fn install_shared_resources(&self) {
|
||
if self.resources.ingest_server || self.resources.delivery_retries {
|
||
let cfg = self.config.engine_config().ok();
|
||
let captures = cfg.as_ref().map(|c| c.resolved_captures_dir());
|
||
let token_file = cfg
|
||
.as_ref()
|
||
.and_then(|c| c.ingest_token_path().map(Path::to_path_buf));
|
||
crate::phone::start_ingest_server_for_resources(
|
||
self.resources,
|
||
self.telemetry.ui_sender.clone(),
|
||
self.clipboard.clone(),
|
||
captures,
|
||
token_file,
|
||
);
|
||
}
|
||
if self.resources.global_shortcuts {
|
||
let (rec, paste) = self.telemetry.shortcuts();
|
||
if rec.is_default_for(ShortcutKind::Record) && paste.is_default_for(ShortcutKind::Paste)
|
||
{
|
||
*self
|
||
.telemetry
|
||
.shortcut
|
||
.lock()
|
||
.unwrap_or_else(std::sync::PoisonError::into_inner) =
|
||
ShortcutConfig::default_record();
|
||
*self
|
||
.telemetry
|
||
.paste_shortcut
|
||
.lock()
|
||
.unwrap_or_else(std::sync::PoisonError::into_inner) =
|
||
ShortcutConfig::default_paste();
|
||
}
|
||
install_global_hotkeys(&self.telemetry);
|
||
}
|
||
}
|
||
|
||
pub fn telemetry(&self) -> &Arc<SharedCaptureTelemetry> {
|
||
&self.telemetry
|
||
}
|
||
|
||
pub fn discover_target(&self, window: Option<&str>, at: Option<[f64; 2]>) -> TargetInfo {
|
||
discover_target_at(window, at)
|
||
}
|
||
|
||
pub fn selectable_windows(&self) -> Vec<TargetInfo> {
|
||
list_selectable_windows()
|
||
}
|
||
|
||
pub fn phones(&self) -> Result<Vec<crate::helper::PhoneDevice>> {
|
||
crate::helper::devices()
|
||
}
|
||
|
||
/// Blocking: performs synchronous target discovery (when `req.target` is
|
||
/// [`Target::ActiveDisplay`]), spawns the recording helper, and waits for stream readiness
|
||
/// (up to ~1.2 s). Call this on a background worker thread rather than the UI main thread,
|
||
/// or call [`Self::discover_target`] first and pass [`Start::screen`] to a worker thread.
|
||
pub fn start(&self, req: Start) -> Result<Capture> {
|
||
self.start_with_preview(req, None)
|
||
}
|
||
|
||
/// Blocking: see [`Self::start`].
|
||
pub fn start_with_preview(
|
||
&self,
|
||
req: Start,
|
||
existing_preview: Option<crate::phone::PhoneSession>,
|
||
) -> Result<Capture> {
|
||
let engine_cfg = self.config.engine_config()?;
|
||
let agent_context = engine_cfg.experiment(crate::handoff::AGENT_CONTEXT);
|
||
let dir = match req.bundle_dir {
|
||
Some(dir) => {
|
||
std::fs::create_dir_all(&dir)
|
||
.with_context(|| format!("could not create {}", dir.display()))?;
|
||
#[cfg(unix)]
|
||
{
|
||
use std::os::unix::fs::PermissionsExt;
|
||
let _ = std::fs::set_permissions(&dir, std::fs::Permissions::from_mode(0o700));
|
||
}
|
||
dir
|
||
}
|
||
None => {
|
||
let root = match std::env::var_os("SCRAWL_BENCH_START") {
|
||
Some(dir) => PathBuf::from(dir),
|
||
None => engine_cfg.resolved_captures_dir(),
|
||
};
|
||
let id = crate::pipeline::make_bundle_id();
|
||
let dir = root.join(format!("{id}.scrawl"));
|
||
std::fs::create_dir_all(&dir)
|
||
.with_context(|| format!("could not create {}", dir.display()))?;
|
||
#[cfg(unix)]
|
||
{
|
||
use std::os::unix::fs::PermissionsExt;
|
||
let _ = std::fs::set_permissions(&dir, std::fs::Permissions::from_mode(0o700));
|
||
}
|
||
dir
|
||
}
|
||
};
|
||
|
||
let saved_excluded = self
|
||
.telemetry
|
||
.excluded_window_ids
|
||
.lock()
|
||
.unwrap_or_else(std::sync::PoisonError::into_inner)
|
||
.clone();
|
||
let saved_passthrough = self
|
||
.telemetry
|
||
.passthrough_window_ids
|
||
.lock()
|
||
.unwrap_or_else(std::sync::PoisonError::into_inner)
|
||
.clone();
|
||
let own = self.host.own_windows();
|
||
let telemetry = if self.custom_telemetry || self.resources.global_shortcuts {
|
||
Arc::clone(&self.telemetry)
|
||
} else {
|
||
let isolated = SharedCaptureTelemetry::new(
|
||
self.telemetry.ui_sender.clone(),
|
||
ShortcutConfig::ansi_defaults(),
|
||
);
|
||
*isolated
|
||
.passthrough_window_ids
|
||
.lock()
|
||
.unwrap_or_else(std::sync::PoisonError::into_inner) = saved_passthrough;
|
||
isolated
|
||
};
|
||
|
||
let (target, source, session_id) = match req.target {
|
||
Target::Phone { device, name } => {
|
||
let movie_path = dir.join("recording.mov");
|
||
let dev_name = name.as_deref().unwrap_or("iPhone");
|
||
let target = crate::phone::phone_target(dev_name, None, None, 0, 0);
|
||
let initial_frame = existing_preview
|
||
.as_ref()
|
||
.and_then(crate::phone::PhoneSession::rect)
|
||
.map_or(
|
||
TargetFrame {
|
||
x: -100_000.0,
|
||
y: -100_000.0,
|
||
width: 1.0,
|
||
height: 1.0,
|
||
scale: 2.0,
|
||
},
|
||
|[rx, ry, rw, rh]| TargetFrame {
|
||
x: f64::from(rx),
|
||
y: f64::from(ry),
|
||
width: f64::from(rw),
|
||
height: f64::from(rh),
|
||
scale: 2.0,
|
||
},
|
||
);
|
||
telemetry.set_capture_mode(CaptureMode::Device);
|
||
let session_id = telemetry.reset_for_session_at_with_agent_context(
|
||
initial_frame,
|
||
Some(req.t0_boot_ns),
|
||
agent_context,
|
||
);
|
||
Self::restore_own_windows(&telemetry, &saved_excluded, &own);
|
||
// Hold interaction telemetry inactive until the helper's `recording` event arrives.
|
||
telemetry.active.store(false, Ordering::Release);
|
||
// The movie's offset is measured from `t0`, the clock input and audio share.
|
||
let since_t0_ms = boot_time_ns().saturating_sub(req.t0_boot_ns) / 1_000_000;
|
||
let origin_epoch_ms = crate::pipeline::now_epoch_ms().saturating_sub(since_t0_ms);
|
||
let mut session = if let Some(mut preview) = existing_preview {
|
||
preview.begin_recording(&movie_path);
|
||
preview
|
||
} else {
|
||
match crate::phone::PhoneSession::start(
|
||
&movie_path,
|
||
device.as_deref(),
|
||
Some(Arc::clone(&telemetry)),
|
||
req.audio == AudioIn::External,
|
||
) {
|
||
Ok(session) => session,
|
||
Err(error) => {
|
||
let _ = telemetry.finish_session();
|
||
let _ = telemetry.take_flags();
|
||
let _ = std::fs::remove_dir_all(&dir);
|
||
return Err(error);
|
||
}
|
||
}
|
||
};
|
||
session.clock_from(origin_epoch_ms);
|
||
(target, InnerSource::Phone(session), session_id)
|
||
}
|
||
Target::ActiveDisplay | Target::Screen(_) => {
|
||
let target = match req.target {
|
||
Target::Screen(t) => t,
|
||
Target::ActiveDisplay => discover_target_at(None, crate::screens::pointer()),
|
||
Target::Phone { .. } => unreachable!(),
|
||
};
|
||
let video_path = dir.join("recording.mp4");
|
||
telemetry.set_capture_mode(target.mode);
|
||
let session_id = telemetry.reset_for_session_at_with_agent_context(
|
||
target.frame,
|
||
Some(req.t0_boot_ns),
|
||
agent_context,
|
||
);
|
||
Self::restore_own_windows(&telemetry, &saved_excluded, &own);
|
||
let audio = start_audio_for(
|
||
req.audio,
|
||
self.host.as_ref(),
|
||
crate::audio::AudioRecorder::start,
|
||
);
|
||
let stage = Instant::now();
|
||
let session = ScreenCaptureSession::start_with_budget(
|
||
target.clone(),
|
||
Some(video_path),
|
||
Arc::clone(&telemetry.excluded_window_ids),
|
||
Arc::clone(&own.recorded),
|
||
engine_cfg.candidate_budget_bytes(),
|
||
req.movie,
|
||
req.portal_token,
|
||
);
|
||
self.host
|
||
.metric("sck_start_ms", stage.elapsed().as_secs_f64() * 1_000.0);
|
||
let windows = (req.window_log && target.mode == CaptureMode::Display).then(|| {
|
||
let started_at = *telemetry
|
||
.started_at
|
||
.lock()
|
||
.unwrap_or_else(std::sync::PoisonError::into_inner);
|
||
crate::windows::WindowLogger::start(
|
||
target.frame,
|
||
started_at,
|
||
Arc::clone(&telemetry.window_log_request),
|
||
Arc::clone(&telemetry.excluded_window_ids),
|
||
)
|
||
});
|
||
(
|
||
target,
|
||
InnerSource::Screen {
|
||
session,
|
||
audio,
|
||
windows,
|
||
},
|
||
session_id,
|
||
)
|
||
}
|
||
};
|
||
|
||
self.host.emit(Event::Progress {
|
||
bundle: dir.clone(),
|
||
stage: Stage::Recording,
|
||
});
|
||
Ok(Capture {
|
||
host: Arc::clone(&self.host),
|
||
transcriber: self.transcriber.clone(),
|
||
engine_cfg,
|
||
input: InputSink::new(Arc::clone(&telemetry)),
|
||
telemetry,
|
||
session_id,
|
||
dir,
|
||
target,
|
||
t0_boot_ns: req.t0_boot_ns,
|
||
note: req.note,
|
||
host_narration: req.audio == AudioIn::External,
|
||
source,
|
||
stopped_input: None,
|
||
})
|
||
}
|
||
|
||
fn restore_own_windows(
|
||
telemetry: &SharedCaptureTelemetry,
|
||
saved_excluded: &[u32],
|
||
own: &OwnWindows,
|
||
) {
|
||
let mut excluded = telemetry
|
||
.excluded_window_ids
|
||
.lock()
|
||
.unwrap_or_else(std::sync::PoisonError::into_inner);
|
||
for &id in saved_excluded.iter().chain(&own.excluded) {
|
||
if !excluded.contains(&id) {
|
||
excluded.push(id);
|
||
}
|
||
}
|
||
drop(excluded);
|
||
let mut passthrough = telemetry
|
||
.passthrough_window_ids
|
||
.lock()
|
||
.unwrap_or_else(std::sync::PoisonError::into_inner);
|
||
for &id in &own.passthrough {
|
||
if !passthrough.contains(&id) {
|
||
passthrough.push(id);
|
||
}
|
||
}
|
||
}
|
||
}
|
||
|
||
pub struct Capture {
|
||
host: Arc<dyn Host>,
|
||
transcriber: Option<Arc<dyn crate::transcribe::Transcriber>>,
|
||
engine_cfg: crate::handoff::EngineConfig,
|
||
telemetry: Arc<SharedCaptureTelemetry>,
|
||
session_id: u64,
|
||
input: InputSink,
|
||
dir: PathBuf,
|
||
target: TargetInfo,
|
||
t0_boot_ns: u64,
|
||
note: Option<String>,
|
||
/// A phone capture's narration is `finish`'s audio, not the movie's.
|
||
host_narration: bool,
|
||
source: InnerSource,
|
||
stopped_input: Option<(Vec<InteractionEvent>, Vec<Stroke>, Vec<u64>)>,
|
||
}
|
||
|
||
/// A capture whose screen/phone recording, window logger, and input collection have already
|
||
/// stopped, ready for [`SealedCapture::finish`] on a background worker.
|
||
pub struct SealedCapture {
|
||
host: Arc<dyn Host>,
|
||
transcriber: Option<Arc<dyn crate::transcribe::Transcriber>>,
|
||
engine_cfg: crate::handoff::EngineConfig,
|
||
dir: PathBuf,
|
||
target: TargetInfo,
|
||
t0_boot_ns: u64,
|
||
note: Option<String>,
|
||
host_narration: bool,
|
||
route_hint: RouteContext,
|
||
events: Vec<InteractionEvent>,
|
||
strokes: Vec<Stroke>,
|
||
flags: Vec<u64>,
|
||
source: SealedSource,
|
||
}
|
||
|
||
impl Capture {
|
||
pub fn dir(&self) -> &Path {
|
||
&self.dir
|
||
}
|
||
|
||
pub fn target(&self) -> &TargetInfo {
|
||
&self.target
|
||
}
|
||
|
||
pub fn is_screen(&self) -> bool {
|
||
matches!(self.source, InnerSource::Screen { .. })
|
||
}
|
||
|
||
pub fn is_phone(&self) -> bool {
|
||
matches!(self.source, InnerSource::Phone(_))
|
||
}
|
||
|
||
pub fn has_phone_recording(&self) -> bool {
|
||
match &self.source {
|
||
InnerSource::Phone(session) => session.has_recording(),
|
||
InnerSource::Screen { .. } => false,
|
||
}
|
||
}
|
||
|
||
pub fn phone_has_ended(&self) -> bool {
|
||
match &self.source {
|
||
InnerSource::Phone(session) => session.has_ended(),
|
||
InnerSource::Screen { .. } => false,
|
||
}
|
||
}
|
||
|
||
/// Why the screen source gave up before the stop, if it did (Linux: the portal's picker
|
||
/// was cancelled): the capture has nothing more to record.
|
||
pub fn screen_failure(&self) -> Option<String> {
|
||
match &self.source {
|
||
InnerSource::Screen { session, .. } => session.failure(),
|
||
InnerSource::Phone(_) => None,
|
||
}
|
||
}
|
||
|
||
pub fn started_at_boot_ns(&self) -> u64 {
|
||
self.t0_boot_ns
|
||
}
|
||
|
||
pub fn input(&self) -> &InputSink {
|
||
&self.input
|
||
}
|
||
|
||
pub fn ink(&self) -> InkSnapshot {
|
||
self.telemetry.ink_snapshot()
|
||
}
|
||
|
||
pub fn elapsed_ms(&self) -> u64 {
|
||
match &self.source {
|
||
InnerSource::Screen { session, .. } => session.elapsed_ms(),
|
||
InnerSource::Phone(session) => session.elapsed_ms(),
|
||
}
|
||
}
|
||
|
||
pub fn request_frame(&self) {
|
||
if let InnerSource::Screen { session, .. } = &self.source {
|
||
session.request_immediate_frame();
|
||
}
|
||
}
|
||
|
||
pub fn route_context(&self) -> RouteContext {
|
||
if !self.is_screen() {
|
||
return RouteContext::default();
|
||
}
|
||
let front_app = (self.target.mode != CaptureMode::Window)
|
||
.then(frontmost_app_info)
|
||
.flatten();
|
||
let app_pid = match (&front_app, self.target.window_id) {
|
||
(Some((pid, ..)), _) => Some(*pid),
|
||
(None, Some(window)) if crate::routing::jev::enabled() => crate::windows::raw_windows()
|
||
.into_iter()
|
||
.find(|w| w.id == window)
|
||
.map(|w| w.pid),
|
||
_ => None,
|
||
};
|
||
RouteContext {
|
||
front_bundle_id: front_app.and_then(|(_, _, id)| id),
|
||
captured_bundle_id: self.target.bundle_id.clone(),
|
||
app_pid,
|
||
}
|
||
}
|
||
|
||
/// Stops collecting pointer/ink input on the calling thread before moving `self` to a
|
||
/// background worker to run `finish`.
|
||
pub fn stop_input(&mut self) {
|
||
if self.stopped_input.is_none() {
|
||
if self.telemetry.session_id.load(Ordering::Acquire) == self.session_id {
|
||
let (events, strokes) = self.telemetry.finish_session();
|
||
let flags = self.telemetry.take_flags();
|
||
self.stopped_input = Some((events, strokes, flags));
|
||
} else {
|
||
self.stopped_input = Some(Default::default());
|
||
}
|
||
}
|
||
}
|
||
|
||
pub fn stop_raw_screen(
|
||
mut self,
|
||
) -> Option<(ScreenRecording, Option<crate::audio::RecordedAudio>)> {
|
||
self.stop_input();
|
||
match self.source {
|
||
InnerSource::Screen {
|
||
session,
|
||
audio,
|
||
windows,
|
||
} => {
|
||
if let Some(windows) = windows {
|
||
let _ = windows.stop();
|
||
}
|
||
let (recording, recorded_audio) = std::thread::scope(|scope| {
|
||
let screen = scope.spawn(|| session.stop());
|
||
let audio = audio.and_then(|r| r.stop().ok());
|
||
(screen.join().unwrap(), audio)
|
||
});
|
||
Some((recording, recorded_audio))
|
||
}
|
||
InnerSource::Phone(_) => None,
|
||
}
|
||
}
|
||
|
||
pub fn cancel(mut self) {
|
||
self.stop_input();
|
||
match self.source {
|
||
InnerSource::Screen {
|
||
session,
|
||
audio,
|
||
windows,
|
||
} => {
|
||
let recording = session.stop();
|
||
if let Some(audio) = audio {
|
||
let _ = audio.stop();
|
||
}
|
||
if let Some(windows) = windows {
|
||
let _ = windows.stop();
|
||
}
|
||
recording.discard();
|
||
}
|
||
InnerSource::Phone(session) => session.cancel(),
|
||
}
|
||
if let Err(error) = std::fs::remove_dir_all(&self.dir) {
|
||
self.host
|
||
.log(&format!("could not remove {}: {error}", self.dir.display()));
|
||
}
|
||
self.host.emit(Event::Progress {
|
||
bundle: self.dir,
|
||
stage: Stage::Cancelled,
|
||
});
|
||
}
|
||
|
||
/// Ends screen/phone recording, window logging, and input collection immediately and returns a
|
||
/// [`SealedCapture`] whose [`SealedCapture::finish`] can run later on a worker thread.
|
||
pub fn stop(mut self) -> SealedCapture {
|
||
self.stop_input();
|
||
let (events, strokes, flags) = self.stopped_input.take().unwrap_or_default();
|
||
let route_hint = self.route_context();
|
||
let bundle_dir = self.dir.clone();
|
||
self.host.emit(Event::Progress {
|
||
bundle: bundle_dir,
|
||
stage: Stage::Stopping,
|
||
});
|
||
let source = match self.source {
|
||
InnerSource::Phone(session) => SealedSource::Phone(session.stop()),
|
||
InnerSource::Screen {
|
||
session,
|
||
audio,
|
||
windows,
|
||
} => {
|
||
let stop_started = Instant::now();
|
||
let (recording, recorded_audio, mut activity) = std::thread::scope(|s| {
|
||
let screen_handle = s.spawn(|| session.stop());
|
||
let win_handle = s.spawn(|| {
|
||
windows
|
||
.map(crate::windows::WindowLogger::stop)
|
||
.unwrap_or_default()
|
||
});
|
||
let audio = audio.and_then(|r| r.stop().ok());
|
||
(
|
||
screen_handle.join().unwrap(),
|
||
audio,
|
||
win_handle.join().unwrap_or_default(),
|
||
)
|
||
});
|
||
activity.stop_started = Some(stop_started);
|
||
activity.stop_ms = Some(stop_started.elapsed().as_millis() as u64);
|
||
SealedSource::Screen {
|
||
recording,
|
||
recorded_audio,
|
||
activity,
|
||
}
|
||
}
|
||
};
|
||
SealedCapture {
|
||
host: self.host,
|
||
transcriber: self.transcriber,
|
||
engine_cfg: self.engine_cfg,
|
||
dir: self.dir,
|
||
target: self.target,
|
||
t0_boot_ns: self.t0_boot_ns,
|
||
note: self.note,
|
||
host_narration: self.host_narration,
|
||
route_hint,
|
||
events,
|
||
strokes,
|
||
flags,
|
||
source,
|
||
}
|
||
}
|
||
|
||
/// Alias for [`Self::stop`].
|
||
pub fn seal(self) -> SealedCapture {
|
||
self.stop()
|
||
}
|
||
|
||
/// Blocking: stops streams, transcribes audio, renders keyframes and writes the bundle.
|
||
pub fn finish(self, external_audio: Option<crate::audio::RecordedAudio>) -> Result<Finished> {
|
||
self.stop().finish(external_audio)
|
||
}
|
||
}
|
||
|
||
impl SealedCapture {
|
||
pub fn dir(&self) -> &Path {
|
||
&self.dir
|
||
}
|
||
|
||
pub fn target(&self) -> &TargetInfo {
|
||
&self.target
|
||
}
|
||
|
||
pub fn started_at_boot_ns(&self) -> u64 {
|
||
self.t0_boot_ns
|
||
}
|
||
|
||
pub fn route_context(&self) -> &RouteContext {
|
||
&self.route_hint
|
||
}
|
||
|
||
pub fn cancel(self) {
|
||
if let SealedSource::Screen { recording, .. } = self.source {
|
||
recording.discard();
|
||
}
|
||
if let Err(error) = std::fs::remove_dir_all(&self.dir) {
|
||
self.host
|
||
.log(&format!("could not remove {}: {error}", self.dir.display()));
|
||
}
|
||
self.host.emit(Event::Progress {
|
||
bundle: self.dir,
|
||
stage: Stage::Cancelled,
|
||
});
|
||
}
|
||
|
||
/// Blocking: transcribes audio, renders keyframes and writes the bundle for a stopped capture.
|
||
pub fn finish(self, external_audio: Option<crate::audio::RecordedAudio>) -> Result<Finished> {
|
||
let bundle_dir = self.dir.clone();
|
||
let route_hint = self.route_hint;
|
||
let flags = self.flags;
|
||
let events = self.events;
|
||
let strokes = self.strokes;
|
||
let result = match self.source {
|
||
SealedSource::Phone(stopped) => {
|
||
let finalized = stopped.and_then(|mut recording| {
|
||
for flag in flags {
|
||
if flag >= recording.offset_ms {
|
||
recording.flags.push(flag - recording.offset_ms);
|
||
}
|
||
}
|
||
let narration = if self.host_narration {
|
||
self.host.emit(Event::Progress {
|
||
bundle: bundle_dir.clone(),
|
||
stage: Stage::Transcribing,
|
||
});
|
||
Some(crate::phone::HostNarration {
|
||
audio: external_audio
|
||
.map(|audio| audio.since(recording.offset_ms))
|
||
.transpose()?,
|
||
transcriber: self.transcriber.as_deref(),
|
||
})
|
||
} else {
|
||
None
|
||
};
|
||
crate::phone::finalize_live_capture_for_config(
|
||
&bundle_dir,
|
||
recording,
|
||
events,
|
||
strokes,
|
||
self.note,
|
||
Some(&self.engine_cfg),
|
||
narration,
|
||
)
|
||
});
|
||
if let Err(error) = &finalized {
|
||
self.host.log(&format!("iPhone capture failed: {error:#}"));
|
||
}
|
||
finalized.map(|(dir, bundle, summary)| {
|
||
let words = crate::transcribe::word_count(&bundle.transcript);
|
||
let duration_ms = bundle.duration_ms;
|
||
let missing_transcript = bundle
|
||
.transcript
|
||
.missing
|
||
.as_ref()
|
||
.map(|m| m.reason.clone())
|
||
.or_else(|| {
|
||
(bundle.transcript.text.trim().is_empty() && bundle.audio.is_none())
|
||
.then(|| "no_audio".to_owned())
|
||
});
|
||
Finished {
|
||
dir,
|
||
bundle,
|
||
summary,
|
||
words,
|
||
duration_ms,
|
||
route_hint,
|
||
gaps: 0,
|
||
missing_transcript,
|
||
}
|
||
})
|
||
}
|
||
SealedSource::Screen {
|
||
recording,
|
||
recorded_audio,
|
||
activity,
|
||
} => {
|
||
let recorded_audio = external_audio.or(recorded_audio);
|
||
let gaps = recorded_audio.as_ref().map_or(0, |a| a.gaps.len());
|
||
let had_audio_samples = recorded_audio
|
||
.as_ref()
|
||
.is_some_and(|a| a.sample_count() > 0 || !a.wav_bytes.is_empty());
|
||
let duration_ms = recording.duration_ms;
|
||
self.host.emit(Event::Progress {
|
||
bundle: bundle_dir.clone(),
|
||
stage: Stage::Transcribing,
|
||
});
|
||
crate::pipeline::finalize_capture_bundle_for_config(
|
||
&bundle_dir,
|
||
self.target,
|
||
recording,
|
||
strokes,
|
||
events,
|
||
flags,
|
||
recorded_audio,
|
||
self.note,
|
||
activity,
|
||
self.transcriber.as_deref(),
|
||
Some(&self.engine_cfg),
|
||
)
|
||
.map(|(dir, bundle, summary)| {
|
||
let words = crate::transcribe::word_count(&bundle.transcript);
|
||
let missing_transcript = bundle
|
||
.transcript
|
||
.missing
|
||
.as_ref()
|
||
.map(|m| m.reason.clone())
|
||
.or_else(|| {
|
||
(bundle.transcript.text.trim().is_empty()
|
||
&& bundle.audio.is_none()
|
||
&& !had_audio_samples)
|
||
.then(|| "no_audio".to_owned())
|
||
});
|
||
Finished {
|
||
dir,
|
||
bundle,
|
||
summary,
|
||
words,
|
||
duration_ms,
|
||
route_hint,
|
||
gaps,
|
||
missing_transcript,
|
||
}
|
||
})
|
||
}
|
||
};
|
||
|
||
self.host.emit(Event::Progress {
|
||
bundle: bundle_dir,
|
||
stage: if result.is_ok() {
|
||
Stage::Done
|
||
} else {
|
||
Stage::Failed
|
||
},
|
||
});
|
||
result
|
||
}
|
||
}
|
||
|
||
#[cfg(test)]
|
||
mod tests {
|
||
use super::*;
|
||
|
||
#[test]
|
||
fn screen_stop_joins_worker_before_collecting_its_final_media() {
|
||
let stopping = Arc::new(AtomicBool::new(false));
|
||
let frames = Arc::new(Mutex::new(CandidateBuffer::new(1 << 20)));
|
||
let worker_stopping = Arc::clone(&stopping);
|
||
let worker_frames = Arc::clone(&frames);
|
||
let path = std::env::temp_dir().join(crate::pipeline::make_bundle_id());
|
||
let worker_path = path.clone();
|
||
let worker = thread::spawn(move || {
|
||
let deadline = Instant::now() + Duration::from_secs(2);
|
||
while !worker_stopping.load(Ordering::Acquire) && Instant::now() < deadline {
|
||
thread::yield_now();
|
||
}
|
||
assert!(worker_stopping.load(Ordering::Acquire));
|
||
worker_frames
|
||
.lock()
|
||
.unwrap()
|
||
.push(RgbaFrame::new(42, 1, 1, vec![1, 2, 3, 255]), false);
|
||
std::fs::write(&worker_path, b"finalized media").unwrap();
|
||
Some(Movie::written(worker_path, 120))
|
||
});
|
||
let session = ScreenCaptureSession {
|
||
failure: Arc::default(),
|
||
started_at: Instant::now(),
|
||
stopping,
|
||
urgent_snapshot: Arc::new(AtomicBool::new(false)),
|
||
frames,
|
||
sampler_thread: Some(worker),
|
||
_recording: crate::retention::Recording::start(),
|
||
};
|
||
let recording = session.stop();
|
||
let frames = recording.frames;
|
||
assert_eq!(frames.len(), 1);
|
||
assert_eq!(frames[0].t_ms, 42);
|
||
assert_eq!(frames[0].rgba, vec![1, 2, 3, 255]);
|
||
assert_eq!(
|
||
recording.video.and_then(Movie::wait),
|
||
Some((path.clone(), 120))
|
||
);
|
||
std::fs::remove_file(path).unwrap();
|
||
}
|
||
|
||
/// A capture stopped with no picture names the screen source's failure when it gave one, and
|
||
/// otherwise the platform's reason, never that failure string.
|
||
#[test]
|
||
fn a_capture_with_no_frames_says_why_the_screen_gave_nothing() {
|
||
let stop_with = |failure: Option<&str>| {
|
||
ScreenCaptureSession {
|
||
failure: Arc::new(Mutex::new(failure.map(str::to_owned))),
|
||
started_at: Instant::now(),
|
||
stopping: Arc::new(AtomicBool::new(false)),
|
||
urgent_snapshot: Arc::new(AtomicBool::new(false)),
|
||
frames: Arc::new(Mutex::new(CandidateBuffer::new(1 << 20))),
|
||
sampler_thread: None,
|
||
_recording: crate::retention::Recording::start(),
|
||
}
|
||
.stop()
|
||
.no_screen
|
||
};
|
||
let failure = "the screen sharing stream sent no frames";
|
||
assert_eq!(stop_with(Some(failure)).as_deref(), Some(failure));
|
||
let reason = stop_with(None).expect("a stopped capture with no frame still says why");
|
||
assert_ne!(reason, failure);
|
||
#[cfg(target_os = "linux")]
|
||
assert_eq!(reason, "the screen recorded no frames");
|
||
}
|
||
|
||
/// Clicks on `scrawl-helper`'s windows (the SwiftUI capsule, Settings, onboarding) are
|
||
/// Scrawl's own, except on the iPhone mirror, which is what's being recorded. The host's
|
||
/// chrome takes clicks unless it lets them through: Hex's capsule is a 640×424 panel, named
|
||
/// and click-through (only its pill, which Hex checks itself, takes clicks).
|
||
#[test]
|
||
fn helper_windows_are_scrawl_s_except_the_capture_source() {
|
||
let (sender, _receiver) = flume::unbounded();
|
||
let telemetry = SharedCaptureTelemetry::new(sender, ShortcutConfig::ansi_defaults());
|
||
let window = |id, pid, bounds| crate::windows::RawWindow {
|
||
id,
|
||
pid,
|
||
owner: String::new(),
|
||
title: String::new(),
|
||
bounds,
|
||
layer: 0,
|
||
alpha: 1.0,
|
||
};
|
||
let own = std::process::id() as i32;
|
||
let (settings, mirror, app) = (own + 1, own + 2, own + 3);
|
||
let windows = [
|
||
crate::windows::RawWindow {
|
||
layer: 26,
|
||
..window(1, own, [0.0, 0.0, 1728.0, 1117.0]) // the ink overlay, click-through
|
||
},
|
||
crate::windows::RawWindow {
|
||
layer: 27,
|
||
..window(5, own, [544.0, 0.0, 640.0, 424.0]) // the host's capsule panel
|
||
},
|
||
crate::windows::RawWindow {
|
||
layer: 25,
|
||
..window(7, own, [700.0, 1000.0, 320.0, 44.0]) // a HUD of the host's
|
||
},
|
||
window(6, own, [1300.0, 900.0, 400.0, 200.0]), // the host's own Settings window
|
||
window(2, settings, [100.0, 100.0, 600.0, 400.0]),
|
||
window(3, mirror, [1300.0, 100.0, 300.0, 650.0]),
|
||
window(4, app, [0.0, 0.0, 1728.0, 1117.0]),
|
||
];
|
||
*telemetry.excluded_window_ids.lock().unwrap() = vec![5, 1];
|
||
*telemetry.passthrough_window_ids.lock().unwrap() = vec![1, 5];
|
||
telemetry.set_capture_source_pid(mirror as u32);
|
||
let helper = |pid| pid == settings || pid == mirror;
|
||
let at = |x, y| telemetry.scrawl_window_in(&windows, x, y, helper);
|
||
assert!(at(300.0, 300.0), "Settings");
|
||
assert!(!at(1400.0, 400.0), "the iPhone mirror");
|
||
assert!(!at(1000.0, 900.0), "the app under the ink overlay");
|
||
assert!(!at(1100.0, 300.0), "the app under the capsule's panel");
|
||
assert!(at(800.0, 1020.0), "the host's HUD, unnamed");
|
||
telemetry
|
||
.passthrough_window_ids
|
||
.lock()
|
||
.unwrap()
|
||
.retain(|id| *id != 5);
|
||
assert!(at(1100.0, 300.0), "a named window that takes clicks");
|
||
assert!(
|
||
!at(1500.0, 1000.0),
|
||
"the host's Settings, which can be what's shown"
|
||
);
|
||
telemetry.set_capture_source_pid(0);
|
||
assert!(at(1400.0, 400.0), "a helper that isn't the source");
|
||
}
|
||
|
||
/// A screen capture on a laptop: the old listen-only event tap was off (no Input
|
||
/// Monitoring), so his clicks on OpenCode's composer never reached the bundle. Clicks now
|
||
/// come from the mouse monitor; polled button state stands in if it couldn't be installed.
|
||
#[test]
|
||
fn clicks_are_recorded_from_polled_buttons_while_the_mouse_monitor_is_missing() {
|
||
let (sender, _receiver) = flume::unbounded();
|
||
let telemetry = SharedCaptureTelemetry::new(sender, ShortcutConfig::ansi_defaults());
|
||
telemetry.reset_for_session(TargetFrame {
|
||
x: 0.0,
|
||
y: 0.0,
|
||
width: 1728.0,
|
||
height: 1117.0,
|
||
scale: 2.0,
|
||
});
|
||
assert!(!telemetry.mouse_live.load(Ordering::Acquire));
|
||
let mut prev = [false; 2];
|
||
let not_scrawl = |_: f64, _: f64| false;
|
||
let composer = (778.0, 1050.0);
|
||
// A ⇧-click is a pin, not a click.
|
||
poll_clicks(
|
||
&telemetry,
|
||
&mut prev,
|
||
[true, false],
|
||
composer,
|
||
true,
|
||
not_scrawl,
|
||
);
|
||
poll_clicks(
|
||
&telemetry,
|
||
&mut prev,
|
||
[false, false],
|
||
composer,
|
||
true,
|
||
not_scrawl,
|
||
);
|
||
// A click on the composer.
|
||
poll_clicks(
|
||
&telemetry,
|
||
&mut prev,
|
||
[true, false],
|
||
composer,
|
||
false,
|
||
not_scrawl,
|
||
);
|
||
poll_clicks(
|
||
&telemetry,
|
||
&mut prev,
|
||
[true, false],
|
||
composer,
|
||
false,
|
||
not_scrawl,
|
||
);
|
||
poll_clicks(
|
||
&telemetry,
|
||
&mut prev,
|
||
[false, false],
|
||
composer,
|
||
false,
|
||
not_scrawl,
|
||
);
|
||
// Then the stop button, which is Scrawl's own.
|
||
let stop = (860.0, 20.0);
|
||
poll_clicks(&telemetry, &mut prev, [true, false], stop, false, |_, _| {
|
||
true
|
||
});
|
||
poll_clicks(
|
||
&telemetry,
|
||
&mut prev,
|
||
[false, false],
|
||
stop,
|
||
false,
|
||
|_, _| true,
|
||
);
|
||
// While the mouse monitor is installed it reports clicks itself.
|
||
telemetry.mouse_live.store(true, Ordering::Release);
|
||
poll_clicks(
|
||
&telemetry,
|
||
&mut prev,
|
||
[true, false],
|
||
composer,
|
||
false,
|
||
not_scrawl,
|
||
);
|
||
poll_clicks(
|
||
&telemetry,
|
||
&mut prev,
|
||
[false, false],
|
||
composer,
|
||
false,
|
||
not_scrawl,
|
||
);
|
||
|
||
let (events, _) = telemetry.finish_session();
|
||
assert_eq!(events.len(), 1, "{events:?}");
|
||
let InteractionEvent::Click { x, y, button, .. } = &events[0] else {
|
||
panic!("{events:?}");
|
||
};
|
||
assert!(
|
||
(x - 0.45).abs() < 0.01 && (y - 0.94).abs() < 0.01,
|
||
"{x}, {y}"
|
||
);
|
||
assert_eq!(button.as_deref(), Some("left"));
|
||
}
|
||
|
||
#[test]
|
||
fn the_capture_shortcut_matches_its_aliases_and_a_custom_one() {
|
||
let s = 1_u16; // the S key on this layout
|
||
let default = ShortcutConfig {
|
||
key_code: u32::from(s),
|
||
carbon_modifiers: CARBON_OPTION_KEY | CARBON_SHIFT_KEY,
|
||
label: "⌥⇧S".into(),
|
||
};
|
||
// (cmd, ctrl, alt, shift)
|
||
let hit = |cfg: &ShortcutConfig, key, (cmd, ctrl, alt, shift)| {
|
||
is_capture_shortcut(cfg, s, key, cmd, ctrl, alt, shift)
|
||
};
|
||
assert!(hit(&default, s, (false, false, true, true)), "⌥⇧S");
|
||
assert!(hit(&default, s, (true, false, true, true)), "⌥⇧⌘S");
|
||
assert!(hit(&default, s, (false, true, true, true)), "⌃⌥⇧S");
|
||
assert!(hit(&default, s, (true, true, true, false)), "⌃⌥⌘S");
|
||
assert!(hit(&default, s, (true, true, true, true)), "hyper S");
|
||
assert!(hit(&default, KEY_S, (false, false, true, true)), "ANSI S");
|
||
assert!(!hit(&default, s, (false, false, false, true)), "⇧S");
|
||
assert!(!hit(&default, s, (true, false, false, false)), "⌘S");
|
||
assert!(!hit(&default, 2, (false, false, true, true)), "⌥⇧D");
|
||
|
||
let custom = ShortcutConfig {
|
||
key_code: 15,
|
||
carbon_modifiers: CARBON_CONTROL_KEY | CARBON_OPTION_KEY,
|
||
label: "⌃⌥R".into(),
|
||
};
|
||
assert!(hit(&custom, 15, (false, true, true, false)));
|
||
assert!(!hit(&custom, 15, (true, true, true, false)));
|
||
assert!(!hit(&custom, 15, (false, false, true, false)));
|
||
// Once changed, ⌥⇧S and its aliases are free for other apps.
|
||
assert!(!hit(&custom, s, (false, false, true, true)), "⌥⇧S");
|
||
assert!(!hit(&custom, KEY_S, (true, true, true, true)), "hyper S");
|
||
assert_eq!(
|
||
hotkey_combos(ShortcutKind::Record, &custom),
|
||
[(15, custom.carbon_modifiers)]
|
||
);
|
||
assert_eq!(
|
||
hotkey_combos(ShortcutKind::Record, &default).len(),
|
||
if u32::from(KEY_S) == default.key_code {
|
||
4
|
||
} else {
|
||
8
|
||
},
|
||
"⌥⇧S and three aliases, on the layout's key and the ANSI one"
|
||
);
|
||
|
||
// The paste shortcut works the same way.
|
||
let v = 9_u16;
|
||
let paste = ShortcutConfig {
|
||
key_code: u32::from(v),
|
||
carbon_modifiers: CARBON_OPTION_KEY | CARBON_SHIFT_KEY,
|
||
label: "⌥⇧V".into(),
|
||
};
|
||
let paste_hit = |cfg: &ShortcutConfig, key, (cmd, ctrl, alt, shift)| {
|
||
shortcut_matches(ShortcutKind::Paste, cfg, v, key, cmd, ctrl, alt, shift)
|
||
};
|
||
assert!(paste_hit(&paste, v, (false, false, true, true)), "⌥⇧V");
|
||
assert!(paste_hit(&paste, v, (true, true, true, false)), "⌃⌥⌘V");
|
||
assert!(
|
||
!paste_hit(&paste, s, (false, false, true, true)),
|
||
"⌥⇧S isn't paste"
|
||
);
|
||
let paste_custom = ShortcutConfig {
|
||
key_code: 35,
|
||
carbon_modifiers: CARBON_CONTROL_KEY | CARBON_OPTION_KEY,
|
||
label: "⌃⌥P".into(),
|
||
};
|
||
assert!(paste_hit(&paste_custom, 35, (false, true, true, false)));
|
||
assert!(!paste_hit(&paste_custom, v, (false, false, true, true)));
|
||
assert!(
|
||
paste.is_default_for(ShortcutKind::Paste)
|
||
&& !paste.is_default_for(ShortcutKind::Record)
|
||
);
|
||
#[cfg(target_os = "macos")]
|
||
assert_eq!(
|
||
modifier_symbols(CARBON_CMD_KEY | CARBON_CONTROL_KEY | CARBON_SHIFT_KEY),
|
||
"⌃⇧⌘"
|
||
);
|
||
#[cfg(target_os = "windows")]
|
||
assert_eq!(
|
||
modifier_symbols(CARBON_CMD_KEY | CARBON_CONTROL_KEY | CARBON_SHIFT_KEY),
|
||
"Ctrl+Win+Shift+"
|
||
);
|
||
|
||
assert_eq!(shortcut_parts("⌥⇧S"), ("⌥⇧", "S"));
|
||
assert_eq!(shortcut_parts("⌃⌥⌘R"), ("⌃⌥⌘", "R"));
|
||
assert_eq!(shortcut_parts("⌥⇧É"), ("⌥⇧", "É"));
|
||
assert_eq!(shortcut_parts("Alt+Shift+S"), ("Alt+Shift", "S"));
|
||
assert_eq!(shortcut_parts("Ctrl+Alt+Ä"), ("Ctrl+Alt", "Ä"));
|
||
assert_eq!(shortcut_parts("Ctrl++"), ("Ctrl", "+"));
|
||
assert_eq!(shortcut_parts(""), ("", ""));
|
||
}
|
||
|
||
#[test]
|
||
fn recordings_use_the_display_under_the_cursor() {
|
||
let displays = [[0.0, 0.0, 1728.0, 1117.0], [1728.0, -200.0, 2560.0, 1440.0]];
|
||
assert_eq!(pick_display(&displays, Some([2000.0, 300.0]), None), 1);
|
||
assert_eq!(pick_display(&displays, Some([100.0, 100.0]), None), 0);
|
||
// No cursor: the display holding the focused window's centre.
|
||
let focused = Some([1900.0, 0.0, 800.0, 600.0]);
|
||
assert_eq!(pick_display(&displays, None, focused), 1);
|
||
assert_eq!(pick_display(&displays, Some([-50.0, 0.0]), focused), 1);
|
||
assert_eq!(pick_display(&displays, None, None), 0);
|
||
// A display left of the laptop, at negative x.
|
||
let left = [[0.0, 0.0, 1728.0, 1117.0], [-1920.0, 37.0, 1920.0, 1080.0]];
|
||
assert_eq!(pick_display(&left, Some([-5.0, 600.0]), None), 1);
|
||
// One above it, at negative y.
|
||
let above = [[0.0, 0.0, 1728.0, 1117.0], [0.0, -1440.0, 2560.0, 1440.0]];
|
||
assert_eq!(pick_display(&above, Some([10.0, -1.0]), None), 1);
|
||
assert_eq!(pick_display(&above, Some([10.0, 0.0]), None), 0);
|
||
}
|
||
|
||
/// Clicks and ink on a display left of, right of or above the laptop normalize into that
|
||
/// display, and land on the recording's pixels at its own scale.
|
||
#[test]
|
||
fn input_on_an_external_display_normalizes_into_it() {
|
||
let (sender, _receiver) = flume::unbounded();
|
||
let telemetry = SharedCaptureTelemetry::new(sender, ShortcutConfig::ansi_defaults());
|
||
let left = [-1920.0, 37.0, 1920.0, 1080.0];
|
||
let right = [1728.0, 0.0, 1920.0, 1080.0];
|
||
let above = [0.0, -1440.0, 2560.0, 1440.0];
|
||
let small_left = [-1280.0, -100.0, 1280.0, 800.0];
|
||
let below = [0.0, 1117.0, 1920.0, 1080.0];
|
||
// (display, scale, global point, expected normalized point, recording size)
|
||
let cases = [
|
||
// 1x displays are recorded at their own pixels
|
||
(left, 1.0, [-960.0, 577.0], (0.5, 0.5), (1920, 1080)),
|
||
(right, 1.0, [2208.0, 270.0], (0.25, 0.25), (1920, 1080)),
|
||
(below, 1.0, [480.0, 1387.0], (0.25, 0.25), (1920, 1080)),
|
||
// 2x ones at the 2560 px long edge
|
||
(above, 2.0, [640.0, -360.0], (0.25, 0.75), (2560, 1440)),
|
||
(small_left, 2.0, [-320.0, 500.0], (0.75, 0.75), (2560, 1600)),
|
||
];
|
||
for (rect, scale, point, expected, size) in cases {
|
||
let frame = crate::screens::with_scale(rect, scale);
|
||
let target = TargetInfo {
|
||
mode: CaptureMode::Display,
|
||
app: None,
|
||
bundle_id: None,
|
||
window_title: None,
|
||
window_id: None,
|
||
display_id: 2,
|
||
frame,
|
||
};
|
||
// Windows frames are pixels already and never upscaled.
|
||
if cfg!(target_os = "macos") {
|
||
assert_eq!(capture_size(&target, 2560.0), size, "{rect:?}");
|
||
}
|
||
telemetry.reset_for_session(frame);
|
||
assert!(telemetry.contains_screen_point(point[0], point[1]));
|
||
// The laptop's centre is outside this target.
|
||
assert!(!telemetry.contains_screen_point(864.0, 558.0));
|
||
let (nx, ny, fw, fh) = telemetry.normalize_point(point[0], point[1]);
|
||
assert_eq!((nx, ny), expected, "{rect:?}");
|
||
assert_eq!((fw, fh), (rect[2] as u32, rect[3] as u32));
|
||
// A click there is recorded at those coordinates.
|
||
telemetry.pointer_down(10, point[0], point[1]);
|
||
telemetry.pointer_up(20, point[0], point[1], false);
|
||
let (events, _) = telemetry.finish_session();
|
||
assert!(
|
||
matches!(events[..], [InteractionEvent::Click { x, y, .. }] if (x, y) == (nx, ny)),
|
||
"{events:?}"
|
||
);
|
||
}
|
||
// In recording pixels: the left 1x display's centre is (960, 540).
|
||
let frame = crate::screens::with_scale([-1920.0, 37.0, 1920.0, 1080.0], 1.0);
|
||
telemetry.reset_for_session(frame);
|
||
let (nx, ny, _, _) = telemetry.normalize_point(-960.0, 577.0);
|
||
assert_eq!((nx * 1920.0, ny * 1080.0), (960.0, 540.0));
|
||
// Above 2x: (640, -360) is a quarter in → (640, 1080) px of 2560×1440.
|
||
let frame = crate::screens::with_scale([0.0, -1440.0, 2560.0, 1440.0], 2.0);
|
||
telemetry.reset_for_session(frame);
|
||
let (nx, ny, _, _) = telemetry.normalize_point(640.0, -360.0);
|
||
assert_eq!((nx * 2560.0, ny * 1440.0), (640.0, 1080.0));
|
||
telemetry.finish_session();
|
||
}
|
||
|
||
fn session(width: f64, height: f64) -> Arc<SharedCaptureTelemetry> {
|
||
let (sender, _receiver) = flume::unbounded();
|
||
let telemetry = SharedCaptureTelemetry::new(sender, ShortcutConfig::ansi_defaults());
|
||
telemetry.reset_for_session(TargetFrame {
|
||
x: 0.0,
|
||
y: 0.0,
|
||
width,
|
||
height,
|
||
scale: 2.0,
|
||
});
|
||
telemetry
|
||
}
|
||
|
||
/// The poller draws ink while ⇧ is held. A ⇧-click is part of the drawing, never a click:
|
||
/// the stroke carries on whole, and a ⇧-click with nothing drawn (only its own jitter)
|
||
/// leaves no stroke once ⇧ comes up.
|
||
#[test]
|
||
fn a_shift_click_is_part_of_the_drawing() {
|
||
let telemetry = session(1000.0, 1000.0);
|
||
let ink = |pts: &[(i32, i32, i32)]| Stroke {
|
||
t0: 0,
|
||
t1: 0,
|
||
fw: 1000,
|
||
fh: 1000,
|
||
on: None,
|
||
pin: None,
|
||
laser: false,
|
||
pts: pts.iter().flat_map(|&(x, y, t)| [x, y, t]).collect(),
|
||
};
|
||
let press = telemetry.elapsed_ms() + 50;
|
||
let p = press as i32;
|
||
let s = SCALE;
|
||
// A line drawn while ⇧ was held, then a click at its end.
|
||
let line = ink(&[
|
||
(s / 10, s / 2, 0),
|
||
(s / 5, s / 2, p - 20),
|
||
(s / 5 + s / 500, s / 2, p + 5),
|
||
]);
|
||
*telemetry.live_ink.lock().unwrap() = Some(line.clone());
|
||
telemetry.shift_down(press, 200.0, 500.0);
|
||
assert!(telemetry.shift_up());
|
||
assert_eq!(telemetry.live_ink.lock().unwrap().as_ref(), Some(&line));
|
||
// Letting go of ⇧ commits it as one stroke.
|
||
let at = (200.0, 500.0);
|
||
track_shift_ink(&telemetry, at, false, true);
|
||
|
||
// Drawn long before the press still counts: the hold is a drawing.
|
||
*telemetry.live_ink.lock().unwrap() = Some(ink(&[(s / 10, s / 10, 0), (s / 5, s / 10, 1)]));
|
||
let t = telemetry.elapsed_ms() + 5_000;
|
||
telemetry.shift_down(t, 200.0, 100.0);
|
||
assert!(telemetry.shift_up());
|
||
*telemetry.live_ink.lock().unwrap() = None;
|
||
|
||
// A ⇧-click with only its own few points of jitter leaves nothing.
|
||
let jitter = ink(&[(s / 2, s / 2, 0), (s / 2 + 2, s / 2, 30)]);
|
||
*telemetry.live_ink.lock().unwrap() = Some(jitter);
|
||
let t = telemetry.elapsed_ms();
|
||
telemetry.shift_down(t, 500.0, 500.0);
|
||
assert!(telemetry.shift_up());
|
||
assert!(!telemetry.shift_up(), "one press, one release");
|
||
track_shift_ink(&telemetry, (500.0, 500.0), false, true);
|
||
|
||
let (events, strokes) = telemetry.finish_session();
|
||
assert!(events.is_empty(), "{events:?}");
|
||
let kinds: Vec<_> = strokes.iter().map(|s| (s.pin, s.pts.len() / 3)).collect();
|
||
assert_eq!(kinds, vec![(None, 3)]);
|
||
}
|
||
|
||
#[test]
|
||
fn shift_clicks_from_polled_buttons_are_neither_clicks_nor_pins() {
|
||
let telemetry = session(1000.0, 1000.0);
|
||
let mut prev = [false; 2];
|
||
// ⇧-clicks at a point, the second on Scrawl's own window.
|
||
for (at, on_scrawl) in [((250.0, 750.0), false), ((400.0, 400.0), true)] {
|
||
for buttons in [[true, false], [false, false]] {
|
||
poll_clicks(&telemetry, &mut prev, buttons, at, true, |_, _| on_scrawl);
|
||
}
|
||
}
|
||
let (events, strokes) = telemetry.finish_session();
|
||
assert!(events.is_empty(), "{events:?}");
|
||
assert!(strokes.is_empty(), "{strokes:?}");
|
||
}
|
||
|
||
#[test]
|
||
fn input_sink_routes_clicks_ink_and_chord_release() {
|
||
let telemetry = session(1000.0, 1000.0);
|
||
let sink = InputSink::new(Arc::clone(&telemetry));
|
||
|
||
// Residual Shift from a multi-modifier chord (⌥⇧S) is ignored until Shift is released.
|
||
let mut await_release = false;
|
||
assert!(!pure_shift_state(true, true, &mut await_release));
|
||
assert!(!pure_shift_state(true, false, &mut await_release));
|
||
assert!(!pure_shift_state(false, false, &mut await_release));
|
||
assert!(pure_shift_state(true, false, &mut await_release));
|
||
|
||
// Shift-ink stroke.
|
||
sink.ink(100, 100.0, 200.0, true);
|
||
sink.ink(120, 140.0, 200.0, true);
|
||
sink.ink(140, 180.0, 200.0, true);
|
||
assert!(sink.snapshot().live.is_some());
|
||
sink.ink(160, 180.0, 200.0, false);
|
||
assert_eq!(sink.snapshot().strokes.len(), 1);
|
||
|
||
// A Shift-click (pointer_down/up while shift_held) is neither a click nor a pin.
|
||
sink.ink(200, 500.0, 500.0, true);
|
||
sink.pointer_down(210, 500.0, 500.0);
|
||
sink.pointer_up(240, 500.0, 500.0, false);
|
||
sink.ink(250, 500.0, 500.0, false);
|
||
assert_eq!(sink.snapshot().strokes.len(), 1);
|
||
|
||
// Key chord and flag.
|
||
sink.key_chord(300, "⌘K");
|
||
sink.flag(350);
|
||
|
||
let flags = telemetry.take_flags();
|
||
let (events, strokes) = telemetry.finish_session();
|
||
assert_eq!(flags, vec![350]);
|
||
assert_eq!(strokes.len(), 1);
|
||
assert_eq!(strokes[0].pin, None);
|
||
assert!(matches!(
|
||
&events[..],
|
||
[InteractionEvent::Key {
|
||
t: 300,
|
||
shortcut
|
||
}] if shortcut == "⌘K"
|
||
));
|
||
}
|
||
|
||
#[test]
|
||
fn capture_finish_writes_bundle_and_emits_progress_with_host_and_transcriber() {
|
||
struct TestHost {
|
||
stages: Mutex<Vec<Stage>>,
|
||
}
|
||
impl Host for TestHost {
|
||
fn emit(&self, event: Event) {
|
||
if let Event::Progress { stage, .. } = event {
|
||
self.stages.lock().unwrap().push(stage);
|
||
}
|
||
}
|
||
}
|
||
struct TestTranscriber;
|
||
impl crate::transcribe::Transcriber for TestTranscriber {
|
||
fn transcribe(
|
||
&self,
|
||
_audio: &crate::audio::RecordedAudio,
|
||
) -> Result<Option<crate::model::TranscriptData>> {
|
||
Ok(Some(crate::model::TranscriptData {
|
||
source: "test".into(),
|
||
model: Some("parakeet".into()),
|
||
text: "hello world".into(),
|
||
words: vec![
|
||
crate::model::WordTiming {
|
||
t0: 0,
|
||
t1: 300,
|
||
text: "hello".into(),
|
||
},
|
||
crate::model::WordTiming {
|
||
t0: 300,
|
||
t1: 600,
|
||
text: "world".into(),
|
||
},
|
||
],
|
||
segments: Vec::new(),
|
||
estimated: false,
|
||
missing: None,
|
||
}))
|
||
}
|
||
}
|
||
|
||
let dir = std::env::temp_dir().join(format!(
|
||
"scrawl-capture-test-{}.scrawl",
|
||
crate::pipeline::make_bundle_id()
|
||
));
|
||
std::fs::create_dir_all(&dir).unwrap();
|
||
let host = Arc::new(TestHost {
|
||
stages: Mutex::new(Vec::new()),
|
||
});
|
||
let telemetry = session(200.0, 100.0);
|
||
let mut buf = CandidateBuffer::new(1 << 20);
|
||
buf.push(RgbaFrame::new(0, 20, 10, vec![40; 20 * 10 * 4]), false);
|
||
buf.push(RgbaFrame::new(500, 20, 10, vec![80; 20 * 10 * 4]), true);
|
||
let screen_session = ScreenCaptureSession {
|
||
failure: Arc::default(),
|
||
started_at: Instant::now(),
|
||
stopping: Arc::new(AtomicBool::new(false)),
|
||
urgent_snapshot: Arc::new(AtomicBool::new(false)),
|
||
frames: Arc::new(Mutex::new(buf)),
|
||
sampler_thread: None,
|
||
_recording: crate::retention::Recording::start(),
|
||
};
|
||
let target = TargetInfo {
|
||
mode: CaptureMode::Window,
|
||
app: Some("TestApp".into()),
|
||
bundle_id: Some("dev.test.app".into()),
|
||
window_title: Some("Main".into()),
|
||
window_id: Some(1),
|
||
display_id: 1,
|
||
frame: TargetFrame {
|
||
x: 0.0,
|
||
y: 0.0,
|
||
width: 200.0,
|
||
height: 100.0,
|
||
scale: 1.0,
|
||
},
|
||
};
|
||
let session_id = telemetry.session_id.load(Ordering::Acquire);
|
||
let capture = Capture {
|
||
host: host.clone(),
|
||
transcriber: Some(Arc::new(TestTranscriber)),
|
||
engine_cfg: crate::handoff::EngineConfig::default(),
|
||
input: InputSink::new(Arc::clone(&telemetry)),
|
||
telemetry,
|
||
session_id,
|
||
dir: dir.clone(),
|
||
target,
|
||
t0_boot_ns: 123_456_789,
|
||
note: Some("unit test".into()),
|
||
host_narration: false,
|
||
source: InnerSource::Screen {
|
||
session: screen_session,
|
||
audio: None,
|
||
windows: None,
|
||
},
|
||
stopped_input: None,
|
||
};
|
||
assert_eq!(capture.started_at_boot_ns(), 123_456_789);
|
||
capture.input().pointer_down(100, 50.0, 50.0);
|
||
capture.input().pointer_up(140, 50.0, 50.0, false);
|
||
|
||
let audio = crate::audio::RecordedAudio::new(vec![0.0; 16_000], 16_000, 200, Vec::new());
|
||
let finished = capture.finish(Some(audio)).unwrap();
|
||
assert_eq!(finished.dir, dir);
|
||
assert_eq!(finished.words, 2);
|
||
assert_eq!(
|
||
finished.route_hint.captured_bundle_id.as_deref(),
|
||
Some("dev.test.app")
|
||
);
|
||
assert!(dir.join("scrawl.json").is_file());
|
||
assert!(dir.join("summary.md").is_file());
|
||
assert!(dir.join("audio.wav").is_file());
|
||
assert_eq!(
|
||
*host.stages.lock().unwrap(),
|
||
vec![Stage::Stopping, Stage::Transcribing, Stage::Done]
|
||
);
|
||
let _ = std::fs::remove_dir_all(&dir);
|
||
}
|
||
|
||
#[test]
|
||
fn seal_stops_recording_immediately_and_captures_do_not_clobber_newer_telemetry() {
|
||
let dir1 = std::env::temp_dir().join(format!(
|
||
"scrawl-seal-1-{}.scrawl",
|
||
crate::pipeline::make_bundle_id()
|
||
));
|
||
let dir2 = std::env::temp_dir().join(format!(
|
||
"scrawl-seal-2-{}.scrawl",
|
||
crate::pipeline::make_bundle_id()
|
||
));
|
||
std::fs::create_dir_all(&dir1).unwrap();
|
||
std::fs::create_dir_all(&dir2).unwrap();
|
||
|
||
// Building an Engine on a background worker thread must not touch TIS/TSM.
|
||
let engine = std::thread::spawn(Engine::default_engine).join().unwrap();
|
||
assert_eq!(
|
||
engine.telemetry().shortcuts(),
|
||
ShortcutConfig::ansi_defaults()
|
||
);
|
||
|
||
let shared = session(200.0, 100.0);
|
||
let s1_id = shared.session_id.load(Ordering::Acquire);
|
||
let stopping1 = Arc::new(AtomicBool::new(false));
|
||
let mut buf1 = CandidateBuffer::new(1 << 20);
|
||
buf1.push(RgbaFrame::new(0, 20, 10, vec![40; 20 * 10 * 4]), false);
|
||
let target = TargetInfo {
|
||
mode: CaptureMode::Window,
|
||
app: Some("TestApp".into()),
|
||
bundle_id: Some("dev.test.app".into()),
|
||
window_title: Some("Main".into()),
|
||
window_id: Some(1),
|
||
display_id: 1,
|
||
frame: TargetFrame {
|
||
x: 0.0,
|
||
y: 0.0,
|
||
width: 200.0,
|
||
height: 100.0,
|
||
scale: 1.0,
|
||
},
|
||
};
|
||
let (tx, _rx) = flume::unbounded();
|
||
let host: Arc<dyn Host> = Arc::new(ChannelHost::new(tx));
|
||
let mut cfg1 = crate::handoff::EngineConfig::default();
|
||
cfg1.experiments
|
||
.insert(crate::handoff::AGENT_CONTEXT.into(), true);
|
||
let c1 = Capture {
|
||
host: Arc::clone(&host),
|
||
transcriber: None,
|
||
engine_cfg: cfg1,
|
||
input: InputSink::new(Arc::clone(&shared)),
|
||
telemetry: Arc::clone(&shared),
|
||
session_id: s1_id,
|
||
dir: dir1.clone(),
|
||
target: target.clone(),
|
||
t0_boot_ns: 100,
|
||
note: None,
|
||
host_narration: false,
|
||
source: InnerSource::Screen {
|
||
session: ScreenCaptureSession {
|
||
failure: Arc::default(),
|
||
started_at: Instant::now(),
|
||
stopping: Arc::clone(&stopping1),
|
||
urgent_snapshot: Arc::new(AtomicBool::new(false)),
|
||
frames: Arc::new(Mutex::new(buf1)),
|
||
sampler_thread: None,
|
||
_recording: crate::retention::Recording::start(),
|
||
},
|
||
audio: None,
|
||
windows: None,
|
||
},
|
||
stopped_input: None,
|
||
};
|
||
c1.input().pointer_down(50, 10.0, 10.0);
|
||
c1.input().pointer_up(80, 10.0, 10.0, false);
|
||
|
||
// Seal c1: stops its ScreenCaptureSession and input immediately.
|
||
let sealed1 = c1.seal();
|
||
assert!(stopping1.load(Ordering::Acquire));
|
||
|
||
// Start a second session on the same telemetry before sealed1 finishes (or if an unsealed
|
||
// capture is cancelled after session 2 starts).
|
||
let s2_id = shared.reset_for_session_at_with_agent_context(target.frame, Some(200), false);
|
||
assert!(s2_id > s1_id);
|
||
shared.flag_at(333);
|
||
let stale = Capture {
|
||
host: Arc::clone(&host),
|
||
transcriber: None,
|
||
engine_cfg: crate::handoff::EngineConfig::default(),
|
||
input: InputSink::new(Arc::clone(&shared)),
|
||
telemetry: Arc::clone(&shared),
|
||
session_id: s1_id,
|
||
dir: dir2.clone(),
|
||
target,
|
||
t0_boot_ns: 100,
|
||
note: None,
|
||
host_narration: false,
|
||
source: InnerSource::Screen {
|
||
session: ScreenCaptureSession {
|
||
failure: Arc::default(),
|
||
started_at: Instant::now(),
|
||
stopping: Arc::new(AtomicBool::new(false)),
|
||
urgent_snapshot: Arc::new(AtomicBool::new(false)),
|
||
frames: Arc::new(Mutex::new(CandidateBuffer::new(1 << 20))),
|
||
sampler_thread: None,
|
||
_recording: crate::retention::Recording::start(),
|
||
},
|
||
audio: None,
|
||
windows: None,
|
||
},
|
||
stopped_input: None,
|
||
};
|
||
// Cancelling the stale capture must not clear session 2's active state or flags.
|
||
stale.cancel();
|
||
assert!(shared.active.load(Ordering::Acquire));
|
||
assert_eq!(shared.take_flags(), vec![333]);
|
||
|
||
// Finishing sealed1 later produces its bundle with the click recorded before seal() and
|
||
// honors c1's ConfigSource-supplied agentContext setting rather than global handoff state.
|
||
assert!(!crate::handoff::experiment(crate::handoff::AGENT_CONTEXT));
|
||
let finished1 = sealed1.finish(None).unwrap();
|
||
assert_eq!(finished1.bundle.events.len(), 1);
|
||
assert!(
|
||
finished1.summary.contains("· 0:00 clicked"),
|
||
"summary: {}",
|
||
finished1.summary
|
||
);
|
||
assert_eq!(finished1.gaps, 0);
|
||
assert_eq!(finished1.missing_transcript.as_deref(), Some("no_audio"));
|
||
let _ = std::fs::remove_dir_all(&dir1);
|
||
let _ = std::fs::remove_dir_all(&dir2);
|
||
}
|
||
|
||
#[test]
|
||
fn candidate_buffer_stays_bounded_and_keeps_the_whole_timeline() {
|
||
// A 1 MB budget of 4 KB frames: a 256-frame cap.
|
||
let mut buffer = CandidateBuffer::new(1 << 20);
|
||
let frame = |t| RgbaFrame::new(t, 32, 32, vec![0; 32 * 32 * 4]);
|
||
let mut t = 0;
|
||
let mut peak = 0;
|
||
// Three hours at the sampler's pace, with a click frame every 10 s.
|
||
while t < 3 * 3_600_000 {
|
||
buffer.push(frame(t), t % 10_000 == 0);
|
||
peak = peak.max(buffer.len());
|
||
t += buffer.interval().as_millis() as u64;
|
||
}
|
||
assert!(peak <= 256, "peak {peak}");
|
||
assert!(buffer.interval() >= FRAME_SAMPLE_INTERVAL * 64);
|
||
let frames = buffer.into_frames();
|
||
assert_eq!(frames[0].t_ms, 0, "the start is kept");
|
||
assert!(frames.windows(2).all(|w| w[0].t_ms < w[1].t_ms));
|
||
// Spread over the whole capture, not bunched at the end.
|
||
let first_half = frames.iter().filter(|f| f.t_ms < 3 * 1_800_000).count();
|
||
assert!(
|
||
first_half * 3 > frames.len(),
|
||
"{first_half} of {}",
|
||
frames.len()
|
||
);
|
||
}
|
||
|
||
#[test]
|
||
fn candidate_buffer_keeps_urgent_frames_when_thinning() {
|
||
let mut buffer = CandidateBuffer::new(40 * 4096);
|
||
let frame = |t| RgbaFrame::new(t, 32, 32, vec![0; 32 * 32 * 4]);
|
||
for t in 0..60 {
|
||
buffer.push(frame(t * 220), t == 33);
|
||
}
|
||
assert!(buffer.len() < 60);
|
||
assert_eq!(buffer.interval(), FRAME_SAMPLE_INTERVAL * 2);
|
||
assert!(buffer.into_frames().iter().any(|f| f.t_ms == 33 * 220));
|
||
}
|
||
|
||
#[test]
|
||
fn frames_that_may_show_a_system_prompt_are_dropped_but_never_the_first() {
|
||
let mut buffer = CandidateBuffer::new(1 << 20);
|
||
for t in [0, 220, 440, 660, 880] {
|
||
buffer.push(RgbaFrame::new(t, 4, 4, vec![0; 64]), false);
|
||
}
|
||
buffer.drop_since(440);
|
||
let times: Vec<u64> = buffer.into_frames().iter().map(|f| f.t_ms).collect();
|
||
assert_eq!(times, [0, 220, 440]);
|
||
let mut buffer = CandidateBuffer::new(1 << 20);
|
||
buffer.push(RgbaFrame::new(500, 4, 4, vec![0; 64]), false);
|
||
buffer.drop_since(0);
|
||
assert_eq!(buffer.len(), 1);
|
||
}
|
||
|
||
#[test]
|
||
fn candidates_are_small_and_recordings_keep_full_size() {
|
||
let target = |mode, width, height| TargetInfo {
|
||
mode,
|
||
app: None,
|
||
bundle_id: None,
|
||
window_title: None,
|
||
window_id: None,
|
||
display_id: 1,
|
||
frame: TargetFrame {
|
||
x: 0.0,
|
||
y: 0.0,
|
||
width,
|
||
height,
|
||
scale: 2.0,
|
||
},
|
||
};
|
||
let display = target(CaptureMode::Display, 1728.0, 1117.0);
|
||
assert_eq!(capture_size(&display, CANDIDATE_EDGE), (480, 310));
|
||
let region = target(CaptureMode::Region, 480.0, 320.0);
|
||
assert_eq!(capture_size(®ion, CANDIDATE_EDGE), (480, 320));
|
||
if cfg!(any(target_os = "windows", target_os = "linux")) {
|
||
// Frames are in pixels: never upscaled past the screen's own.
|
||
assert_eq!(capture_size(&display, 2560.0), (1728, 1116));
|
||
assert_eq!(capture_size(®ion, 1280.0), (480, 320));
|
||
let wide = target(CaptureMode::Display, 3440.0, 1440.0);
|
||
assert_eq!(capture_size(&wide, 2560.0), (2560, 1072));
|
||
} else {
|
||
assert_eq!(capture_size(&display, 2560.0), (2560, 1654));
|
||
assert_eq!(capture_size(®ion, 1280.0), (960, 640));
|
||
}
|
||
|
||
// A 1x external display is recorded at its own pixels, never upscaled.
|
||
let external = |mode, width, height| TargetInfo {
|
||
frame: crate::screens::with_scale([-1920.0, 37.0, width, height], 1.0),
|
||
..target(mode, width, height)
|
||
};
|
||
let screen = external(CaptureMode::Display, 1920.0, 1080.0);
|
||
assert_eq!(capture_size(&screen, 2560.0), (1920, 1080));
|
||
assert_eq!(capture_size(&screen, CANDIDATE_EDGE), (480, 270));
|
||
let region = external(CaptureMode::Region, 480.0, 320.0);
|
||
assert_eq!(capture_size(®ion, 1280.0), (480, 320));
|
||
// A 2x one above the laptop keeps the 2560 px long edge.
|
||
let above = TargetInfo {
|
||
frame: crate::screens::with_scale([0.0, -1440.0, 2560.0, 1440.0], 2.0),
|
||
..target(CaptureMode::Display, 2560.0, 1440.0)
|
||
};
|
||
assert_eq!(capture_size(&above, 2560.0), (2560, 1440));
|
||
}
|
||
|
||
#[test]
|
||
fn cached_window_hit_preserves_front_to_back_order_and_screen_coordinates() {
|
||
let window = |id, x, y| TargetInfo {
|
||
mode: CaptureMode::Window,
|
||
app: Some(format!("App {id}")),
|
||
bundle_id: None,
|
||
window_title: Some(format!("Window {id}")),
|
||
window_id: Some(id),
|
||
display_id: 1,
|
||
frame: TargetFrame {
|
||
x,
|
||
y,
|
||
width: 200.0,
|
||
height: 100.0,
|
||
scale: 2.0,
|
||
},
|
||
};
|
||
let windows = [window(1, -100.0, 20.0), window(2, 0.0, 0.0)];
|
||
assert_eq!(
|
||
selectable_window_at(&windows, 50.0, 50.0)
|
||
.unwrap()
|
||
.window_id,
|
||
Some(1)
|
||
);
|
||
assert_eq!(
|
||
selectable_window_at(&windows, 150.0, 50.0)
|
||
.unwrap()
|
||
.window_id,
|
||
Some(2)
|
||
);
|
||
assert_eq!(
|
||
selectable_window_at(&windows, -100.0, 20.0)
|
||
.unwrap()
|
||
.window_id,
|
||
Some(1)
|
||
);
|
||
assert!(selectable_window_at(&windows, -101.0, 50.0).is_none());
|
||
assert!(selectable_window_at(&[], 0.0, 0.0).is_none());
|
||
}
|
||
|
||
#[test]
|
||
fn shared_resources_default_to_none_and_yield_when_peer_runs() {
|
||
let engine = Engine::default_engine();
|
||
assert_eq!(engine.resources(), SharedResources::none());
|
||
assert_eq!(
|
||
SharedResources::when_peer_absent(true),
|
||
SharedResources::none()
|
||
);
|
||
assert_eq!(
|
||
SharedResources::when_peer_absent(false),
|
||
SharedResources::all()
|
||
);
|
||
let custom = SharedResources::none()
|
||
.with_ingest_server(true)
|
||
.with_delivery_retries(false)
|
||
.with_global_shortcuts(false);
|
||
let engine = engine.with_resources(custom);
|
||
assert_eq!(engine.resources(), custom);
|
||
// `install_shared_resources` on `SharedResources::none()` is a no-op.
|
||
Engine::default_engine().install_shared_resources();
|
||
}
|
||
|
||
#[test]
|
||
fn start_defaults_to_external_audio_and_never_opens_microphone() {
|
||
let target = TargetInfo {
|
||
mode: CaptureMode::Display,
|
||
app: None,
|
||
bundle_id: None,
|
||
window_title: None,
|
||
window_id: None,
|
||
display_id: 1,
|
||
frame: TargetFrame {
|
||
x: 0.0,
|
||
y: 0.0,
|
||
width: 100.0,
|
||
height: 100.0,
|
||
scale: 1.0,
|
||
},
|
||
};
|
||
assert_eq!(Start::default().audio, AudioIn::External);
|
||
assert_eq!(Start::screen(target.clone()).audio, AudioIn::External);
|
||
assert_eq!(Start::screen_with_mic(target).audio, AudioIn::Microphone);
|
||
|
||
let (tx, _rx) = flume::unbounded();
|
||
let host = ChannelHost::new(tx);
|
||
let mic_opened = AtomicBool::new(false);
|
||
assert!(
|
||
start_audio_for(AudioIn::External, &host, || -> Result<()> {
|
||
mic_opened.store(true, Ordering::Release);
|
||
Ok(())
|
||
})
|
||
.is_none()
|
||
);
|
||
assert!(
|
||
start_audio_for(AudioIn::None, &host, || -> Result<()> {
|
||
mic_opened.store(true, Ordering::Release);
|
||
Ok(())
|
||
})
|
||
.is_none()
|
||
);
|
||
assert!(!mic_opened.load(Ordering::Acquire));
|
||
assert!(
|
||
start_audio_for(AudioIn::Microphone, &host, || -> Result<u32> {
|
||
mic_opened.store(true, Ordering::Release);
|
||
Ok(42)
|
||
}) == Some(42)
|
||
);
|
||
assert!(mic_opened.load(Ordering::Acquire));
|
||
}
|
||
}
|