From 96808aec6f4271c337ed130d7c94a878c3e6a0d7 Mon Sep 17 00:00:00 2001 From: Seungmin Kim <8457324+ehfd@users.noreply.github.com> Date: Fri, 17 Jul 2026 14:58:54 +0900 Subject: [PATCH 01/21] fix(core): Improve recording pathway --- pixelflux/src/encoders/nvenc.rs | 8 +- pixelflux/src/encoders/oh264.rs | 22 ++-- pixelflux/src/encoders/software.rs | 24 ++-- pixelflux/src/lib.rs | 36 ++--- pixelflux/src/pipeline.rs | 2 +- pixelflux/src/recording_sink.rs | 205 ++++++++++++++++++----------- 6 files changed, 176 insertions(+), 121 deletions(-) diff --git a/pixelflux/src/encoders/nvenc.rs b/pixelflux/src/encoders/nvenc.rs index bf6a32c..4ae22fa 100644 --- a/pixelflux/src/encoders/nvenc.rs +++ b/pixelflux/src/encoders/nvenc.rs @@ -1968,7 +1968,7 @@ mod gpu_tests { fn gpu_resolution_reconfigure_roundtrip() { let mut s = settings(1280, 720, 60.0); let t0 = std::time::Instant::now(); - let mut enc = NvencEncoder::new(&s, ptr::null(), None).expect("NVENC init"); + let mut enc = NvencEncoder::new(&s, ptr::null()).expect("NVENC init"); let init_ms = t0.elapsed().as_secs_f64() * 1000.0; let mut stream: Vec = Vec::new(); @@ -2049,7 +2049,7 @@ mod gpu_tests { let mut s = settings(1280, 720, 60.0); s.video_cbr_mode = true; s.video_bitrate_kbps = 4000; - let mut enc = NvencEncoder::new(&s, ptr::null(), None).expect("NVENC init"); + let mut enc = NvencEncoder::new(&s, ptr::null()).expect("NVENC init"); let f720 = frame(1280, 720, 10); for i in 0..3u64 { enc.encode_cpu_argb(&f720, 1280 * 4, i, 25, i == 0) @@ -2082,7 +2082,7 @@ mod gpu_tests { } let s = settings(1920, 1080, 60.0); let before = used_mb(); - let mut enc = NvencEncoder::new(&s, ptr::null(), None).expect("init"); + let mut enc = NvencEncoder::new(&s, ptr::null()).expect("init"); let f = frame(1920, 1080, 5); for i in 0..3u64 { enc.encode_cpu_argb(&f, 1920 * 4, i, 25, i == 0).expect("encode"); @@ -2097,7 +2097,7 @@ mod gpu_tests { #[ignore] fn gpu_init_above_default_headroom() { let s = settings(2160, 4096, 30.0); - let mut enc = NvencEncoder::new(&s, ptr::null(), None).expect("NVENC init portrait 4K"); + let mut enc = NvencEncoder::new(&s, ptr::null()).expect("NVENC init portrait 4K"); assert_eq!(enc.init_params.maxEncodeWidth, 4096); assert_eq!(enc.init_params.maxEncodeHeight, 4096); let f = frame(2160, 4096, 20); diff --git a/pixelflux/src/encoders/oh264.rs b/pixelflux/src/encoders/oh264.rs index 1d3cff6..a959275 100644 --- a/pixelflux/src/encoders/oh264.rs +++ b/pixelflux/src/encoders/oh264.rs @@ -459,7 +459,7 @@ mod tests { target_fps: 30.0, ..Default::default() }; - let mut enc = Openh264Encoder::new(&s, None).expect("openh264 init"); + let mut enc = Openh264Encoder::new(&s).expect("openh264 init"); let stride = 128 * 4; let idr = enc.encode_host_argb(&busy_frame(128, 96, 0), stride, 0, true, false).expect("encode idr"); assert!(idr.len() > 10, "IDR frame should produce output"); @@ -489,7 +489,7 @@ mod tests { omit_stripe_headers: true, ..Default::default() }; - let mut enc = Openh264Encoder::new(&s, None).expect("openh264 init"); + let mut enc = Openh264Encoder::new(&s).expect("openh264 init"); let out = enc.encode_host_argb(&busy_frame(128, 96, 0), 128 * 4, 0, true, false).expect("encode"); assert!( out.starts_with(&[0, 0, 0, 1]) || out.starts_with(&[0, 0, 1]), @@ -511,7 +511,7 @@ mod tests { target_fps: 30.0, ..Default::default() }; - let mut e = Openh264Encoder::new(&s, None).unwrap(); + let mut e = Openh264Encoder::new(&s).unwrap(); let _ = e.encode_host_argb(&busy_frame(w, h, 0), stride, 0, true, false).unwrap(); (1..24).map(|t| e.encode_host_argb(&busy_frame(w, h, t), stride, t as u64, false, false).unwrap().len()).sum() }; @@ -533,7 +533,7 @@ mod tests { target_fps: 30.0, ..Default::default() }; - let mut e = Openh264Encoder::new(&s, None).unwrap(); + let mut e = Openh264Encoder::new(&s).unwrap(); let _ = e.encode_host_argb(&busy_frame(w, h, 0), stride, 0, true, false).unwrap(); let high: usize = (1..24).map(|t| e.encode_host_argb(&busy_frame(w, h, t), stride, t as u64, false, false).unwrap().len()).sum(); @@ -557,7 +557,7 @@ mod tests { target_fps: 30.0, ..Default::default() }; - let mut e = Openh264Encoder::new(&s, None).unwrap(); + let mut e = Openh264Encoder::new(&s).unwrap(); let _ = e.encode_host_argb(&busy_frame(w, h, 0), stride, 0, true, false).unwrap(); let low: usize = (1..24).map(|t| e.encode_host_argb(&busy_frame(w, h, t), stride, t as u64, false, false).unwrap().len()).sum(); @@ -584,7 +584,7 @@ mod tests { target_fps: 30.0, ..Default::default() }; - let mut e = Openh264Encoder::new(&s, None).unwrap(); + let mut e = Openh264Encoder::new(&s).unwrap(); let mut total = e.encode_host_argb(&gradient_frame(w, h, 0), stride, 0, true, false).unwrap().len(); total += (1..24) .map(|t| e.encode_host_argb(&gradient_frame(w, h, t), stride, t as u64, false, false).unwrap().len()) @@ -614,7 +614,7 @@ mod tests { }; let frame = busy_frame(w, h, 0); for rgba in [false, true] { - let mut e = Openh264Encoder::new(&s, None).unwrap(); + let mut e = Openh264Encoder::new(&s).unwrap(); let out = e.encode_host_argb(&frame, stride, 0, true, rgba).expect("encode"); assert!(out.len() > 10, "rgba_input={rgba} must produce output"); assert_eq!(out[0], 0x04, "rgba_input={rgba} output must carry the wire header"); @@ -642,7 +642,7 @@ mod slice_tests { video_cbr_mode: true, ..Default::default() }; - let mut enc = Openh264Encoder::new(&s, None).expect("encoder"); + let mut enc = Openh264Encoder::new(&s).expect("encoder"); let frame = vec![0x80u8; 640 * 480 * 4]; let out = enc.encode_host_argb(&frame, 640 * 4, 0, true, false).expect("encode"); let mut nals = 0; @@ -705,7 +705,7 @@ mod slice_tests { target_fps: 60.0, ..Default::default() }; - let mut e = Openh264Encoder::new(&s, None).unwrap(); + let mut e = Openh264Encoder::new(&s).unwrap(); let mut total = 0usize; let mut idrs = 0usize; for t in 0..60u64 { @@ -752,7 +752,7 @@ mod slice_tests { target_fps: 60.0, ..Default::default() }; - let mut e = Openh264Encoder::new(&s, None).unwrap(); + let mut e = Openh264Encoder::new(&s).unwrap(); unsafe { let mut p: openh264_sys2::SEncParamExt = std::mem::zeroed(); let ret = e.encoder.raw_api().get_option( @@ -798,7 +798,7 @@ mod rebuild_cost { ..Default::default() }; let t = std::time::Instant::now(); - let mut e = Openh264Encoder::new(&s, None).expect("init"); + let mut e = Openh264Encoder::new(&s).expect("init"); let init_ms = t.elapsed().as_secs_f64() * 1000.0; let frame = vec![128u8; 1920 * 1080 * 4]; let t = std::time::Instant::now(); diff --git a/pixelflux/src/encoders/software.rs b/pixelflux/src/encoders/software.rs index 2bf9ed3..362b50c 100644 --- a/pixelflux/src/encoders/software.rs +++ b/pixelflux/src/encoders/software.rs @@ -17,6 +17,7 @@ use rayon::prelude::*; use smithay::utils::{Physical, Rectangle}; use std::ffi::CString; use std::ptr; +use std::sync::Arc; use yuv::{BufferStoreMut, YuvConversionMode, YuvPlanarImageMut, YuvRange, YuvStandardMatrix}; /// Upper bound on the horizontal stripes the CPU encoder splits a frame into, so the @@ -580,13 +581,14 @@ impl StripeState { /// /// # Fields /// -/// * `data` - Compressed payload (JPEG or H.264 NAL units). +/// * `data` - Compressed payload (JPEG or H.264 NAL units). `Arc`-shared so the recording +/// tap can retain the frame for its client queues without copying the bytes. /// * `data_type` - Codec tag: **1 = JPEG**, **2 = H.264**. /// * `stripe_y_start` - Y pixel coordinate of the stripe's top edge within the frame. /// * `stripe_height` - Height of the stripe in pixels. /// * `frame_id` - Frame sequence number this stripe belongs to. pub struct EncodedStripe { - pub data: Vec, + pub data: Arc>, pub data_type: i32, pub stripe_y_start: i32, pub stripe_height: i32, @@ -932,7 +934,7 @@ pub fn encode_cpu( std::mem::take(&mut stripe_state.packet_buf) }; Some(EncodedStripe { - data, + data: Arc::new(data), data_type: 1, stripe_y_start: y_start as i32, stripe_height: actual_height as i32, @@ -1025,7 +1027,7 @@ pub fn encode_cpu( &mut stripe_state.packet_buf, ) { Some(EncodedStripe { - data: std::mem::take(&mut stripe_state.packet_buf), + data: Arc::new(std::mem::take(&mut stripe_state.packet_buf)), data_type: 2, stripe_y_start: y_start as i32, stripe_height: actual_height as i32, @@ -1132,14 +1134,14 @@ mod tests { (w, h).into(), )]; let dirty = super::encode_cpu( - &mut stripes, &pixels, w, h, &full, &settings, 0, false, false, None, false, + &mut stripes, &pixels, w, h, &full, &settings, 0, false, false, false, ); assert!(!dirty.is_empty(), "damaged frame must encode"); let mut fired_at = None; for frame in 1..=20u16 { let out = super::encode_cpu( - &mut stripes, &pixels, w, h, &[], &settings, frame, false, false, None, false, + &mut stripes, &pixels, w, h, &[], &settings, frame, false, false, false, ); if !out.is_empty() { assert!(fired_at.is_none(), "paint-over must fire exactly once"); @@ -1177,14 +1179,14 @@ mod tests { }; let mut stripes = Vec::new(); let first = super::encode_cpu( - &mut stripes, &static_px, w, h, &[], &settings, 0, false, true, None, false, + &mut stripes, &static_px, w, h, &[], &settings, 0, false, true, false, ); assert!(!first.is_empty(), "first frame hashes as changed and encodes"); let mut fired_at = None; for frame in 1..=20u16 { let out = super::encode_cpu( - &mut stripes, &static_px, w, h, &[], &settings, frame, false, true, None, false, + &mut stripes, &static_px, w, h, &[], &settings, frame, false, true, false, ); if !out.is_empty() { assert!(fired_at.is_none(), "paint-over must fire exactly once while static"); @@ -1194,7 +1196,7 @@ mod tests { assert_eq!(fired_at, Some(settings.paint_over_trigger_frames as u16)); let woke = super::encode_cpu( - &mut stripes, &changed_px, w, h, &[], &settings, 21, false, true, None, false, + &mut stripes, &changed_px, w, h, &[], &settings, 21, false, true, false, ); assert!(!woke.is_empty(), "content change after idle must encode"); } @@ -1313,7 +1315,7 @@ mod qp_bound_sweep { let mut out = Vec::new(); enc.encode_with_headers( &y, &u, &v, W as i32, (W / 2) as i32, (W / 2) as i32, - i as i64, i == 0, &[], true, &mut out, None, + i as i64, i == 0, &[], true, &mut out, ); out }) @@ -1336,7 +1338,7 @@ mod qp_bound_sweep { video_max_qp: max_qp, ..Default::default() }; - let mut enc = Openh264Encoder::new(&s, None).expect("oh264 init"); + let mut enc = Openh264Encoder::new(&s).expect("oh264 init"); (0..FRAMES) .map(|i| { let y = text_luma(i); diff --git a/pixelflux/src/lib.rs b/pixelflux/src/lib.rs index 7ca8e2b..30c0b09 100644 --- a/pixelflux/src/lib.rs +++ b/pixelflux/src/lib.rs @@ -314,7 +314,7 @@ pub struct RustCaptureSettings { /// infinite GOP, 3 when scheduled keyframes are enabled. pub video_vbv_multiplier: f64, /// Seconds between scheduled recovery keyframes; `<= 0` keeps the GOP infinite - /// (IDRs only on demand: client join / reset, recording cadence). + /// (IDRs only on demand: client join / reset, recorder connect). pub keyframe_interval_s: f64, /// Rate-controlled (CBR) QP clamp: `video_max_qp` bounds the quality FLOOR (screen text stays /// legible under motion at the cost of overshooting impossible targets), `video_min_qp` bounds @@ -496,9 +496,6 @@ pub(crate) fn extract_settings(settings: &Bound<'_, PyAny>) -> PyResult().ok()) - .filter(|s| !s.is_empty()) - .or_else(|| std::env::var("PIXELFLUX_RECORDING_SOCKET").ok().filter(|s| !s.is_empty())) - .or_else(|| std::env::var("SELKIES_RECORDING_SOCKET").ok().filter(|s| !s.is_empty())) .unwrap_or_default(), omit_stripe_headers: settings .getattr("omit_stripe_headers") @@ -1183,7 +1180,7 @@ fn wayland_encode_loop(pool: &WlFramePool, cfg: WlEncodeConfig) -> Option out.push(EncodedStripe { - data, + data: Arc::new(data), data_type: 2, stripe_y_start: 0, stripe_height: height, @@ -1207,7 +1204,7 @@ fn wayland_encode_loop(pool: &WlFramePool, cfg: WlEncodeConfig) -> Option out.push(EncodedStripe { - data, + data: Arc::new(data), data_type: 2, stripe_y_start: 0, stripe_height: height, @@ -1708,12 +1705,17 @@ fn run_wayland_thread( crate::recording_sink::RecordingSink::try_bind(&settings.recording_socket); if let Some(output) = state.outputs.first() { - let current_mode = output.current_mode().unwrap(); - let current_w = current_mode.size.w; - let current_h = current_mode.size.h; - let current_scale = output.current_scale().fractional_scale(); - let current_refresh = current_mode.refresh; + // Never panic the compositor thread: an output momentarily + // without a current mode falls back to the requested geometry so + // the reconfigure below is a no-op for size/refresh instead of + // unwrap-panicking, which would drop every Wayland client with no + // recovery short of a process restart. let target_refresh = (settings.target_fps * 1000.0).round() as i32; + let (current_w, current_h, current_refresh) = match output.current_mode() { + Some(m) => (m.size.w, m.size.h, m.refresh), + None => (settings.width, settings.height, target_refresh), + }; + let current_scale = output.current_scale().fractional_scale(); let scale = settings.scale.max(0.1); let logical_width = (settings.width as f64 / scale).round() as i32; @@ -2858,7 +2860,7 @@ fn run_wayland_thread( state.encode_stats.stripes.fetch_add(1, Ordering::Relaxed); if let Some(ref tx) = state.deliver_tx { let stripes = vec![EncodedStripe { - data, data_type: 2, stripe_y_start: 0, + data: Arc::new(data), data_type: 2, stripe_y_start: 0, stripe_height: height, frame_id: state.frame_counter as i32, }]; if let Some(ref socket) = state.recording_sink { @@ -2954,7 +2956,7 @@ fn run_wayland_thread( /// four stripe-metadata ints as Python attributes. #[pyclass] struct StripeFrame { - data: Vec, + data: Arc>, #[pyo3(get, set)] data_type: i32, #[pyo3(get, set)] @@ -2966,10 +2968,10 @@ struct StripeFrame { } impl StripeFrame { - /// Hot-path constructor: MOVES the encoded buffer in (no copy) and carries stripe + /// Hot-path constructor: shares the encoder's buffer by `Arc` (no copy) and carries stripe /// metadata as attributes, so the consumer can read it without parsing a header /// (required for omit_stripe_headers). - fn new_owned_meta(data: Vec, data_type: i32, stripe_y_start: i32, stripe_height: i32, frame_id: i32) -> Self { + fn new_owned_meta(data: Arc>, data_type: i32, stripe_y_start: i32, stripe_height: i32, frame_id: i32) -> Self { Self { data, data_type, stripe_y_start, stripe_height, frame_id } } } @@ -2981,7 +2983,7 @@ impl StripeFrame { #[new] #[pyo3(signature = (data, data_type = 0, stripe_y_start = 0, stripe_height = 0, frame_id = 0))] fn new(data: Vec, data_type: i32, stripe_y_start: i32, stripe_height: i32, frame_id: i32) -> Self { - Self { data, data_type, stripe_y_start, stripe_height, frame_id } + Self { data: Arc::new(data), data_type, stripe_y_start, stripe_height, frame_id } } fn __len__(&self) -> usize { @@ -3210,7 +3212,7 @@ fn stripe_frame_from_buffer( stripe_height: i32, frame_id: i32, ) -> StripeFrame { - StripeFrame::new_owned_meta(data, data_type, stripe_y_start, stripe_height, frame_id) + StripeFrame::new_owned_meta(Arc::new(data), data_type, stripe_y_start, stripe_height, frame_id) } /// Capture configuration read by `start_capture` (each field by attribute name via diff --git a/pixelflux/src/pipeline.rs b/pixelflux/src/pipeline.rs index 6c504c5..fdd7f35 100644 --- a/pixelflux/src/pipeline.rs +++ b/pixelflux/src/pipeline.rs @@ -377,7 +377,7 @@ impl X11Pipeline { match res { Ok(data) if !data.is_empty() => { vec![EncodedStripe { - data, + data: Arc::new(data), data_type: 2, stripe_y_start: 0, stripe_height: height, diff --git a/pixelflux/src/recording_sink.rs b/pixelflux/src/recording_sink.rs index b0fa05d..f4ddf65 100644 --- a/pixelflux/src/recording_sink.rs +++ b/pixelflux/src/recording_sink.rs @@ -1,48 +1,58 @@ //! Unix-socket H.264 fan-out for external recording. //! -//! A background listener thread binds a Unix domain socket and accepts an arbitrary number of -//! concurrent clients. Each encoded H.264 stripe written through [`RecordingSink`] is broadcast -//! to every connected client as a plain Annex-B elementary stream — the 10-byte wire header -//! (tag, keyframe flag, frame number, y-start, width, height) is stripped so the output is -//! directly muxable. +//! Frames are intercepted at the delivery layer (not inside each encoder), so the tap works +//! uniformly for every full-frame encoder. The 10-byte pixelflux wire header is skipped so +//! consumers receive a plain Annex-B elementary stream that is directly muxable. //! -//! The sink is created once per capture session and shared (via `Arc`) between the encode -//! threads and the delivery layer. When a new client connects, [`RecordingSink::should_force_idr`] -//! returns `true` on the next call, signalling the encoder to emit an IDR so the fresh consumer -//! can start decoding from a clean reference frame. +//! The tap must never perturb the live viewer transport, and it never copies frame bytes: +//! stripe payloads are `Arc`-shared, so the encode thread only clones a handle into a bounded +//! per-client channel drained by a dedicated writer thread. A slow or stalled recorder blocks +//! nothing but itself and is dropped once its queue overflows. A newly connected client arms +//! [`RecordingSink::should_force_idr`] so the next encode emits an IDR it can decode from. use std::fs; use std::io::{ErrorKind, Write}; -use std::os::unix::net::{UnixListener, UnixStream}; +use std::os::unix::net::UnixListener; use std::sync::atomic::{AtomicBool, Ordering}; use std::sync::{Arc, Mutex}; use std::thread; use std::time::Duration; -/// Write timeout applied to each accepted client stream. Keeps a slow consumer from blocking -/// the encode thread; a stalled write simply drops that client. +use crossbeam_channel::{bounded, Sender, TrySendError}; + +use crate::encoders::software::EncodedStripe; + +/// Per-write timeout on a client stream; a stalled write surfaces as a soft error that +/// [`write_all_frame`] retries, keeping the writer thread responsive to teardown. const WRITE_TIMEOUT: Duration = Duration::from_millis(100); /// How often the non-blocking accept loop retries when no client is waiting. const ACCEPT_POLL_INTERVAL: Duration = Duration::from_millis(50); -/// Environment variable consulted as a fallback when the Python `CaptureSettings.recording_socket` -/// field is empty. -pub const RECORDING_SOCKET_ENV: &str = "PIXELFLUX_RECORDING_SOCKET"; +/// Per-client backlog bound. A recorder that falls this far behind is dropped rather than +/// allowed to grow memory or push back on the shared encode thread. +const CLIENT_QUEUE_CAP: usize = 256; + +/// A queued frame: the `Arc`-shared payload plus the byte offset where the recordable +/// Annex-B stream starts (past the wire header, or `0` when the payload is bare). +type QueuedFrame = (Arc>, usize); + +/// The sink's handle to one connected recorder: the feed end of its bounded queue and a kill +/// switch for its writer thread. The socket itself is owned solely by that writer thread. +struct ClientHandle { + tx: Sender, + stop: Arc, +} /// Unix-socket fan-out that broadcasts every encoded H.264 frame to connected consumers. /// -/// Internally a listener thread accepts connections on a [`UnixListener`] and appends the -/// resulting streams to a shared `Vec`. The encode / delivery threads then call -/// [`write_encoded_frame`] to push data to every client; clients that fall behind or disconnect -/// are removed automatically. -/// -/// [`write_encoded_frame`]: RecordingSink::write_encoded_frame +/// A listener thread accepts connections and gives each its own bounded queue and writer thread +/// (see [`ClientHandle`]) so one slow reader cannot stall the others or the encode thread. pub struct RecordingSink { /// Filesystem path of the Unix socket; removed on drop. path: String, - /// Connected client streams, shared between the accept thread and the write path. - clients: Arc>>, + /// Feed handles for the connected clients, shared with the accept thread. + clients: Arc>>, /// Signals the accept thread to exit; set in [`Drop`]. shutdown: Arc, /// Flipped to `true` each time a new client connects; consumed by [`should_force_idr`]. @@ -52,22 +62,14 @@ pub struct RecordingSink { } impl RecordingSink { - /// Try to bind a Unix socket at `settings_path` (or the `PIXELFLUX_RECORDING_SOCKET` - /// environment variable when the path is empty). - /// - /// Returns `None` when no path is configured or when the bind fails. On success the - /// listener thread is spawned and the sink is ready to accept clients immediately. + /// Bind a Unix socket at `settings_path`, or return `None` when no path is configured or the + /// bind fails. Recording is an optional tap that must never take the pipeline down, so a bind + /// error is logged and swallowed. pub fn try_bind(settings_path: &str) -> Option> { - let path = if !settings_path.is_empty() { - settings_path.to_string() - } else { - match std::env::var(RECORDING_SOCKET_ENV) { - Ok(p) if !p.is_empty() => p, - _ => return None, - } - }; - - match Self::bind(path) { + if settings_path.is_empty() { + return None; + } + match Self::bind(settings_path.to_string()) { Ok(sink) => Some(Arc::new(sink)), Err(e) => { eprintln!("[recording_sink] bind failed: {:?}", e); @@ -76,13 +78,14 @@ impl RecordingSink { } } - /// Create the socket, spawn the accept thread, and return the sink. + /// Create the socket and spawn the accept thread. Each accepted connection gets a write + /// timeout, a bounded queue, and a writer thread; the sink keeps only the feed handle. fn bind(path: String) -> std::io::Result { let _ = fs::remove_file(&path); let listener = UnixListener::bind(&path)?; listener.set_nonblocking(true)?; - let clients = Arc::new(Mutex::new(Vec::new())); + let clients: Arc>> = Arc::new(Mutex::new(Vec::new())); let shutdown = Arc::new(AtomicBool::new(false)); let client_connected = Arc::new(AtomicBool::new(false)); @@ -100,13 +103,32 @@ impl RecordingSink { eprintln!("[recording_sink] set_write_timeout failed: {:?}", e); continue; } + + let (tx, rx) = bounded::(CLIENT_QUEUE_CAP); + let stop = Arc::new(AtomicBool::new(false)); + let stop_writer = stop.clone(); + thread::spawn(move || { + let mut stream = stream; + for (buf, offset) in rx.iter() { + if stop_writer.load(Ordering::Relaxed) { + break; + } + if let Err(e) = + write_all_frame(&mut stream, &buf[offset..], &stop_writer) + { + eprintln!( + "[recording_sink] writer thread exiting; write failed: {:?}", + e + ); + break; + } + } + }); + let mut guard = clients_acc.lock().unwrap(); - guard.push(stream); + guard.push(ClientHandle { tx, stop }); client_connected_acc.store(true, Ordering::Relaxed); - eprintln!( - "[recording_sink] client connected; total {}", - guard.len() - ); + eprintln!("[recording_sink] client connected; total {}", guard.len()); } Err(e) if e.kind() == ErrorKind::WouldBlock => { thread::sleep(ACCEPT_POLL_INTERVAL); @@ -128,62 +150,91 @@ impl RecordingSink { }) } - /// Returns `true` exactly once after a new client connects, signalling that the next - /// encode should produce an IDR so the consumer starts from a clean reference frame. + /// Returns `true` exactly once after a new client connects, signalling that the next encode + /// should produce an IDR so the consumer starts from a clean reference frame. pub fn should_force_idr(&self) -> bool { self.client_connected.swap(false, Ordering::Relaxed) } - /// Write `data` to every connected client, silently removing any that are disconnected - /// or too slow. - fn write_all_clients(&self, data: &[u8]) { - if data.is_empty() { + /// Delivery-layer tap for one encoded stripe. Only H.264 frames (`data_type == 2`) are + /// forwarded; the 10-byte wire header (`0x04` tag) is skipped via the queued offset so the + /// output is a plain Annex-B elementary stream. + /// + /// Never blocks and never copies: the `Arc` payload is cloned into each client's bounded + /// queue with `try_send`, and a client whose queue is full or whose writer died is dropped. + pub fn write_encoded_frame(&self, stripe: &EncodedStripe) { + if stripe.data.is_empty() || stripe.data_type != 2 { return; } + let offset = if stripe.data.len() > 10 && stripe.data[0] == 0x04 { + 10 + } else { + 0 + }; + let mut clients = self.clients.lock().unwrap(); if clients.is_empty() { return; } let mut to_remove: Vec = Vec::new(); - for (idx, client) in clients.iter_mut().enumerate() { - if let Err(e) = client.write_all(data) { - if matches!( - e.kind(), - ErrorKind::BrokenPipe | ErrorKind::ConnectionReset | ErrorKind::UnexpectedEof - ) { + for (idx, client) in clients.iter().enumerate() { + match client.tx.try_send((stripe.data.clone(), offset)) { + Ok(()) => {} + Err(TrySendError::Full(_)) => { + eprintln!("[recording_sink] dropping slow client (idx {})", idx); to_remove.push(idx); - } else { - eprintln!("[recording_sink] dropping client (idx {}): {:?}", idx, e); + } + Err(TrySendError::Disconnected(_)) => { to_remove.push(idx); } } } for idx in to_remove.into_iter().rev() { - clients.swap_remove(idx); - } - } - - /// Write a single encoded stripe to all connected clients. - /// - /// Only H.264 frames (`data_type == 2`) are forwarded. When the frame carries the - /// pixelflux wire header (`0x04` tag, 10 bytes) the header is stripped so the output - /// is a plain Annex-B elementary stream suitable for direct muxing. - pub fn write_encoded_frame(&self, stripe: &crate::encoders::software::EncodedStripe) { - if stripe.data.is_empty() || stripe.data_type != 2 { - return; - } - let data = &stripe.data; - if data.len() > 10 && data[0] == 0x04 { - self.write_all_clients(&data[10..]); - } else { - self.write_all_clients(data); + let removed = clients.swap_remove(idx); + removed.stop.store(true, Ordering::Relaxed); } } } impl Drop for RecordingSink { + /// Stop accepting, release every writer thread (set each `stop`, then drop its sender so an + /// idle writer parked on `rx.iter()` wakes), and remove the socket file. fn drop(&mut self) { self.shutdown.store(true, Ordering::Relaxed); + if let Ok(mut clients) = self.clients.lock() { + for client in clients.iter() { + client.stop.store(true, Ordering::Relaxed); + } + clients.clear(); + } let _ = fs::remove_file(&self.path); } } + +/// Write one whole frame to a recorder's socket, resuming across the soft timeouts a slow reader +/// induces so a partial Annex-B NAL is never left behind. Aborts if `stop` is set (the client was +/// dropped by [`RecordingSink::write_encoded_frame`]) or a hard error occurs. +fn write_all_frame(stream: &mut W, buf: &[u8], stop: &AtomicBool) -> std::io::Result<()> { + let mut written = 0usize; + while written < buf.len() { + if stop.load(Ordering::Relaxed) { + return Err(std::io::Error::other("writer stopped (client dropped)")); + } + match stream.write(&buf[written..]) { + Ok(0) => { + return Err(std::io::Error::new( + ErrorKind::WriteZero, + "failed to write whole frame", + )); + } + Ok(n) => written += n, + Err(ref e) if e.kind() == ErrorKind::TimedOut => {} + Err(ref e) if e.kind() == ErrorKind::WouldBlock => { + thread::sleep(Duration::from_millis(5)); + } + Err(ref e) if e.kind() == ErrorKind::Interrupted => {} + Err(e) => return Err(e), + } + } + Ok(()) +} From 97b9caa5be139247d1af551db89f33e8da98fa6a Mon Sep 17 00:00:00 2001 From: Seungmin Kim <8457324+ehfd@users.noreply.github.com> Date: Fri, 17 Jul 2026 15:34:04 +0900 Subject: [PATCH 02/21] fix(core): Additional fixes --- pixelflux/src/encoders/oh264.rs | 14 +++++----- pixelflux/src/lib.rs | 47 ++++++++++++++++++++++++--------- pixelflux/src/pipeline.rs | 17 +++++++++--- pixelflux/src/x11/mod.rs | 6 ++--- 4 files changed, 56 insertions(+), 28 deletions(-) diff --git a/pixelflux/src/encoders/oh264.rs b/pixelflux/src/encoders/oh264.rs index a959275..4a59df2 100644 --- a/pixelflux/src/encoders/oh264.rs +++ b/pixelflux/src/encoders/oh264.rs @@ -134,6 +134,10 @@ impl Openh264Encoder { .skip_frames(true) .adaptive_quantization(false) .background_detection(false) + // Scene-change detection would inject IDRs the pipeline never asked for, + // breaking the infinite-GOP / on-demand-keyframe contract (x264 parity: + // `i_scenecut_threshold = 0`). + .scene_change_detect(false) .debug(settings.debug_logging) .intra_frame_period(IntraFramePeriod::from_num_frames(INFINITE_INTRA_PERIOD)) .num_threads(threads); @@ -338,16 +342,12 @@ impl Openh264Encoder { /// byte-for-byte the layout the NVENC/VAAPI/x264 full-frame paths emit, which is what lets a /// single client demuxer serve every encoder. Its type byte is read from the *actually /// encoded* picture type (IDR = `0x01`, I = `0x02`, P = `0x00`) rather than from `force_idr`, - /// because OpenH264's scene-change detection can emit an IDR the caller never asked for, and - /// the header must label a real decode entry point, not the request. It also carries the - /// frame number, a zero y-start, and the width/height (all big-endian). With - /// `omit_stripe_headers` the output is bare Annex-B. + /// because the encoder may re-type a frame, and the header must label a real decode entry + /// point, not the request. It also carries the frame number, a zero y-start, and the + /// width/height (all big-endian). With `omit_stripe_headers` the output is bare Annex-B. /// 5. **Empty payload**: if the encoder produced no bitstream (e.g. a skipped frame), an empty /// vec is returned rather than a lone header — a header with no Annex-B behind it would be a /// malformed frame to the client, and the pipeline reads empty as "nothing to send". - /// 6. **Recording sink**: the raw Annex-B payload (past the header) is also written to the - /// recording sink when one is attached, since a recorder wants the bare elementary stream, - /// not the wire framing. pub fn encode_host_argb( &mut self, argb: &[u8], diff --git a/pixelflux/src/lib.rs b/pixelflux/src/lib.rs index 30c0b09..e7418f4 100644 --- a/pixelflux/src/lib.rs +++ b/pixelflux/src/lib.rs @@ -1115,7 +1115,14 @@ fn wayland_encode_loop(pool: &WlFramePool, cfg: WlEncodeConfig) -> Option = Vec::new(); if let Some(ref mut encoder) = video_encoder { @@ -1238,15 +1245,17 @@ fn wayland_encode_loop(pool: &WlFramePool, cfg: WlEncodeConfig) -> Option)> = None; @@ -2827,16 +2850,11 @@ fn run_wayland_thread( let force_idr = decision.force_idr; let target_qp = decision.target_qp; + let mut frame_out = false; if send_frame { if let Some(sync) = render_sync.take() { let _ = sync.wait(); } - let force_idr_for_recording = state - .recording_sink - .as_ref() - .map(|s| s.should_force_idr()) - .unwrap_or(false); - let force_idr = force_idr || force_idr_for_recording; let result = match encoder { GpuEncoder::Nvenc(enc) => { if let Some((_, ref dmabuf)) = state.offscreen_buffer { @@ -2856,6 +2874,7 @@ fn run_wayland_thread( if let Ok(data) = result { if !data.is_empty() { + frame_out = true; state.encode_stats.frames.fetch_add(1, Ordering::Relaxed); state.encode_stats.stripes.fetch_add(1, Ordering::Relaxed); if let Some(ref tx) = state.deliver_tx { @@ -2865,7 +2884,7 @@ fn run_wayland_thread( }]; if let Some(ref socket) = state.recording_sink { for stripe in &stripes { - let _ = socket.write_encoded_frame(stripe); + socket.write_encoded_frame(stripe); } } // Non-blocking: a full slot parks the frame @@ -2883,7 +2902,9 @@ fn run_wayland_thread( eprintln!("HW Encode Error: {}", e); } } - state.pending_force_idr = false; + // An unserved request stays armed: on an infinite GOP an IDR lost + // to an encode error would never self-heal. + state.pending_force_idr = requested_idr && !frame_out; state.frame_counter = state.frame_counter.wrapping_add(1); } } diff --git a/pixelflux/src/pipeline.rs b/pixelflux/src/pipeline.rs index fdd7f35..f62a47f 100644 --- a/pixelflux/src/pipeline.rs +++ b/pixelflux/src/pipeline.rs @@ -340,7 +340,14 @@ impl X11Pipeline { pub fn process(&mut self, argb: &[u8], stride: usize) -> Vec { let width = self.settings.width; let height = self.settings.height; - let requested = self.pending_force_idr; + // A recorder connecting is a first-class IDR request: folding it in here lets the + // decision layer send a decodable frame promptly even when the screen is static. + let requested = self.pending_force_idr + || self + .recording_sink + .as_ref() + .map(|s| s.should_force_idr()) + .unwrap_or(false); let threshold = self.settings.damage_block_threshold; let duration = self.settings.damage_block_duration as i32; @@ -360,8 +367,7 @@ impl X11Pipeline { ); if d.send { let fc = self.frame_counter as u64; - let force_idr = d.force_idr - || self.recording_sink.as_ref().map(|s| s.should_force_idr()).unwrap_or(false); + let force_idr = d.force_idr; let res = match &mut self.hw { X11Encoder::Nvenc(enc) => { enc.encode_cpu_argb(argb, stride, fc, d.target_qp, force_idr) @@ -416,7 +422,10 @@ impl X11Pipeline { ) }; - self.pending_force_idr = false; + // An unserved request stays armed: on an infinite GOP an IDR lost to an encode + // error or skip would never self-heal, leaving a joining consumer with an + // undecodable stream. + self.pending_force_idr = requested && out.is_empty(); self.frame_counter = self.frame_counter.wrapping_add(1); out } diff --git a/pixelflux/src/x11/mod.rs b/pixelflux/src/x11/mod.rs index a664bbe..2a73268 100644 --- a/pixelflux/src/x11/mod.rs +++ b/pixelflux/src/x11/mod.rs @@ -771,12 +771,10 @@ where if !stripes.is_empty() { frame_count += 1; stripe_count += stripes.len() as u64; + // IDR arming is consumed inside X11Pipeline::process; this tap only fans out. if let Some(ref socket) = recording_sink { for stripe in &stripes { - let _ = socket.write_encoded_frame(stripe); - } - if socket.should_force_idr() { - controls.force_idr.store(true, Ordering::Relaxed); + socket.write_encoded_frame(stripe); } } on_frame(stripes); From 78e4765330e0f66a3d20f17bd767adf77a4a69d0 Mon Sep 17 00:00:00 2001 From: Seungmin Kim <8457324+ehfd@users.noreply.github.com> Date: Sat, 18 Jul 2026 17:10:39 +0900 Subject: [PATCH 03/21] fix(core): Multi-display, Wayland parity, Computer Use, and recording --- pixelflux/Cargo.toml | 2 +- pixelflux/src/computer_use.rs | 718 +++-- pixelflux/src/encoders/nvenc.rs | 3 +- pixelflux/src/encoders/overlay.rs | 9 +- pixelflux/src/encoders/software.rs | 30 +- pixelflux/src/encoders/vaapi.rs | 7 +- pixelflux/src/lib.rs | 3945 ++++++++++++++++++---------- pixelflux/src/pipeline.rs | 23 +- pixelflux/src/recorder/mod.rs | 744 ++++++ pixelflux/src/recorder/mp4.rs | 806 ++++++ pixelflux/src/recording_sink.rs | 81 + pixelflux/src/wayland/cursor.rs | 210 ++ pixelflux/src/wayland/frontend.rs | 1143 +++++--- pixelflux/src/wayland/keymap.rs | 384 +++ pixelflux/src/wayland/mod.rs | 4 +- pixelflux/src/x11/computer_use.rs | 503 ++++ pixelflux/src/x11/mod.rs | 15 +- 17 files changed, 6660 insertions(+), 1967 deletions(-) create mode 100644 pixelflux/src/recorder/mod.rs create mode 100644 pixelflux/src/recorder/mp4.rs create mode 100644 pixelflux/src/wayland/keymap.rs create mode 100644 pixelflux/src/x11/computer_use.rs diff --git a/pixelflux/Cargo.toml b/pixelflux/Cargo.toml index e52d899..4baaf5e 100644 --- a/pixelflux/Cargo.toml +++ b/pixelflux/Cargo.toml @@ -31,7 +31,7 @@ libloading = "0.9" xcursor = "0.3.1" # X11 host capture (replaces the C++ XShm/XFixes path): shm = XShm zero-copy grab, # xfixes = hardware cursor image. Pure-Rust XCB; links the system libxcb. -x11rb = { version = "0.13", features = ["shm", "xfixes"] } +x11rb = { version = "0.13", features = ["shm", "xfixes", "xtest", "randr"] } image = "=0.25.9" # The crate's build.rs probes the FFmpeg found via pkg-config and emits # per-version cfgs; one crate version supports FFmpeg 3.0 UP TO the release it diff --git a/pixelflux/src/computer_use.rs b/pixelflux/src/computer_use.rs index 98439f2..18b8798 100644 --- a/pixelflux/src/computer_use.rs +++ b/pixelflux/src/computer_use.rs @@ -8,13 +8,22 @@ //! //! Enabled by setting the `PIXELFLUX_CU` environment variable to the listen port. The server //! handles `POST /computer-use` requests for screenshots, mouse/keyboard injection, scrolling, -//! and cursor position queries — all targeting the Wayland compositor owned by the same process. +//! and cursor position queries. Actions run against a [`CuBackend`], resolved per request: the +//! Wayland compositor owned by this process when one is registered, otherwise the X server named +//! by `DISPLAY` (XTEST injection on a private connection, no active capture required). +//! +//! The same server also exposes the built-in MP4 recorder at `/record_start`, `/record_stop` +//! and `/record_status`, so a headless script can drive a session and record it over plain HTTP. +use std::collections::HashMap; use std::sync::mpsc; +use std::sync::{Mutex, OnceLock}; use std::thread; use std::time::Duration; use std::io::Cursor; +use smithay::input::keyboard::xkb; + use base64::Engine; use base64::engine::general_purpose::STANDARD as BASE64; use image::{ImageBuffer, Rgba, ImageFormat}; @@ -41,58 +50,12 @@ pub fn encode_png_rgba(data: &[u8], width: u32, height: u32) -> Result, Ok(png) } -fn scancode_for_char(c: char) -> Option<(u32, bool)> { - Some(match c { - 'a' | 'A' => (38, c.is_uppercase()), - 'b' | 'B' => (56, c.is_uppercase()), - 'c' | 'C' => (54, c.is_uppercase()), - 'd' | 'D' => (40, c.is_uppercase()), - 'e' | 'E' => (26, c.is_uppercase()), - 'f' | 'F' => (41, c.is_uppercase()), - 'g' | 'G' => (42, c.is_uppercase()), - 'h' | 'H' => (43, c.is_uppercase()), - 'i' | 'I' => (31, c.is_uppercase()), - 'j' | 'J' => (44, c.is_uppercase()), - 'k' | 'K' => (45, c.is_uppercase()), - 'l' | 'L' => (46, c.is_uppercase()), - 'm' | 'M' => (58, c.is_uppercase()), - 'n' | 'N' => (57, c.is_uppercase()), - 'o' | 'O' => (32, c.is_uppercase()), - 'p' | 'P' => (33, c.is_uppercase()), - 'q' | 'Q' => (24, c.is_uppercase()), - 'r' | 'R' => (27, c.is_uppercase()), - 's' | 'S' => (39, c.is_uppercase()), - 't' | 'T' => (28, c.is_uppercase()), - 'u' | 'U' => (30, c.is_uppercase()), - 'v' | 'V' => (55, c.is_uppercase()), - 'w' | 'W' => (25, c.is_uppercase()), - 'x' | 'X' => (53, c.is_uppercase()), - 'y' | 'Y' => (29, c.is_uppercase()), - 'z' | 'Z' => (52, c.is_uppercase()), - '0' => (19, false), '1' => (10, false), '2' => (11, false), - '3' => (12, false), '4' => (13, false), '5' => (14, false), - '6' => (15, false), '7' => (16, false), '8' => (17, false), - '9' => (18, false), - '!' => (10, true), '@' => (11, true), '#' => (12, true), - '$' => (13, true), '%' => (14, true), '^' => (15, true), - '&' => (16, true), '*' => (17, true), '(' => (18, true), - ')' => (19, true), - '-' => (20, false), '_' => (20, true), - '=' => (21, false), '+' => (21, true), - '[' => (34, false), '{' => (34, true), - ']' => (35, false), '}' => (35, true), - ';' => (47, false), ':' => (47, true), - '\'' => (48, false), '"' => (48, true), - '`' => (49, false), '~' => (49, true), - '\\' => (51, false), '|' => (51, true), - ',' => (59, false), '<' => (59, true), - '.' => (60, false), '>' => (60, true), - '/' => (61, false), '?' => (61, true), - ' ' => (65, false), - '\t' => (23, false), - '\n' => (36, false), - _ => return None, - }) +/// Keysym for one literal character: Latin-1 printables map 1:1, control characters map +/// onto their `0xffXX` function keysyms, everything else onto the `0x01000000 | codepoint` +/// Unicode form (0 = unmappable). The keysym then resolves against the backend's ACTIVE +/// keymap, never a hardcoded layout table. +fn keysym_for_char(c: char) -> u32 { + xkb::utf32_to_keysym(c as u32).raw() } fn scancode_for_keyname(name: &str) -> Option { @@ -147,62 +110,239 @@ fn scancode_for_keyname(name: &str) -> Option { }) } +/// Keysym for one component of an xdotool-style key spec that is not a named key: a single +/// literal character (`a`, `7`, `-`, `#`) via the Unicode mapping, or a keysym NAME +/// (`minus`, `bracketleft`, `udiaeresis`, …) via xkb's name lookup (case-tolerant fallback +/// so `Minus` still resolves). Returns 0 when the spec names no keysym. +fn keysym_for_key_spec(name: &str) -> u32 { + let mut chars = name.chars(); + if let (Some(c), None) = (chars.next(), chars.next()) { + return keysym_for_char(c); + } + let sym = xkb::keysym_from_name(name, xkb::KEYSYM_NO_FLAGS).raw(); + if sym != 0 { + return sym; + } + xkb::keysym_from_name(name, xkb::KEYSYM_CASE_INSENSITIVE).raw() +} + fn is_modifier(scancode: u32) -> bool { matches!(scancode, 37 | 105 | 50 | 62 | 64 | 108 | 133 | 134) } -fn send_key(tx: &smithay::reexports::calloop::channel::Sender, scancode: u32, pressed: bool) { - let _ = tx.send(ThreadCommand::KeyboardKey { - scancode, - state: if pressed { 1 } else { 0 }, - }); +/// Semantic mouse-button vocabulary of the action layer; each backend maps it to its native +/// codes (evdev `BTN_` on Wayland, X core buttons 1/2/3 on X11). +#[derive(Clone, Copy)] +pub enum CuButton { + Left, + Right, + Middle, } -fn send_mouse_move(tx: &smithay::reexports::calloop::channel::Sender, x: f64, y: f64) { - let _ = tx.send(ThreadCommand::PointerMotion { x, y }); +/// The desktop-side primitives one Computer Use action decomposes into. Keycodes are X/xkb +/// keycodes (evdev + 8) — the numbering both the smithay seat and XTEST consume — and +/// coordinates are framebuffer/root pixels, already clamped by the action layer. +pub trait CuBackend { + fn name(&self) -> &'static str; + fn fb_size(&self) -> Result<(i32, i32), String>; + /// One display's framebuffer size; 0 = the primary. Unknown ids are an error. The + /// X11 backend serves only the root (display 0); Wayland resolves any live output id. + fn display_fb_size(&self, display: u32) -> Result<(i32, i32), String> { + if display == 0 { + self.fb_size() + } else { + Err(format!("Unknown display: {display}")) + } + } + fn key(&self, scancode: u32, pressed: bool); + fn mouse_move(&self, x: f64, y: f64); + fn button(&self, btn: CuButton, pressed: bool); + fn scroll(&self, dx: f64, dy: f64); + /// PNG of one display's framebuffer; 0 = the primary. Unknown ids are an error. + fn screenshot_png(&self, display: u32) -> Result, String>; + fn cursor_pos(&self) -> Result<(f64, f64), String>; + /// Run `seq` with every keysym in `keysyms` (which `resolve_keysyms` could not place) + /// made temporarily typeable, when the backend can arrange that; `seq` receives + /// keysym -> keycode for the transient bindings (empty when none could be arranged, + /// in which case those keysyms simply stay untypeable). Wayland's `resolve_keysyms` + /// already overlay-binds anything the base keymap lacks, so the default runs `seq` + /// with no bindings; the X11 backend overrides this with a grab-atomic spare-keycode + /// remap. + fn with_transient_keysyms(&self, keysyms: &[u32], seq: &mut dyn FnMut(&HashMap)) { + let _ = keysyms; + seq(&HashMap::new()); + } + /// Resolve keysyms against the backend's ACTIVE keymap: one `(keycode, shift level)` + /// per input keysym, `(0, 0)` where the keymap cannot produce it. Level bit 0 = Shift, + /// bit 1 = AltGr — the order xkb two/four-level key types use. Wayland resolves through + /// the compositor's keymap policy (overlay-binding what the base lacks); X11 through + /// the server's own `GetKeyboardMapping`. + fn resolve_keysyms(&self, keysyms: &[u32]) -> Vec<(u32, u32)>; + /// Keycode synthesized around AltGr-level hits (level bit 1). The default is the + /// pc105 right-Alt position the compositor's seat keymap binds to `ISO_Level3_Shift`; + /// the X11 backend overrides it with the level-3 modifier key found in the server's + /// keymap (0 = none, in which case `resolve_keysyms` reports no AltGr levels). + fn altgr_keycode(&self) -> u32 { + KC_ALTGR + } } const BTN_LEFT: u32 = 0x110; const BTN_RIGHT: u32 = 0x111; const BTN_MIDDLE: u32 = 0x112; -fn send_mouse_button(tx: &smithay::reexports::calloop::channel::Sender, btn: u32, pressed: bool) { - let _ = tx.send(ThreadCommand::PointerButton { - btn, - state: if pressed { 1 } else { 0 }, - }); +/// Reply deadline for compositor round-trips (screenshot readback, cursor position, fb size). +/// Bounded so a wedged render path turns into a JSON error instead of hanging the sequential +/// HTTP loop — and every request behind it — forever. +const REPLY_TIMEOUT: Duration = Duration::from_secs(5); + +/// Wayland implementation: every primitive is a `ThreadCommand` on the compositor's calloop +/// channel, so injection and readback serialize naturally with rendering and encoding. +pub struct CuWaylandBackend { + tx: smithay::reexports::calloop::channel::Sender, } -fn send_scroll(tx: &smithay::reexports::calloop::channel::Sender, x: f64, y: f64) { - let _ = tx.send(ThreadCommand::PointerAxis { x, y }); +impl CuBackend for CuWaylandBackend { + fn name(&self) -> &'static str { + "wayland" + } + + fn fb_size(&self) -> Result<(i32, i32), String> { + self.display_fb_size(0) + } + + fn display_fb_size(&self, display: u32) -> Result<(i32, i32), String> { + let (resp_tx, resp_rx) = mpsc::channel(); + self.tx.send(ThreadCommand::CuGetInfo { display_id: display, resp: resp_tx }) + .map_err(|_| "Failed to request compositor info".to_string())?; + let (w, h, _) = resp_rx.recv_timeout(REPLY_TIMEOUT) + .map_err(|_| "Compositor info request failed".to_string())?; + // CuGetInfo reports zeros for output ids that don't exist (and for an output + // with no mode, which live outputs always have). + if w <= 0 || h <= 0 { + return Err(format!("Unknown display: {display}")); + } + Ok((w, h)) + } + + fn key(&self, scancode: u32, pressed: bool) { + let _ = self.tx.send(ThreadCommand::KeyboardKey { + scancode, + state: if pressed { 1 } else { 0 }, + }); + } + + fn mouse_move(&self, x: f64, y: f64) { + let _ = self.tx.send(ThreadCommand::PointerMotion { x, y }); + } + + fn button(&self, btn: CuButton, pressed: bool) { + let btn = match btn { + CuButton::Left => BTN_LEFT, + CuButton::Right => BTN_RIGHT, + CuButton::Middle => BTN_MIDDLE, + }; + let _ = self.tx.send(ThreadCommand::PointerButton { + btn, + state: if pressed { 1 } else { 0 }, + }); + } + + fn scroll(&self, dx: f64, dy: f64) { + let _ = self.tx.send(ThreadCommand::PointerAxis { x: dx, y: dy }); + } + + fn screenshot_png(&self, display: u32) -> Result, String> { + let (resp_tx, resp_rx) = mpsc::channel(); + self.tx.send(ThreadCommand::CuScreenshot { display_id: display, resp: resp_tx }) + .map_err(|_| "Failed to request screenshot".to_string())?; + resp_rx.recv_timeout(REPLY_TIMEOUT).map_err(|_| "Screenshot failed".to_string())? + } + + fn cursor_pos(&self) -> Result<(f64, f64), String> { + let (resp_tx, resp_rx) = mpsc::channel(); + self.tx.send(ThreadCommand::CuCursorPosition { resp: resp_tx }) + .map_err(|_| "Failed to request cursor position".to_string())?; + resp_rx.recv_timeout(REPLY_TIMEOUT).map_err(|_| "Cursor position failed".to_string()) + } + + fn resolve_keysyms(&self, keysyms: &[u32]) -> Vec<(u32, u32)> { + let (resp_tx, resp_rx) = mpsc::channel(); + if self + .tx + .send(ThreadCommand::BindKeysyms { keysyms: keysyms.to_vec(), reply: resp_tx }) + .is_err() + { + return vec![(0, 0); keysyms.len()]; + } + resp_rx + .recv_timeout(REPLY_TIMEOUT) + .unwrap_or_else(|_| vec![(0, 0); keysyms.len()]) + } } -fn screenshot( - tx: &smithay::reexports::calloop::channel::Sender, -) -> Result, String> { - let (resp_tx, resp_rx) = mpsc::channel(); - tx.send(ThreadCommand::CuScreenshot { resp: resp_tx }) - .map_err(|_| "Failed to request screenshot".to_string())?; - resp_rx.recv().map_err(|_| "Screenshot failed".to_string()) +/// Per-request literal-key resolver: batches keysym lookups against the backend's active +/// keymap and caches them for the request's burst of key events (a fresh backend — and thus +/// a fresh cache — is resolved per HTTP request, so a runtime layout switch is picked up by +/// the next request). +struct KeyResolver<'a> { + backend: &'a dyn CuBackend, + cache: HashMap, } -fn cursor_position( - tx: &smithay::reexports::calloop::channel::Sender, -) -> Result<(f64, f64), String> { - let (resp_tx, resp_rx) = mpsc::channel(); - tx.send(ThreadCommand::CuCursorPosition { resp: resp_tx }) - .map_err(|_| "Failed to request cursor position".to_string())?; - resp_rx.recv().map_err(|_| "Cursor position failed".to_string()) +impl<'a> KeyResolver<'a> { + fn new(backend: &'a dyn CuBackend) -> Self { + Self { backend, cache: HashMap::new() } + } + + /// Resolve a batch up front so a `type` action costs one backend round trip. + fn prefetch(&mut self, keysyms: &[u32]) { + let missing: Vec = { + let mut seen = std::collections::HashSet::new(); + keysyms + .iter() + .copied() + .filter(|&s| s != 0 && !self.cache.contains_key(&s) && seen.insert(s)) + .collect() + }; + if missing.is_empty() { + return; + } + let out = self.backend.resolve_keysyms(&missing); + for (sym, hit) in missing.into_iter().zip(out) { + self.cache.insert(sym, hit); + } + } + + /// `(keycode, shift level)` for `keysym`, or `None` when the active keymap cannot + /// produce it. + fn resolve(&mut self, keysym: u32) -> Option<(u32, u32)> { + if keysym == 0 { + return None; + } + self.prefetch(&[keysym]); + let hit = self.cache.get(&keysym).copied().unwrap_or((0, 0)); + (hit.0 != 0).then_some(hit) + } } -fn get_fb_size( - tx: &smithay::reexports::calloop::channel::Sender, -) -> Result<(i32, i32), String> { - let (resp_tx, resp_rx) = mpsc::channel(); - tx.send(ThreadCommand::CuGetInfo { resp: resp_tx }) - .map_err(|_| "Failed to request compositor info".to_string())?; - let (w, h, _) = resp_rx.recv().map_err(|_| "Compositor info request failed".to_string())?; - Ok((w, h)) +/// Shift and default AltGr (pc105 right Alt / ISO_Level3_Shift) keycodes, held to reach a +/// resolved key's shift level: bit 0 = Shift, bit 1 = AltGr. +const KC_SHIFT: u32 = 50; +const KC_ALTGR: u32 = 108; + +/// The level modifiers `level` requires beyond whatever is already held (`held_mask` uses +/// the same bit layout), in press order. `altgr` is the backend's AltGr keycode +/// ([`CuBackend::altgr_keycode`]). +fn level_modifiers(level: u32, held_mask: u32, altgr: u32) -> Vec { + let mut out = Vec::new(); + if level & 1 != 0 && held_mask & 1 == 0 { + out.push(KC_SHIFT); + } + if level & 2 != 0 && held_mask & 2 == 0 && altgr != 0 { + out.push(altgr); + } + out } #[derive(Deserialize)] @@ -216,17 +356,17 @@ struct CuActionRequest { scroll_amount: Option, duration: Option, region: Option<[f64; 4]>, + /// Display id for `screenshot` / `zoom` (default 0, the primary; `record_start` uses + /// its own `display` field with the same meaning). Unknown ids get an error reply. + display: Option, } fn ok_json() -> String { "{\"result\":\"ok\"}".to_string() } -fn handle_action( - req: CuActionRequest, - tx: &smithay::reexports::calloop::channel::Sender, -) -> String { - let result = handle_action_inner(req, tx); +fn handle_action(req: CuActionRequest, backend: &dyn CuBackend) -> String { + let result = handle_action_inner(req, backend); match result { Ok(response) => response, Err(e) => format!("{{\"error\":\"{}\"}}", e.replace('"', "\\\"")), @@ -236,20 +376,17 @@ fn handle_action( /// Turn one parsed Computer Use request into a real action on the captured desktop — the /// bridge that lets an external AI agent see and drive the session. /// -/// Every action ultimately becomes a compositor thread command or a framebuffer read, so this is -/// where the API's vocabulary meets the running compositor. Coordinates are clamped to the current +/// Every action ultimately becomes a backend input primitive or a framebuffer read, so this is +/// where the API's vocabulary meets the running desktop. Coordinates are clamped to the current /// framebuffer so a mistaken agent click cannot address off-screen pixels; keyboard and pointer /// actions translate into input events; and actions that must look at the screen (`screenshot`, /// `zoom`) request a fresh capture first, so the agent reasons about the current frame rather than /// a stale one. Returns the JSON response, or an error string explaining why the action could not /// run. -fn handle_action_inner( - req: CuActionRequest, - tx: &smithay::reexports::calloop::channel::Sender, -) -> Result { +fn handle_action_inner(req: CuActionRequest, b: &dyn CuBackend) -> Result { let sleep_ms = |ms: u64| thread::sleep(Duration::from_millis(ms)); - let (fb_w, fb_h) = get_fb_size(tx)?; + let (fb_w, fb_h) = b.fb_size()?; let handle_coord = |coord: [f64; 2]| -> (f64, f64) { ( @@ -265,7 +402,7 @@ fn handle_action_inner( match req.action.as_str() { "screenshot" => { - let png = screenshot(tx)?; + let png = b.screenshot_png(req.display.unwrap_or(0))?; let b64 = BASE64.encode(&png); Ok(format!("{{\"data\":\"{}\"}}", b64)) } @@ -273,34 +410,34 @@ fn handle_action_inner( "mouse_move" => { let coord = req.coordinate.ok_or("Missing coordinate")?; let (fx, fy) = handle_coord(coord); - send_mouse_move(tx, fx, fy); + b.mouse_move(fx, fy); Ok(ok_json()) } "left_click" | "right_click" | "middle_click" => { - let btn: u32 = match req.action.as_str() { - "left_click" => BTN_LEFT, - "right_click" => BTN_RIGHT, - _ => BTN_MIDDLE, + let btn = match req.action.as_str() { + "left_click" => CuButton::Left, + "right_click" => CuButton::Right, + _ => CuButton::Middle, }; if let Some(coord) = req.coordinate { let (fx, fy) = handle_coord(coord); - send_mouse_move(tx, fx, fy); + b.mouse_move(fx, fy); sleep_ms(30); } if let Some(ref mod_name) = req.text { if let Some(sc) = handle_modifier(mod_name) { - send_key(tx, sc, true); + b.key(sc, true); sleep_ms(20); } } - send_mouse_button(tx, btn, true); + b.button(btn, true); sleep_ms(20); - send_mouse_button(tx, btn, false); + b.button(btn, false); if let Some(ref mod_name) = req.text { if let Some(sc) = handle_modifier(mod_name) { sleep_ms(10); - send_key(tx, sc, false); + b.key(sc, false); } } Ok(ok_json()) @@ -310,25 +447,25 @@ fn handle_action_inner( let n = if req.action == "double_click" { 2 } else { 3 }; if let Some(coord) = req.coordinate { let (fx, fy) = handle_coord(coord); - send_mouse_move(tx, fx, fy); + b.mouse_move(fx, fy); sleep_ms(30); } if let Some(ref mod_name) = req.text { if let Some(sc) = handle_modifier(mod_name) { - send_key(tx, sc, true); + b.key(sc, true); sleep_ms(20); } } for _ in 0..n { - send_mouse_button(tx, BTN_LEFT, true); + b.button(CuButton::Left, true); sleep_ms(10); - send_mouse_button(tx, BTN_LEFT, false); + b.button(CuButton::Left, false); sleep_ms(10); } if let Some(ref mod_name) = req.text { if let Some(sc) = handle_modifier(mod_name) { sleep_ms(10); - send_key(tx, sc, false); + b.key(sc, false); } } Ok(ok_json()) @@ -339,91 +476,150 @@ fn handle_action_inner( let end = req.coordinate.ok_or("Missing coordinate")?; let (sx, sy) = handle_coord(start); let (ex, ey) = handle_coord(end); - send_mouse_move(tx, sx, sy); + b.mouse_move(sx, sy); sleep_ms(30); - send_mouse_button(tx, BTN_LEFT, true); + b.button(CuButton::Left, true); sleep_ms(30); - send_mouse_move(tx, ex, ey); + b.mouse_move(ex, ey); sleep_ms(30); - send_mouse_button(tx, BTN_LEFT, false); + b.button(CuButton::Left, false); Ok(ok_json()) } "left_mouse_down" => { - send_mouse_button(tx, BTN_LEFT, true); + b.button(CuButton::Left, true); Ok(ok_json()) } "left_mouse_up" => { - send_mouse_button(tx, BTN_LEFT, false); + b.button(CuButton::Left, false); Ok(ok_json()) } "type" => { let text = req.text.as_deref().ok_or("Missing text")?; - for chunk in text.as_bytes().chunks(50) { - let chunk_str = std::str::from_utf8(chunk).map_err(|_| "Invalid UTF-8")?; - for ch in chunk_str.chars() { + let mut resolver = KeyResolver::new(b); + let syms: Vec = text.chars().map(keysym_for_char).collect(); + resolver.prefetch(&syms); + // Base+AltGr resolution stays the preferred path; only what the active keymap + // cannot reach at all goes through the backend's transient-bind fallback. + let mut unresolved: Vec = Vec::new(); + for (ch, &sym) in text.chars().zip(&syms) { + if ch != '\n' + && sym != 0 + && resolver.resolve(sym).is_none() + && !unresolved.contains(&sym) + { + unresolved.push(sym); + } + } + b.with_transient_keysyms(&unresolved, &mut |bound| { + for (i, (ch, &sym)) in text.chars().zip(&syms).enumerate() { + if i > 0 && i % 50 == 0 { + sleep_ms(20); + } if ch == '\n' { - send_key(tx, 36, true); + b.key(36, true); sleep_ms(10); - send_key(tx, 36, false); + b.key(36, false); sleep_ms(10); continue; } - if let Some((sc, need_shift)) = scancode_for_char(ch) { - if need_shift { - send_key(tx, 50, true); + if let Some((sc, level)) = resolver.resolve(sym) { + let level_mods = level_modifiers(level, 0, b.altgr_keycode()); + for &m in &level_mods { + b.key(m, true); sleep_ms(5); - send_key(tx, sc, true); - sleep_ms(10); - send_key(tx, sc, false); - send_key(tx, 50, false); - } else { - send_key(tx, sc, true); - sleep_ms(10); - send_key(tx, sc, false); } + b.key(sc, true); + sleep_ms(10); + b.key(sc, false); + for &m in level_mods.iter().rev() { + b.key(m, false); + } + sleep_ms(8); + } else if let Some(&kc) = bound.get(&sym) { + // Transient binds sit at the plain level: no modifiers needed. + b.key(kc, true); + sleep_ms(10); + b.key(kc, false); sleep_ms(8); } } - sleep_ms(20); - } + }); Ok(ok_json()) } "key" => { let text = req.text.as_deref().ok_or("Missing text")?; - let parts: Vec<&str> = text.split('+').collect(); + let mut resolver = KeyResolver::new(b); let mut mods: Vec = Vec::new(); - let mut main_key: Option = None; - for part in &parts { + let mut main_key: Option<(u32, u32)> = None; + // Keysym of the last main-key spec the active keymap could not place (cleared + // when a later part resolves): typed via the transient-bind fallback. + let mut unresolved_sym: Option = None; + for part in text.split('+') { let trimmed = part.trim(); if let Some(sc) = scancode_for_keyname(trimmed) { if is_modifier(sc) { mods.push(sc); } else { - main_key = Some(sc); + main_key = Some((sc, 0)); + unresolved_sym = None; + } + } else { + let sym = keysym_for_key_spec(trimmed); + if let Some(hit) = resolver.resolve(sym) { + main_key = Some(hit); + unresolved_sym = None; + } else if sym != 0 { + main_key = None; + unresolved_sym = Some(sym); } } } - for &sc in &mods { - send_key(tx, sc, true); - sleep_ms(10); - } - if let Some(sc) = main_key { - send_key(tx, sc, true); - sleep_ms(30); - send_key(tx, sc, false); - } else if let Some(&last_mod) = mods.last() { - send_key(tx, last_mod, true); - sleep_ms(30); - send_key(tx, last_mod, false); - } - for &sc in mods.iter().rev() { - sleep_ms(10); - send_key(tx, sc, false); - } + let unresolved: Vec = unresolved_sym.into_iter().collect(); + b.with_transient_keysyms(&unresolved, &mut |bound| { + let main_key = main_key.or_else(|| { + // Transient binds sit at the plain level: no modifiers needed. + unresolved_sym.and_then(|s| bound.get(&s)).map(|&kc| (kc, 0)) + }); + for &sc in &mods { + b.key(sc, true); + sleep_ms(10); + } + if let Some((sc, level)) = main_key { + // Modifiers reaching the key's shift level are added unless the spec + // already asked for them (Shift as 50/62, AltGr as 108 or the backend's + // resolved level-3 keycode). + let altgr = b.altgr_keycode(); + let held = mods.iter().fold(0u32, |m, &sc| match sc { + 50 | 62 => m | 1, + sc if sc == KC_ALTGR || sc == altgr => m | 2, + _ => m, + }); + let level_mods = level_modifiers(level, held, altgr); + for &m in &level_mods { + b.key(m, true); + sleep_ms(10); + } + b.key(sc, true); + sleep_ms(30); + b.key(sc, false); + for &m in level_mods.iter().rev() { + sleep_ms(10); + b.key(m, false); + } + } else if let Some(&last_mod) = mods.last() { + b.key(last_mod, true); + sleep_ms(30); + b.key(last_mod, false); + } + for &sc in mods.iter().rev() { + sleep_ms(10); + b.key(sc, false); + } + }); Ok(ok_json()) } @@ -431,10 +627,25 @@ fn handle_action_inner( let text = req.text.as_deref().ok_or("Missing text")?; let duration = req.duration.ok_or("Missing duration")?; let duration = duration.min(100.0); - if let Some(sc) = scancode_for_keyname(text) { - send_key(tx, sc, true); + let trimmed = text.trim(); + let mut resolver = KeyResolver::new(b); + let hit = match scancode_for_keyname(trimmed) { + Some(sc) => Some((sc, 0)), + None => resolver.resolve(keysym_for_key_spec(trimmed)), + }; + if let Some((sc, level)) = hit { + let level_mods = level_modifiers(level, 0, b.altgr_keycode()); + for &m in &level_mods { + b.key(m, true); + sleep_ms(10); + } + b.key(sc, true); sleep_ms((duration * 1000.0) as u64); - send_key(tx, sc, false); + b.key(sc, false); + for &m in level_mods.iter().rev() { + sleep_ms(10); + b.key(m, false); + } } Ok(ok_json()) } @@ -444,12 +655,12 @@ fn handle_action_inner( let amount = req.scroll_amount.unwrap_or(1).max(0) as f64; if let Some(coord) = req.coordinate { let (fx, fy) = handle_coord(coord); - send_mouse_move(tx, fx, fy); + b.mouse_move(fx, fy); sleep_ms(30); } if let Some(ref mod_name) = req.text { if let Some(sc) = handle_modifier(mod_name) { - send_key(tx, sc, true); + b.key(sc, true); sleep_ms(20); } } @@ -460,19 +671,19 @@ fn handle_action_inner( "right" => (amount, 0.0), _ => return Err(format!("Invalid scroll_direction: {}", dir)), }; - send_scroll(tx, dx, dy); + b.scroll(dx, dy); sleep_ms(30); if let Some(ref mod_name) = req.text { if let Some(sc) = handle_modifier(mod_name) { sleep_ms(10); - send_key(tx, sc, false); + b.key(sc, false); } } Ok(ok_json()) } "cursor_position" => { - let (x, y) = cursor_position(tx)?; + let (x, y) = b.cursor_pos()?; Ok(format!("{{\"text\":\"X={},Y={}\"}}", x.round() as i32, y.round() as i32)) } @@ -484,14 +695,18 @@ fn handle_action_inner( "zoom" => { let region = req.region.ok_or("Missing region")?; + let display = req.display.unwrap_or(0); + // The region clamps against the TARGET display's own framebuffer, which need + // not match the primary's. + let (dw, dh) = b.display_fb_size(display)?; let (x0, y0, x1, y1) = (region[0], region[1], region[2], region[3]); - let left = clamp(x0.round() as u32, 0, fb_w as u32 - 1); - let top = clamp(y0.round() as u32, 0, fb_h as u32 - 1); - let right = clamp(x1.round() as u32, left + 1, fb_w as u32); - let bottom = clamp(y1.round() as u32, top + 1, fb_h as u32); + let left = clamp(x0.round() as u32, 0, dw as u32 - 1); + let top = clamp(y0.round() as u32, 0, dh as u32 - 1); + let right = clamp(x1.round() as u32, left + 1, dw as u32); + let bottom = clamp(y1.round() as u32, top + 1, dh as u32); let crop_w = right - left; let crop_h = bottom - top; - let png = screenshot(tx)?; + let png = b.screenshot_png(display)?; if crop_w > 0 && crop_h > 0 { if let Ok(img) = image::load_from_memory(&png) { let cropped = img.crop_imm(left, top, crop_w, crop_h); @@ -510,19 +725,131 @@ fn handle_action_inner( } } +static WAYLAND_TX: Mutex>> = + Mutex::new(None); + +/// Body of `POST /record_start`. All fields are optional; unset ones fall back to the +/// `PIXELFLUX_RECORD_*` environment variables and built-in defaults. +#[derive(Deserialize, Default)] +struct RecordStartRequest { + /// Output MP4 path (default: `$PIXELFLUX_RECORD`, else `/tmp/pixelflux-record-.mp4`). + path: Option, + /// Wayland output id to record (default 0; ignored on X11). + display: Option, + /// Capture fps cap for a recorder-owned capture. + fps: Option, + /// Bitrate override in kbps for a recorder-owned capture. + bitrate_kbps: Option, +} + +/// Handle the recorder REST endpoints sharing the CU server: `record_start`, +/// `record_stop` and `record_status`, all one JSON round-trip into the same recorder +/// implementation the Python API and env vars use. Returns `None` for any other URL. +fn handle_record_endpoint(url: &str, body: &str) -> Option { + let reply = match url { + "/record_start" => { + let req: RecordStartRequest = if body.trim().is_empty() { + RecordStartRequest::default() + } else { + match serde_json::from_str(body) { + Ok(r) => r, + Err(e) => return Some(format!("{{\"error\":\"Invalid JSON: {}\"}}", e)), + } + }; + let path = req + .path + .or_else(|| std::env::var("PIXELFLUX_RECORD").ok().filter(|p| !p.is_empty())) + .unwrap_or_else(|| { + let ts = std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .map(|d| d.as_secs()) + .unwrap_or(0); + format!("/tmp/pixelflux-record-{ts}.mp4") + }); + let mut opts = crate::recorder::RecordOptions::from_env(path); + if let Some(d) = req.display { + opts.display_id = d; + } + if let Some(f) = req.fps { + opts.fps = f; + } + if let Some(b) = req.bitrate_kbps { + opts.bitrate_kbps = b; + } + match crate::recorder::start(opts) { + Ok(s) => crate::recorder::status_to_json(&s).to_string(), + Err(e) => serde_json::json!({ "error": e }).to_string(), + } + } + "/record_stop" => match crate::recorder::stop() { + Ok(s) => { + let mut v = crate::recorder::status_to_json(&s); + v["stopped"] = serde_json::Value::Bool(true); + v.to_string() + } + Err(e) => serde_json::json!({ "error": e }).to_string(), + }, + "/record_status" => match crate::recorder::status() { + Some(s) => crate::recorder::status_to_json(&s).to_string(), + None => serde_json::json!({ "active": false }).to_string(), + }, + _ => return None, + }; + Some(reply) +} + +/// Make the Wayland compositor the preferred CU backend: once a live calloop sender is +/// registered, every subsequent request routes to it instead of an X11 connection. +pub fn register_wayland_backend(tx: smithay::reexports::calloop::channel::Sender) { + *WAYLAND_TX.lock().unwrap() = Some(tx); +} + +/// The registered compositor's command channel, if a Wayland compositor is running in this +/// process. The recorder uses it to attach to (or start) a capture without any Python client. +pub(crate) fn wayland_command_sender( +) -> Option> { + WAYLAND_TX.lock().unwrap().clone() +} + +/// Start the CU server if `PIXELFLUX_CU` names a port. Guarded so that exactly one server +/// binds per process no matter how many call sites (module import, Wayland compositor init) +/// race to spawn it; backend selection stays per-request, so a server bound at import serves +/// a compositor that only starts later. +pub fn spawn_cu_from_env() { + let Ok(port_str) = std::env::var("PIXELFLUX_CU") else { return }; + let Ok(port) = port_str.parse::() else { + println!("[ComputerUse] Invalid PIXELFLUX_CU value: '{}' - expected a port number", port_str); + return; + }; + static SPAWNED: OnceLock<()> = OnceLock::new(); + let mut first = false; + SPAWNED.get_or_init(|| { first = true; }); + if first { + thread::spawn(move || run_cu_server(port)); + } +} + +/// Pick the backend for one request: the registered Wayland compositor when present, +/// otherwise a fresh private connection to `DISPLAY`. The X11 connection is per-request so a +/// restarted X server never leaves the CU thread holding a dead connection. +fn resolve_backend() -> Result, String> { + if let Some(tx) = WAYLAND_TX.lock().unwrap().clone() { + return Ok(Box::new(CuWaylandBackend { tx })); + } + crate::x11::computer_use::CuX11Backend::connect() + .map(|be| Box::new(be) as Box) +} + /// Expose the captured desktop to an AI agent over HTTP, so a Computer Use client can drive /// the session much as a human viewer would. /// /// It runs on its own thread listening on `0.0.0.0:` for POST `/computer-use` JSON actions -/// (screenshot, mouse_move, click, key, scroll, …). Framebuffer dimensions are re-queried from the -/// compositor on every request rather than cached, because the stream can start, stop, or resize -/// underneath the agent and a stale size would misplace every coordinate. A screenshot forces a -/// one-frame GPU readback when the pipeline is in zero-copy mode, since that path never otherwise -/// brings host pixels back to the CPU and the agent needs an image to look at. -pub fn run_cu_server( - tx: smithay::reexports::calloop::channel::Sender, - port: u16, -) { +/// (screenshot, mouse_move, click, key, scroll, …). The backend and the framebuffer dimensions +/// are re-resolved on every request rather than cached, because the compositor can start, the +/// stream can resize, or the X server can restart underneath the agent — a stale size would +/// misplace every coordinate. On Wayland a screenshot forces a one-frame GPU readback when the +/// pipeline is in zero-copy mode; on X11 it is a one-shot `GetImage` of the root window. +pub fn run_cu_server(port: u16) { println!( "[ComputerUse] Server listening on port {}", port, @@ -536,6 +863,7 @@ pub fn run_cu_server( } }; + let mut last_backend = ""; for mut request in server.incoming_requests() { let mut body = String::new(); if let Err(e) = request.as_reader().read_to_string(&mut body) { @@ -549,6 +877,17 @@ pub fn run_cu_server( continue; } + if let Some(json) = handle_record_endpoint(request.url(), &body) { + let _ = request.respond( + tiny_http::Response::from_string(json) + .with_status_code(200) + .with_header( + "Content-Type: application/json".parse::().unwrap() + ) + ); + continue; + } + let parsed: CuActionRequest = match serde_json::from_str(&body) { Ok(r) => r, Err(e) => { @@ -563,7 +902,16 @@ pub fn run_cu_server( } }; - let json_response = handle_action(parsed, &tx); + let json_response = match resolve_backend() { + Ok(backend) => { + if backend.name() != last_backend { + last_backend = backend.name(); + println!("[ComputerUse] Using {} backend", last_backend); + } + handle_action(parsed, backend.as_ref()) + } + Err(e) => format!("{{\"error\":\"{}\"}}", e.replace('"', "\\\"")), + }; let _ = request.respond( tiny_http::Response::from_string(json_response) .with_status_code(200) diff --git a/pixelflux/src/encoders/nvenc.rs b/pixelflux/src/encoders/nvenc.rs index 4ae22fa..a74c52a 100644 --- a/pixelflux/src/encoders/nvenc.rs +++ b/pixelflux/src/encoders/nvenc.rs @@ -1417,8 +1417,7 @@ impl NvencEncoder { /// — a `0x04` tag, a picture-type byte derived from the *actual* encoded `pictureType` /// (IDR = `0x01`, I = `0x02`, P = `0x00`) rather than the `force_idr` request, the low 16 bits /// of the frame number, a zero field, and the width and height (all big-endian). - /// 4. **Emit and fan out**: append the encoded bytes, mirror them to the recording sink when one - /// is attached, unlock the bitstream, and return the framed buffer. + /// 4. **Emit**: append the encoded bytes, unlock the bitstream, and return the framed buffer. unsafe fn submit_frame( &mut self, mapped_buffer: NV_ENC_INPUT_PTR, diff --git a/pixelflux/src/encoders/overlay.rs b/pixelflux/src/encoders/overlay.rs index e4978f8..1015c5e 100644 --- a/pixelflux/src/encoders/overlay.rs +++ b/pixelflux/src/encoders/overlay.rs @@ -211,12 +211,15 @@ impl OverlayState { } } - /// Render element for a software cursor `image` at `pos` (logical coords). + /// Render element for a software cursor `image` at logical `pos` on an output with + /// fractional `scale`; the position converts to physical here so the composited cursor + /// lands where the damage tracker and clients (which work in physical pixels) expect it. pub fn get_cursor_element( &self, renderer: &mut R, image: xcursor::parser::Image, - pos: Point, + pos: Point, + scale: f64, ) -> Option> where R: Renderer + ImportMem, @@ -233,7 +236,7 @@ impl OverlayState { let hot: Point = (image.xhot as i32, image.yhot as i32).into(); - let phys_pos: Point = (pos.x, pos.y).into(); + let phys_pos = pos.to_physical(smithay::utils::Scale::from(scale)).to_i32_round(); MemoryRenderBufferRenderElement::from_buffer( renderer, diff --git a/pixelflux/src/encoders/software.rs b/pixelflux/src/encoders/software.rs index 362b50c..f25afd6 100644 --- a/pixelflux/src/encoders/software.rs +++ b/pixelflux/src/encoders/software.rs @@ -9,8 +9,7 @@ //! Frames are split into horizontal stripes processed in parallel via rayon. Each stripe is //! independently hashed against the previous frame for change detection, and only dirty stripes //! are encoded. The x264 path maintains per-stripe encoder state across frames for inter-prediction; -//! the JPEG path is stateless. An optional recording sink receives the H.264 NAL units for -//! muxing into a file or socket stream. +//! the JPEG path is stateless. use crate::RustCaptureSettings; use rayon::prelude::*; @@ -371,10 +370,9 @@ impl H264EncoderWrapper { /// /// The boolean return is load-bearing: `x264_encoder_encode` can legitimately produce nothing on /// a given call, and the caller must forward a stripe only when real bytes exist — never an empty - /// or header-only packet. The output also serves two consumers at once, which is why framing is - /// conditional: the transport needs the pipeline's small wire header to route the stripe, while - /// the optional recording sink needs the *bare* Annex-B elementary stream with no wire header so - /// it can be muxed directly. + /// or header-only packet. Framing is conditional because the transport needs the pipeline's small + /// wire header to route the stripe, while `omit_headers` consumers take the bare Annex-B + /// elementary stream. /// /// 1. **Picture setup**: wraps the borrowed Y/U/V planes and their strides in an /// `x264_picture_t` with the encoder's CSP, stamps the presentation timestamp with `frame_id`, @@ -387,9 +385,8 @@ impl H264EncoderWrapper { /// the client keys its decode-recovery on the frame type it truly received (IDR = `0x01`, /// I = `0x02`, else `0x00`), then the caller's `fixed_header` (frame number, y-start, width, /// height). With `omit_headers` the output is bare Annex-B. - /// 4. **Payload + recording**: every NAL payload is appended to `output_buf`, and each is also - /// forwarded to the recording sink when one is attached — giving the sink raw Annex-B without - /// the wire header. + /// 4. **Payload**: every NAL payload is appended to `output_buf` after the optional header, + /// so the bytes past the wire header are always a contiguous Annex-B access unit. #[allow(clippy::too_many_arguments)] pub fn encode_with_headers( &mut self, @@ -581,8 +578,8 @@ impl StripeState { /// /// # Fields /// -/// * `data` - Compressed payload (JPEG or H.264 NAL units). `Arc`-shared so the recording -/// tap can retain the frame for its client queues without copying the bytes. +/// * `data` - Compressed payload (JPEG or H.264 NAL units). `Arc`-shared so every +/// delivery-layer consumer can retain the frame without copying the bytes. /// * `data_type` - Codec tag: **1 = JPEG**, **2 = H.264**. /// * `stripe_y_start` - Y pixel coordinate of the stripe's top edge within the frame. /// * `stripe_height` - Height of the stripe in pixels. @@ -603,8 +600,7 @@ pub struct EncodedStripe { /// parallel stripes; bandwidth is precious, so unchanged stripes are skipped. Each stripe is /// independently hashed against the previous frame for change detection, and only dirty stripes /// are encoded. The x264 path maintains per-stripe encoder state across frames for -/// inter-prediction; the JPEG path is stateless. The recording sink is attached only in -/// single-stripe mode because several independent sub-frame bitstreams cannot be muxed. +/// inter-prediction; the JPEG path is stateless. /// /// # Arguments /// @@ -676,10 +672,10 @@ pub struct EncodedStripe { /// OpenH264 encoder — one fewer than the available cores, clamped to `[1, 4]`. The slice threads /// keep the in-frame encode latency inside the frame budget at high resolutions; the cap is four /// because `zerolatency` makes x264 slice-threaded and more than four slices trips decode -/// glitches in some Chromium builds, and the minus-one leaves headroom for the capture thread. Multiple stripes instead run across the -/// rayon pool with a single x264 thread and one conversion band each, since the parallelism there -/// already comes from encoding the stripes concurrently. The recording sink is attached only in single-stripe mode, because the several -/// independent sub-frame bitstreams of striped mode cannot be muxed into one recording. +/// glitches in some Chromium builds, and the minus-one leaves headroom for the capture thread. +/// Multiple stripes instead run across the rayon pool with a single x264 thread and one +/// conversion band each, since the parallelism there already comes from encoding the stripes +/// concurrently. #[allow(clippy::too_many_arguments)] pub fn encode_cpu( stripes: &mut Vec, diff --git a/pixelflux/src/encoders/vaapi.rs b/pixelflux/src/encoders/vaapi.rs index 2600e53..77e09bf 100644 --- a/pixelflux/src/encoders/vaapi.rs +++ b/pixelflux/src/encoders/vaapi.rs @@ -138,9 +138,8 @@ fn ff_err_str(err: i32) -> String { /// /// `current_qp` / `qp_hysteresis_counter` drive the CQP hysteresis in `update_qp`. `cbr_mode`, /// `current_bitrate_kbps`, `current_vbv_mult`, and `current_kf_s` cache the live rate-control state -/// so `reconfigure_rate` re-opens the codec only when a value actually changes. `recording_sink` is -/// the optional Unix-socket H.264 fan-out; `omit_stripe_headers` drops the 10-byte framing when the -/// consumer wants a bare Annex-B stream. +/// so `reconfigure_rate` re-opens the codec only when a value actually changes. +/// `omit_stripe_headers` drops the 10-byte framing when the consumer wants a bare Annex-B stream. pub struct VaapiEncoder { encoder_ctx: *mut ff::AVCodecContext, codec: *const ff::AVCodec, @@ -797,7 +796,7 @@ impl VaapiEncoder { /// described to the client as one full-height stripe: tag `0x04`, a keyframe flag (`0x01`/`0x00` /// from `AV_PKT_FLAG_KEY`), the frame number as a big-endian `u16`, a `0` y-start (a whole frame /// starts at the top), then width and height as big-endian `u16`s, followed by the raw Annex-B - /// payload. The recording sink intercepts frames at the delivery layer. + /// payload. /// `omit_stripe_headers` drops the header on the network path too, for a consumer that already /// wants raw Annex-B. The loop drains `avcodec_receive_packet` until the encoder is empty, /// unref'ing each packet before the next iteration. diff --git a/pixelflux/src/lib.rs b/pixelflux/src/lib.rs index e7418f4..0dce26a 100644 --- a/pixelflux/src/lib.rs +++ b/pixelflux/src/lib.rs @@ -28,6 +28,7 @@ //! | [`x11`] | X11/XShm capture loop, XFixes out-of-band cursor monitor, and stripe dispatch | //! | [`pipeline`] | Frame-processing policy shared by both backends (send/QP/keyframe decisions) | //! | [`recording_sink`] | Unix-socket H.264 fan-out for external recording | +//! | [`recorder`] | Built-in MP4 recorder (fMP4 muxer + Python/env/REST control surfaces) | //! | [`computer_use`] | HTTP API for AI-agent desktop control (screenshots, input injection) | //! | [`nvgpufilter`] | Multi-GPU NVENC device filtering via ioctl | //! @@ -96,7 +97,7 @@ use smithay::{ output::{Mode as OutputMode, Output, PhysicalProperties, Scale as OutputScale, Subpixel}, reexports::{ calloop::{ - channel::Event as CalloopEvent, generic::Generic, timer::{TimeoutAction, Timer}, + generic::Generic, timer::{TimeoutAction, Timer}, EventLoop, Interest, Mode, PostAction, }, pixman, @@ -113,7 +114,6 @@ use smithay::{ shell::xdg::XdgShellState, shm::ShmState, socket::ListeningSocketSource, - virtual_keyboard::VirtualKeyboardManagerState, pointer_warp::PointerWarpManager, relative_pointer::RelativePointerManagerState, pointer_constraints::PointerConstraintsState, @@ -173,6 +173,8 @@ pub mod encoders { pub mod wayland; /// Unix-socket H.264 recording fan-out for external capture tools. pub mod recording_sink; +/// Built-in MP4 recorder: independent capture-to-file with Python/env/REST control. +pub mod recorder; /// HTTP server implementing the Anthropic Computer Use spec for AI agent desktop control. pub mod computer_use; /// Frame-processing policy shared by the X11 and Wayland backends. @@ -226,7 +228,9 @@ use encoders::overlay::OverlayState; use encoders::software::MAX_STRIPE_CAPACITY; use encoders::vaapi::VaapiEncoder; -use wayland::cursor::Cursor; +use smithay::reexports::wayland_protocols_misc::zwp_virtual_keyboard_v1::server::zwp_virtual_keyboard_manager_v1::ZwpVirtualKeyboardManagerV1; + +use wayland::cursor::{Cursor, CursorJob}; use wayland::frontend::{AppState, ClientState, FocusTarget, GpuEncoder, next_serial, wayland_time, wayland_utime}; smithay::backend::renderer::element::render_elements! { @@ -534,20 +538,79 @@ pub(crate) fn extract_settings(settings: &Bound<'_, PyAny>) -> PyResult, RustCaptureSettings), - StopCapture, + /// Start (or in-place reconfigure) the capture bound to output `display_id`. + /// `callback` is the Python per-frame delivery target; `None` starts an internal + /// capture with no Python consumer (the built-in recorder taps the delivery layer). + StartCapture { display_id: u32, callback: Option>, settings: RustCaptureSettings }, + /// Stop the capture bound to output `display_id` (other displays keep running). + StopCapture { display_id: u32 }, + /// Create an additional output: `WxH` physical pixels at fractional `scale`, mapped + /// into the layout at offset `(x, y)`. Replies false when the id is taken/reserved or + /// the GPU render target cannot be allocated. + CreateOutput { + id: u32, + width: i32, + height: i32, + x: i32, + y: i32, + scale: f64, + reply: std::sync::mpsc::Sender, + }, + /// Destroy a secondary output: its capture ends, its windows relocate to the primary + /// output. Replies false for the primary (id 0) or an unknown id. + DestroyOutput { id: u32, reply: std::sync::mpsc::Sender }, + /// Remap an existing output (the primary included) to layout offset `(x, y)`: the + /// Space mapping, the offsets used for absolute input injection and cursor + /// compositing, and the windows placed on it all follow, and the output is damaged so + /// the next frames render correctly. Replies false for an unknown id. + RepositionOutput { id: u32, x: i32, y: i32, reply: std::sync::mpsc::Sender }, + /// Reply with every live output as `(id, x, y, width, height, scale, capturing)`. + ListOutputs { reply: std::sync::mpsc::Sender> }, + /// Move the window with the given id onto output `output_id` (fullscreened there). + MoveWindowToOutput { window_id: u32, output_id: u32, reply: std::sync::mpsc::Sender }, + /// Reply with every mapped window as `(window_id, title, app_id, output_id)`. + ListWindows { reply: std::sync::mpsc::Sender> }, SetCursorCallback(Py), SetClipboardCallback(Py), /// Server-side clipboard offer: the compositor owns the selection and serves `data` as /// `mime` (plus text aliases) to pasting clients. SetClipboard { mime: String, data: Vec }, KeyboardKey { scancode: u32, state: u32 }, - /// Swap the seat keyboard's xkb keymap (XKB_KEYMAP_FORMAT_TEXT_V1 text); the caller owns - /// keysym-to-keycode policy and injects the keycodes it defined via `KeyboardKey`. + /// Set the seat's BASE keymap from a full XKB_KEYMAP_FORMAT_TEXT_V1 string. The + /// compositor's keymap policy rebuilds on top: overlay binds are re-spliced onto the new + /// base (same keycodes) and the combined keymap is applied in one swap. SetKeymapString(String), + /// Set the seat's BASE layout from RMLVO names (empty strings = xkbcommon defaults); + /// replies whether compilation succeeded. Overlay binds rebuild on top as for + /// `SetKeymapString`. + SetXkbLayout { + rules: String, + model: String, + layout: String, + variant: String, + options: String, + reply: std::sync::mpsc::Sender, + }, + /// Resolve keysyms to `(keycode, level)` against the seat keymap, overlay-binding every + /// keysym the base cannot produce — ONE keymap swap for the whole batch, and a keycode that + /// is currently pressed is never recycled. `(0, 0)` marks an unbindable keysym. + BindKeysyms { + keysyms: Vec, + reply: std::sync::mpsc::Sender>, + }, + /// Debug/verification readback: currently pressed xkb keycodes plus the modifier state + /// bitmask (1 ctrl, 2 shift, 4 alt, 8 logo, 16 caps, 32 num, 64 altgr, 128 level5). + GetKeyboardState { + reply: std::sync::mpsc::Sender<(Vec, u32)>, + }, /// Reply with the smithay keyboard's keymap as an XKB_KEYMAP_FORMAT_TEXT_V1 string so a /// consumer (selkies) can build its reverse keysym map from the IDENTICAL keymap. GetXkbKeymap { reply: std::sync::mpsc::Sender }, + /// Ack once every previously queued command has been fully processed (the channel is + /// FIFO). The atexit sweep sends StopCapture + Barrier and waits, so the interpreter never + /// exits while the calloop thread is still mid-teardown (an NVENC/CUDA session drop racing + /// process exit segfaults). + Barrier { reply: std::sync::mpsc::Sender<()> }, PointerMotion { x: f64, y: f64 }, PointerRelativeMotion { dx: f64, dy: f64 }, /// `btn` is an evdev `BTN_` code by contract (e.g. 272 = BTN_LEFT, 273 = BTN_RIGHT, @@ -556,19 +619,29 @@ pub enum ThreadCommand { PointerButton { btn: u32, state: u32 }, PointerAxis { x: f64, y: f64 }, UpdateCursorConfig { render_on_framebuffer: bool }, - /// On-demand keyframe request (client reconnect / decoder reset): forces a send and an IDR - /// even on a static screen. - RequestIdr, - /// Live rate-control change for the Wayland calloop thread (parity with the X11 `rate_dirty` + /// Recreate the cursor theme handles at a new pixel size — the calloop's compositing + /// helper (the burned-in cursor) and, through its job channel, the `wl-cursor` worker's + /// (named-cursor PNG delivery). Replies false for a non-positive size. + SetCursorSize { size: i32, reply: std::sync::mpsc::Sender }, + /// On-demand keyframe request (client reconnect / decoder reset) for one display's + /// capture: forces a send and an IDR even on a static screen. + RequestIdr { display_id: u32 }, + /// Live rate-control change for one display's capture (parity with the X11 `rate_dirty` /// path). Each field is `None` when that dimension is unchanged. - UpdateRate { bitrate_kbps: Option, vbv_multiplier: Option, fps: Option }, - /// Live per-frame tunables (quality / paint-over / streaming / cursor), applied to the - /// calloop's settings and mirrored to the readback encode thread — no capture or encoder - /// restart. - UpdateTunables(LiveTunables), - CuScreenshot { resp: std::sync::mpsc::Sender> }, + UpdateRate { + display_id: u32, + bitrate_kbps: Option, + vbv_multiplier: Option, + fps: Option, + }, + /// Live per-frame tunables (quality / paint-over / streaming / cursor) for one + /// display's capture, mirrored to its readback encode thread — no restart. + UpdateTunables { display_id: u32, tunables: LiveTunables }, + /// One-shot PNG of one output's next rendered frame (0 = primary); an unknown + /// display id replies with an error immediately. + CuScreenshot { display_id: u32, resp: std::sync::mpsc::Sender, String>> }, CuCursorPosition { resp: std::sync::mpsc::Sender<(f64, f64)> }, - CuGetInfo { resp: std::sync::mpsc::Sender<(i32, i32, f64)> }, + CuGetInfo { display_id: u32, resp: std::sync::mpsc::Sender<(i32, i32, f64)> }, } /// Read the kernel driver bound to a render node for encoder routing. @@ -947,6 +1020,8 @@ impl WlEncodeStats { /// flow through `controls`; damage and the IDR request arrive per-frame in `WlFrame`. struct WlEncodeConfig { settings: RustCaptureSettings, + /// Output/display id this encode loop serves; keys the recorder's delivery-layer tap. + display_id: u32, /// GLES readback is RGBA; the pixman framebuffer is BGRA. Selects CSC + encoder input kind. use_gpu: bool, /// Attempt a HW (NVENC/VAAPI) readback session before falling back to the CPU encoders. @@ -1258,6 +1333,7 @@ fn wayland_encode_loop(pool: &WlFramePool, cfg: WlEncodeConfig) -> Option= 0 a device) -/// always wins, and only the unset `-2` sentinel is filled from the auto-picked render node. -/// H.264 output masks the dimensions even, because 4:2:0 needs even width and height. -/// 5. **Encode-path choice + render loop**: only a same-GPU GLES session encodes zero-copy on this -/// calloop thread, because the dmabuf and its EGL context are calloop-affine; every readback -/// flavor (OpenH264, striped x264/JPEG, Pixman, or a cross-GPU hardware encoder) builds its -/// encoders on the dedicated encode thread instead. A high-frequency timer composites the mapped -/// windows plus cursor and watermark, applies the shared paint-over / recovery-IDR policy, and -/// delivers the encoded stripes back to Python through the frame callback. The zero-copy encode -/// waits the GL render fence first, so a hardware encoder reading the dmabuf through CUDA/VA -/// never maps a half-rasterized (torn) frame. -/// 6. **Thread lifecycle**: the Python frame callback runs on a dedicated delivery thread so its -/// GIL never stalls calloop input / control dispatch, and in readback mode the encoders run on -/// the `wl-encode` thread. On a restart or stop the encode thread is torn down before the -/// delivery thread — it feeds the delivery sender and must be gone first — and the retained -/// callbacks are dropped and gated by a process-shutdown flag so nothing fires into a finalizing -/// interpreter. -fn run_wayland_thread( - command_rx: smithay::reexports::calloop::channel::Channel, - command_tx: smithay::reexports::calloop::channel::Sender, - initial_width: i32, - initial_height: i32, - explicit_dri_node: String, - auto_gpu_selected: bool, - cursor_size: i32, +/// Tear down a capture's encode/delivery threads and pools, returning any readback +/// hardware session for in-place reuse by a following start. The encode thread is joined +/// before the delivery sender drops (it feeds that sender), and the zero-copy session (if +/// any) is left on the capture for the caller to reuse or drop. +fn teardown_capture(cap: &mut wayland::frontend::WlCapture) -> Option { + if let Some(p) = cap.encode_pool.take() { + p.shutdown(); + } + let prior = cap.encode_join.take().and_then(|j| j.join().ok()).flatten(); + if let Some(tx) = cap.deliver_tx.take() { + drop(tx); + } + if let Some(j) = cap.deliver_join.take() { + let _ = j.join(); + } + cap.pending_hw_delivery = None; + cap.pending_hw_damage = false; + cap.recording_sink = None; + prior +} + +/// Stop the capture bound to `display_id`, leaving the output (and every other display's +/// capture) running. +fn stop_capture_on_display(state: &mut AppState, display_id: u32) { + let Some(idx) = state.node_idx_for_id(display_id) else { return }; + if let Some(mut cap) = state.output_nodes[idx].capture.take() { + println!("[Wayland] Capture loop stopped (display {display_id})."); + cap.video_encoder = None; + let _ = teardown_capture(&mut cap); + } + wayland_alive().lock().unwrap().remove(&display_id); +} + +/// Start (or in-place reconfigure) the capture bound to output `display_id`: reprogram the +/// output's mode/scale/refresh, size the render targets, fullscreen the display's windows at +/// the new logical size, resolve the encode path (zero-copy vs readback), and spawn the +/// delivery (and readback-mode encode) threads. The single-display behavior of the former +/// global StartCapture is preserved exactly for display 0. +fn start_capture_on_display( + state: &mut AppState, + display_id: u32, + cb: Option>, + mut settings: RustCaptureSettings, ) { - let width: i32 = if initial_width > 0 { initial_width } else { 1024 }; - let height: i32 = if initial_height > 0 { initial_height } else { 768 }; + use smithay::wayland::fractional_scale::with_fractional_scale; - let mut event_loop = EventLoop::::try_new().expect("Unable to create event_loop"); - let display: Display = Display::new().unwrap(); - let dh: DisplayHandle = display.handle(); - unsafe { - if let Ok(lib) = libloading::Library::new("libwayland-server.so.0") { - if let Ok(set_max) = lib.get::( - b"wl_display_set_default_max_buffer_size\0", - ) { - set_max( - dh.backend_handle().display_ptr() as *mut std::ffi::c_void, - 10 * 1024 * 1024, - ); + let Some(node_idx) = state.node_idx_for_id(display_id) else { + eprintln!("[Wayland] StartCapture: no output with display id {display_id}."); + return; + }; + let mut node = state.output_nodes.remove(node_idx); + + if state.auto_gpu_selected && settings.encode_node_index < -1 { + if let Some(idx_str) = state.render_node_path.strip_prefix("/dev/dri/renderD") { + if let Ok(idx) = idx_str.parse::() { + settings.encode_node_index = idx - 128; } - std::mem::forget(lib); } } - let dri_node = explicit_dri_node; + if settings.output_mode == 1 { + settings.width &= !1; + settings.height &= !1; + } - let mut use_gpu = !dri_node.is_empty(); - let render_node_path = dri_node.clone(); + let recording_sink = crate::recording_sink::RecordingSink::try_bind(&settings.recording_socket); - let mut gles_renderer = None; - let mut pixman_renderer = None; - let mut offscreen_buffer: Option<(BufferObject<()>, Dmabuf)> = None; - let mut dmabuf_global = None; - let mut gbm_device_raw = None; - let mut dmabuf_state = DmabufState::new(); + // Tear down this display's previous capture first; its hardware sessions are the + // reuse candidates below (zero-copy inline, readback via the encode config). + let mut prior_zero_copy: Option = None; + let mut prior_readback_encoder: Option = None; + if let Some(mut old) = node.capture.take() { + prior_zero_copy = old.video_encoder.take(); + prior_readback_encoder = teardown_capture(&mut old); + } - let mut gpu_success = false; - if use_gpu { - println!("[Wayland] Initializing GL Renderer using device: {}", dri_node); - let init_res: Result<(), String> = (|| { - let device_path = std::path::Path::new(&dri_node); - let file = File::options().read(true).write(true).open(device_path) - .map_err(|e| format!("Failed to open render device: {}", e))?; - let file_for_alloc = file.try_clone() - .map_err(|e| format!("Failed to clone file for GBM Allocator: {}", e))?; - let gbm_allocator = RawGbmDevice::new(file_for_alloc) - .map_err(|_| "Failed to create Raw GBM Device")?; - let gbm = GbmDevice::new(file) - .map_err(|_| "Failed to create GBM device")?; - let egl = unsafe { EGLDisplay::new(gbm) } - .map_err(|_| "Failed to create EGL display")?; - let context = EGLContext::new(&egl) - .map_err(|_| "Failed to create EGL context")?; - let mut renderer = unsafe { GlesRenderer::new(context) } - .map_err(|_| "Failed to init GlesRenderer")?; - - if let Err(e) = renderer.bind_wl_display(&dh) { - println!("[Wayland] Warning: Failed to bind EGL to Wayland Display (Optional): {:?}", e); + { + // Never panic the compositor thread: an output momentarily without a current + // mode falls back to the requested geometry so the reconfigure below is a + // no-op for size/refresh instead of unwrap-panicking. + let target_refresh = (settings.target_fps * 1000.0).round() as i32; + let (current_w, current_h, current_refresh) = match node.output.current_mode() { + Some(m) => (m.size.w, m.size.h, m.refresh), + None => (settings.width, settings.height, target_refresh), + }; + let current_scale = node.output.current_scale().fractional_scale(); + + if current_w != settings.width + || current_h != settings.height + || (current_scale - settings.scale).abs() > 0.001 + || current_refresh != target_refresh + { + // Allocate the GPU backing for the new dimensions BEFORE committing + // anything: if the driver refuses (VRAM exhaustion, dimensions it will + // not back), the whole reconfigure is skipped and the previous mode + + // buffers stay live. A failed resize must degrade to "no resize", never + // panic the compositor thread. + let mut new_offscreen = None; + let mut gbm_resize_failed = false; + if state.use_gpu { + if let Some(gbm) = state.gbm_device.as_mut() { + match gbm.create_buffer_object( + settings.width as u32, + settings.height as u32, + GbmFormat::Argb8888, + BufferObjectFlags::RENDERING, + ) { + Ok(bo) => { + let dmabuf = create_dmabuf_from_bo(&bo); + new_offscreen = Some((bo, dmabuf)); + } + Err(e) => { + eprintln!( + "[Wayland] GBM buffer resize to {}x{} failed ({:?}); keeping previous output mode.", + settings.width, settings.height, e + ); + gbm_resize_failed = true; + } + } + } } + if gbm_resize_failed { + // The mode commit below is skipped wholesale, so the rest of this + // StartCapture (encoder setup, stored settings) must see the + // dimensions actually live. + settings.width = current_w; + settings.height = current_h; + settings.scale = current_scale; + settings.target_fps = current_refresh as f64 / 1000.0; + } else { + println!( + "[Wayland] Configuring Output {} ({}): {}x{} @ {:.2} FPS (Scale {:.2})", + display_id, node.output.name(), + settings.width, settings.height, settings.target_fps, settings.scale + ); + let new_mode = OutputMode { + size: (settings.width, settings.height).into(), + refresh: target_refresh, + }; + node.output.change_current_state( + Some(new_mode), + Some(Transform::Normal), + Some(OutputScale::Fractional(settings.scale)), + Some(Point::from(node.pos)), + ); + node.output.set_preferred(new_mode); - let formats = Bind::::supported_formats(&renderer) - .ok_or("Failed to query formats")? - .into_iter() - .collect::>(); + let pixel_count = + (settings.width.max(0) as usize) * (settings.height.max(0) as usize); + node.frame_buffer = vec![0u8; pixel_count * 4]; - let node = DrmNode::from_path(device_path) - .map_err(|_| "Failed to create DrmNode")?; - let dmabuf_default_feedback = DmabufFeedbackBuilder::new(node.dev_id(), formats.clone()).build(); + if let Some(off) = new_offscreen.take() { + node.offscreen_buffer = Some(off); + } + } + } - dmabuf_global = Some(if let Ok(default_feedback) = dmabuf_default_feedback { - dmabuf_state.create_global_with_default_feedback::(&dh, &default_feedback) - } else { - dmabuf_state.create_global::(&dh, formats) - }); + let scale = settings.scale.max(0.1); + let logical_width = (settings.width as f64 / scale).round() as i32; + let logical_height = (settings.height as f64 / scale).round() as i32; - let bo = gbm_allocator.create_buffer_object( - width as u32, height as u32, GbmFormat::Argb8888, BufferObjectFlags::RENDERING - ).map_err(|_| "Failed to allocate GBM buffer")?; + for window in state.space.elements() { + if wayland::frontend::window_output_id(window) != display_id { + continue; + } + if let Some(surface) = window.wl_surface() { + node.output.enter(&surface); + with_states(&surface, |states| { + with_fractional_scale(states, |fs| { + fs.set_preferred_scale(scale); + }); + }); + } + if let Some(toplevel) = window.toplevel() { + toplevel.with_pending_state(|state| { + use smithay::reexports::wayland_protocols::xdg::shell::server::xdg_toplevel::State; + state.states.set(State::Fullscreen); + state.states.set(State::Activated); + state.size = Some((logical_width, logical_height).into()); + }); + toplevel.send_configure(); + } + } + } - let dmabuf = create_dmabuf_from_bo(&bo); - offscreen_buffer = Some((bo, dmabuf)); - gbm_device_raw = Some(gbm_allocator); - gles_renderer = Some(renderer); - Ok(()) - })(); + let use_cpu_explicit = settings.use_cpu || settings.encode_node_index == -1; + let gpu_intent = settings.output_mode == 1 && !settings.use_openh264 && !use_cpu_explicit; + if use_cpu_explicit && !(settings.output_mode == 1 && settings.use_openh264) { + println!("[Wayland] CPU encoding selected (use_cpu=true or encode_node_index=-1)."); + } - match init_res { - Ok(_) => gpu_success = true, - Err(e) => { - println!("[Wayland] GPU Initialization failed: {}. Falling back to Software Renderer (Pixman).", e); - use_gpu = false; - } + let mut different_gpu = false; + if gpu_intent { + let encode_node_idx = settings.encode_node_index.max(0); + if !state.render_node_path.is_empty() + && !state.render_node_path.contains(&format!("renderD{}", 128 + encode_node_idx)) + { + different_gpu = true; } } - if !gpu_success { - if !dri_node.is_empty() && !use_gpu { + let mut video_encoder: Option = None; + if gpu_intent && state.use_gpu && !different_gpu { + let encode_driver = get_gpu_driver(settings.encode_node_index.max(0)); + println!( + "[Wayland] Encode Node Index: {} | Driver: {}", + settings.encode_node_index.max(0), encode_driver + ); + + if encode_driver.contains("nvidia") { + let reused = match prior_zero_copy.as_mut() { + Some(GpuEncoder::Nvenc(enc)) => match enc.reconfigure_resolution(&settings) { + Ok(()) => { + println!("[Wayland] NVENC session reconfigured in place."); + true + } + Err(e) => { + eprintln!("[Wayland] NVENC in-place reconfigure unavailable ({e}); rebuilding."); + false + } + }, + _ => false, + }; + if reused { + video_encoder = prior_zero_copy.take(); + } else { + prior_zero_copy = None; + println!("[Wayland] Nvidia Encoder detected. Initializing NVENC..."); + let egl_display = if let Some(renderer) = state.gles_renderer.as_ref() { + renderer.egl_context().display().get_display_handle().handle + } else { + std::ptr::null() + }; + + match NvencEncoder::new(&settings, egl_display) { + Ok(encoder) => { + video_encoder = Some(GpuEncoder::Nvenc(encoder)); + println!("[Wayland] NVENC Encoder initialized successfully."); + } + Err(e) => eprintln!( + "[Wayland] Failed to init NVENC: {}. Falling back to CPU.", + e + ), + } + } } else { - println!("[Wayland] No render node. Initializing Software Renderer (Pixman)."); + prior_zero_copy = None; + println!("[Wayland] Initializing Unified VAAPI Encoder..."); + if settings.video_fullcolor { + println!("[Wayland] 4:4:4 Fullcolor requested. VAAPI does not support this profile reliably. Falling back to CPU."); + } else { + match VaapiEncoder::new(&settings) { + Ok(encoder) => { + video_encoder = Some(GpuEncoder::Vaapi(encoder)); + println!("[Wayland] VAAPI Encoder initialized successfully."); + } + Err(e) => eprintln!( + "[Wayland] Failed to init VAAPI: {}. Falling back to CPU.", + e + ), + } + } } - pixman_renderer = Some(PixmanRenderer::new().expect("Failed to init PixmanRenderer")); - use_gpu = false; } + drop(prior_zero_copy); - let compositor_state = CompositorState::new::(&dh); - let fractional_scale_state = FractionalScaleManagerState::new::(&dh); - let shm_state = ShmState::new::(&dh, vec![]); - let output_state = OutputManagerState::new_with_xdg_output::(&dh); - let mut seat_state = SeatState::new(); - let shell_state = XdgShellState::new::(&dh); - let space = Space::default(); - let layer_shell_state = WlrLayerShellState::new::(&dh); - let data_device_state = DataDeviceState::new::(&dh); - let data_control_state = DataControlState::new::(&dh, None, |_| true); - let virtual_keyboard_state = VirtualKeyboardManagerState::new::(&dh, |_client| true); - let pointer_warp_state = PointerWarpManager::new::(&dh); - let relative_pointer_state = RelativePointerManagerState::new::(&dh); - let pointer_constraints_state = PointerConstraintsState::new::(&dh); + if different_gpu { + println!("[Wayland] Decision: Rendering and Encoding GPUs differ -> Forcing Readback (CPU path for pixels)."); + } + if video_encoder.is_none() { + println!("[Wayland] Decision: Readback path (encode thread) active."); + } else if !different_gpu { + println!("[Wayland] Decision: Zero-Copy path active."); + } - let foreign_toplevel_list = ForeignToplevelListState::new::(&dh); - let xdg_decoration_state = XdgDecorationState::new::(&dh); - let single_pixel_buffer = SinglePixelBufferState::new::(&dh); - let viewporter_state = ViewporterState::new::(&dh); - let presentation_state = PresentationState::new::(&dh, 1); - let xdg_activation_state = XdgActivationState::new::(&dh); - let primary_selection_state = PrimarySelectionState::new::(&dh); - let popups = PopupManager::default(); + if recording_sink.is_some() && settings.output_mode == 0 { + eprintln!( + "[recording_sink] WARNING: recording_socket is set but output_mode is JPEG (0). \ + The recording socket requires a single H.264 stream. Please set output_mode=1 \ + on the Python CaptureSettings to produce a recordable output." + ); + } - let mut seat = seat_state.new_wl_seat(&dh, "seat0"); - seat.add_keyboard(XkbConfig::default(), 200, 25) - .expect("Failed to init keyboard"); - seat.add_pointer(); + // Every display's capture composites its own watermark, uploaded at this output's + // scale and placed against this output's frame dimensions. + let watermark_output_scale = node.output.current_scale().fractional_scale(); + node.overlay_state + .load_watermark(&settings.watermark_path, watermark_output_scale); + if display_id == 0 { + state.settings = settings.clone(); + if state.cursor_callback_set { + if let Some(icon) = state.current_cursor_icon.clone() { + state.send_cursor_image(&icon); + } + } + } + state.render_cursor_on_framebuffer = settings.capture_cursor; - let mut state = AppState { - compositor_state, - fractional_scale_state, - viewporter_state, - presentation_state, - shm_state, - single_pixel_buffer, - dmabuf_state, - dmabuf_global, - output_state, - seat_state, - shell_state, - layer_shell_state, - space, - data_device_state, - data_control_state, - dh: dh.clone(), - seat, - virtual_keyboard_state, - pointer_warp_state, - relative_pointer_state, - pointer_constraints_state, - outputs: Vec::new(), - pending_windows: Vec::new(), - foreign_toplevel_list, - xdg_decoration_state, - xdg_activation_state, - primary_selection_state, - popups, - frame_buffer: vec![0u8; (width.max(0) as usize) * (height.max(0) as usize) * 4], - gles_renderer, - pixman_renderer, - gbm_device: gbm_device_raw, - offscreen_buffer, - is_capturing: false, - settings: RustCaptureSettings { - width, - height, - ..RustCaptureSettings::default() - }, - callback: None, - cursor_callback: None, - clipboard_callback: None, - pending_clipboard_read: None, - last_log_time: Instant::now(), - start_time: Instant::now(), - clock: Clock::new(), - frame_counter: 0, - pending_force_idr: false, - needs_full_render: true, - use_gpu, - video_encoder: None, + let mut cap = wayland::frontend::WlCapture { + settings: settings.clone(), + video_encoder, vaapi_state: StripeState::default(), - cursor_helper: Cursor::load(cursor_size), - overlay_state: OverlayState::default(), - current_cursor_icon: None, - cursor_buffer: None, - cursor_cache: std::collections::HashMap::new(), - render_cursor_on_framebuffer: false, - render_node_path, - auto_gpu_selected, - recording_sink: None, + recording_sink, deliver_tx: None, deliver_join: None, pending_hw_delivery: None, @@ -1661,420 +1804,1352 @@ fn run_wayland_thread( render_seq: 0, pool_content_gen: Vec::new(), content_gen: 0, - pending_screenshot: None, + frame_counter: 0, + pending_force_idr: false, + needs_full_render: true, + last_tick: None, }; - let output = Output::new( - "HEADLESS-1".into(), - PhysicalProperties { - size: (width, height).into(), - subpixel: Subpixel::Unknown, - make: "Pixelflux".into(), - model: "Virtual".into(), - serial_number: "001".into(), - }, - ); - output.change_current_state( - Some(OutputMode { - size: (width, height).into(), - refresh: 60_000, - }), - Some(Transform::Normal), - Some(OutputScale::Fractional(1.0)), - Some((0, 0).into()), - ); - output.set_preferred(OutputMode { - size: (width, height).into(), - refresh: 60_000, - }); - state.space.map_output(&output, (0, 0)); - state.outputs.push(output.clone()); - let _global = output.create_global::(&dh); - let mut damage_tracker = OutputDamageTracker::from_output(&output); - - event_loop - .handle() - .insert_source(command_rx, move |event, _, state| { - match event { - CalloopEvent::Msg(ThreadCommand::StartCapture(cb, mut settings)) => { - if state.auto_gpu_selected && settings.encode_node_index < -1 { - if let Some(idx_str) = state.render_node_path.strip_prefix("/dev/dri/renderD") { - if let Ok(idx) = idx_str.parse::() { - settings.encode_node_index = idx - 128; + { + let (tx, rx) = std::sync::mpsc::sync_channel::>(1); + // With no Python callback (internal recorder-owned capture) the delivery thread + // only drains the channel: the recorder already consumed the frames at the + // delivery-layer tap, upstream of this per-consumer handoff. + let join = match cb { + Some(cb) => thread::spawn(move || { + crate::boost_thread_priority(-10); + while let Ok(stripes) = rx.recv() { + if PY_SHUTDOWN.load(Ordering::Relaxed) { continue; } + Python::attach(|py| { + for s in stripes { + match Py::new(py, StripeFrame::new_owned_meta( + s.data, s.data_type, s.stripe_y_start, + s.stripe_height, s.frame_id, + )) { + Ok(f) => { if let Err(e) = cb.call1(py, (f,)) { e.print(py); } } + Err(e) => eprintln!("[wayland] frame alloc error: {e:?}"), } } - } + }); + } + }), + None => thread::spawn(move || while rx.recv().is_ok() {}), + }; + cap.deliver_tx = Some(tx); + cap.deliver_join = Some(join); + } - if settings.output_mode == 1 { - settings.width &= !1; - settings.height &= !1; - } + if cap.video_encoder.is_none() { + if let Some(deliver_tx) = cap.deliver_tx.clone() { + let pool = Arc::new(WlFramePool::new( + WL_POOL_SURFACES, + (settings.width.max(0) as usize) * (settings.height.max(0) as usize) * 4, + )); + cap.pool_last_render = vec![0; WL_POOL_SURFACES]; + cap.render_seq = 0; + // u64::MAX marks every slot stale so each one is read back before its + // first publish, whatever the damage says. + cap.pool_content_gen = vec![u64::MAX; WL_POOL_SURFACES]; + cap.content_gen = 0; + let c = &cap.encode_controls; + c.bitrate_kbps.store(settings.video_bitrate_kbps, Ordering::Relaxed); + c.vbv_mult_milli.store( + (settings.video_vbv_multiplier * 1000.0).round() as i32, + Ordering::Relaxed, + ); + c.fps_milli.store( + (settings.target_fps.max(1.0) * 1000.0) as u64, + Ordering::Relaxed, + ); + let cfg = WlEncodeConfig { + settings: settings.clone(), + display_id, + use_gpu: state.use_gpu, + try_gpu: gpu_intent && (!state.use_gpu || different_gpu), + prior: prior_readback_encoder.take(), + recording_sink: cap.recording_sink.clone(), + deliver_tx, + controls: cap.encode_controls.clone(), + stats: cap.encode_stats.clone(), + }; + let pool2 = pool.clone(); + cap.encode_join = Some( + thread::Builder::new() + .name(format!("wl-encode-{display_id}")) + .spawn(move || wayland_encode_loop(&pool2, cfg)) + .expect("failed to spawn wl-encode thread"), + ); + cap.encode_pool = Some(pool); + } + } else { + cap.encode_stats.n_stripes.store(1, Ordering::Relaxed); + *cap.encode_stats.desc.lock().unwrap() = + encoder_desc(&settings, cap.video_encoder.as_ref(), false, true); + log_stream_settings(&settings, 1, cap.video_encoder.as_ref(), false); + } + drop(prior_readback_encoder); + // Force the keyframe on the now-active path (as RequestIdr does), unconditionally: + // the damage tracker and offscreen buffer stay warm across stop/start, so a + // restarted capture on a static screen otherwise produces no damage, no first + // frame, and no IDR in either path. + if cap.encode_pool.is_some() { + cap.encode_controls.force_idr.store(true, Ordering::Relaxed); + } else { + cap.pending_force_idr = true; + } + cap.needs_full_render = true; - state.recording_sink = - crate::recording_sink::RecordingSink::try_bind(&settings.recording_socket); - - if let Some(output) = state.outputs.first() { - // Never panic the compositor thread: an output momentarily - // without a current mode falls back to the requested geometry so - // the reconfigure below is a no-op for size/refresh instead of - // unwrap-panicking, which would drop every Wayland client with no - // recovery short of a process restart. - let target_refresh = (settings.target_fps * 1000.0).round() as i32; - let (current_w, current_h, current_refresh) = match output.current_mode() { - Some(m) => (m.size.w, m.size.h, m.refresh), - None => (settings.width, settings.height, target_refresh), - }; - let current_scale = output.current_scale().fractional_scale(); + node.capture = Some(cap); + state.output_nodes.insert(node_idx, node); + wayland_alive().lock().unwrap().insert(display_id); +} - let scale = settings.scale.max(0.1); - let logical_width = (settings.width as f64 / scale).round() as i32; - let logical_height = (settings.height as f64 / scale).round() as i32; +/// One output's render + capture tick: composite the elements overlapping this output +/// (positions made output-local by subtracting its layout origin), track damage, feed the +/// display's own encode path, and answer a pending screenshot on the primary. Returns true +/// when the tick was skipped because this display's encode pool was still busy (the caller +/// then retries shortly instead of waiting a full frame interval). +fn render_node_tick( + state: &mut AppState, + node: &mut wayland::frontend::OutputNode, + is_memory_throttling: bool, +) -> bool { + let take_screenshot = state + .pending_screenshot + .as_ref() + .is_some_and(|(id, _)| *id == node.id); + if node.capture.is_none() && !take_screenshot { + return false; + } - if current_w != settings.width - || current_h != settings.height - || (current_scale - settings.scale).abs() > 0.001 - || current_refresh != target_refresh - { - // Allocate the GPU backing for the new dimensions BEFORE - // committing anything: if the driver refuses (VRAM - // exhaustion, dimensions it will not back), the whole - // reconfigure is skipped and the previous mode + buffers - // stay live. A failed resize must degrade to "no resize", - // never panic the compositor thread — that would drop - // every Wayland client with no recovery short of a - // process restart. (The INITIAL allocation path already - // degrades gracefully, to the software renderer.) - let mut new_offscreen = None; - let mut gbm_resize_failed = false; - if state.use_gpu { - if let Some(gbm) = state.gbm_device.as_mut() { - match gbm.create_buffer_object( - settings.width as u32, - settings.height as u32, - GbmFormat::Argb8888, - BufferObjectFlags::RENDERING, - ) { - Ok(bo) => { - let dmabuf = create_dmabuf_from_bo(&bo); - new_offscreen = Some((bo, dmabuf)); - } - Err(e) => { - eprintln!( - "[Wayland] GBM buffer resize to {}x{} failed ({:?}); keeping previous output mode.", - settings.width, settings.height, e - ); - gbm_resize_failed = true; + // Per-display frame pacing under the one shared timer (which fires at the fastest + // active capture's rate). + if let Some(cap) = node.capture.as_ref() { + if !take_screenshot { + let fps = (if is_memory_throttling { 5.0 } else { cap.settings.target_fps }).max(1.0); + if let Some(last) = cap.last_tick { + if last.elapsed().as_secs_f64() < (1.0 / fps) * 0.9 { + return false; + } + } + } + } + + let output = node.output.clone(); + let origin: Point = node.pos.into(); + let output_scale_val = output.current_scale().fractional_scale(); + let (width, height) = match node.capture.as_ref() { + Some(c) => (c.settings.width, c.settings.height), + None => output + .current_mode() + .map(|m| (m.size.w, m.size.h)) + .unwrap_or((0, 0)), + }; + if width <= 0 || height <= 0 { + return false; + } + if node.frame_buffer.len() < (width as usize) * (height as usize) * 4 { + node.frame_buffer = vec![0u8; (width as usize) * (height as usize) * 4]; + } + let logical_w = (width as f64 / output_scale_val).round(); + let logical_h = (height as f64 / output_scale_val).round(); + + // A recorder connecting counts as an IDR request, kept armed across skipped ticks. + if let Some(cap) = node.capture.as_mut() { + if cap + .recording_sink + .as_ref() + .map(|s| s.should_force_idr()) + .unwrap_or(false) + { + cap.pending_force_idr = true; + cap.encode_controls.force_idr.store(true, Ordering::Relaxed); + } + } + let requested_idr = node.capture.as_ref().map(|c| c.pending_force_idr).unwrap_or(false); + + let mut pool_slot: Option<(usize, Vec)> = None; + if let Some(cap) = node.capture.as_ref() { + if let Some(ref pool) = cap.encode_pool { + if !is_memory_throttling { + pool_slot = pool.try_begin(); + if pool_slot.is_none() { + return true; + } + } + } + } + + let loc_enum = node + .capture + .as_ref() + .map(|c| c.settings.watermark_location_enum) + .unwrap_or(state.settings.watermark_location_enum); + node.overlay_state.update_position(width, height, loc_enum); + + if let Some(cap) = node.capture.as_mut() { + cap.last_tick = Some(Instant::now()); + } + + // The cursor is composited only on the output the pointer is on, at that output's + // scale; its position is output-local. + let pointer_local: Option> = state + .seat + .get_pointer() + .map(|p| p.current_location()) + .and_then(|pos| { + let rect = Rectangle::::new( + origin.to_f64(), + (logical_w, logical_h).into(), + ); + if rect.contains(pos) { + Some(pos - origin.to_f64()) + } else { + None + } + }); + + let mut render_success = false; + let mut render_sync = None; + let mut damage_rects: Vec> = Vec::new(); + let needs_full = node.capture.as_ref().map(|c| c.needs_full_render).unwrap_or(true); + + if state.use_gpu { + if let Some(renderer) = state.gles_renderer.as_mut() { + let mut cap = node.capture.as_mut(); + if let Some((_bo, dmabuf)) = node.offscreen_buffer.as_mut() { + let render_age = if node.overlay_state.is_animated() || needs_full { 0 } else { 1 }; + match renderer.bind(dmabuf) { + Ok(mut frame) => { + let mut elements: Vec>> = Vec::new(); + + if state.render_cursor_on_framebuffer { + if let Some(pos) = pointer_local { + let scale = Scale::from(output_scale_val); + + if let Some(CursorImageStatus::Named(icon)) = &state.current_cursor_icon { + let name = wayland::frontend::cursor_icon_to_str(icon); + let time = Duration::from_millis(state.clock.now().as_millis() as u64); + if let Some(image) = state.cursor_helper.get_image_by_name(name, output_scale_val.round() as u32, time) { + if let Some(elem) = node.overlay_state.get_cursor_element(renderer, image, pos, output_scale_val) { + elements.push(CompositionElements::Cursor(elem)); } } + } else if let Some(CursorImageStatus::Surface(surface)) = &state.current_cursor_icon { + let phys_pos = pos.to_physical(scale); + let elem_result = with_states(surface, |states| { + WaylandSurfaceRenderElement::from_surface(renderer, surface, states, phys_pos, 1.0, smithay::backend::renderer::element::Kind::Cursor) + }); + if let Ok(Some(cursor_elem)) = elem_result { + elements.push(CompositionElements::Surface(cursor_elem)); + } + } else if state.current_cursor_icon.is_none() { + let time = Duration::from_millis(state.clock.now().as_millis() as u64); + let image = state.cursor_helper.get_image(output_scale_val.round() as u32, time); + if let Some(elem) = node.overlay_state.get_cursor_element(renderer, image, pos, output_scale_val) { + elements.push(CompositionElements::Cursor(elem)); + } } } - if gbm_resize_failed { - // The mode commit below is skipped wholesale, so the - // rest of this StartCapture (encoder setup, stored - // settings) must also see the dimensions that are - // actually live — otherwise the encoder and the - // still-old buffers disagree on frame geometry. - settings.width = current_w; - settings.height = current_h; - settings.scale = current_scale; - settings.target_fps = current_refresh as f64 / 1000.0; - } else { - println!( - "[Wayland] Configuring Output: {}x{} @ {:.2} FPS (Scale {:.2})", - settings.width, settings.height, settings.target_fps, settings.scale - ); - let new_mode = OutputMode { - size: (settings.width, settings.height).into(), - refresh: target_refresh, - }; - output.change_current_state( - Some(new_mode), - Some(Transform::Normal), - Some(OutputScale::Fractional(settings.scale)), - Some((0, 0).into()), - ); - output.set_preferred(new_mode); + } - let pixel_count = - (settings.width.max(0) as usize) * (settings.height.max(0) as usize); - state.frame_buffer = vec![0u8; pixel_count * 4]; + if let Some(elem) = node.overlay_state.get_watermark_element(renderer) { + elements.push(CompositionElements::Cursor(elem)); + } - if let Some(off) = new_offscreen.take() { - state.offscreen_buffer = Some(off); + { + let layer_map = layer_map_for_output(&output); + + let draw_layer = |renderer: &mut GlesRenderer, elements: &mut Vec>>, target_layer: smithay::wayland::shell::wlr_layer::Layer| { + for surface in layer_map.layers().rev() { + let current_layer = surface.layer(); + if current_layer == target_layer { + if let Some(geo) = layer_map.layer_geometry(surface) { + let elem = smithay::wayland::compositor::with_states(surface.wl_surface(), |states| { + WaylandSurfaceRenderElement::from_surface( + renderer, surface.wl_surface(), states, + geo.loc.to_physical_precise_round(output_scale_val), 1.0, + smithay::backend::renderer::element::Kind::Unspecified + ) + }); + if let Ok(Some(e)) = elem { + elements.push(CompositionElements::Surface(e)); + } + } + } } - } + }; + + draw_layer(renderer, &mut elements, smithay::wayland::shell::wlr_layer::Layer::Overlay); + draw_layer(renderer, &mut elements, smithay::wayland::shell::wlr_layer::Layer::Top); } - for window in state.space.elements() { + for window in state.space.elements_for_output(&output).collect::>().into_iter().rev() { + let window_loc = state.space.element_location(window).unwrap_or_default() - origin; + if let Some(surface) = window.wl_surface() { - output.enter(&surface); - with_states(&surface, |states| { - smithay::wayland::fractional_scale::with_fractional_scale(states, |fs| { - fs.set_preferred_scale(scale); + let popups = PopupManager::popups_for_surface(&surface); + for (popup, location) in popups { + let popup_surface = popup.wl_surface(); + let popup_pos = window_loc + location; + let elem = smithay::wayland::compositor::with_states(popup_surface, |states| { + WaylandSurfaceRenderElement::from_surface( + renderer, + popup_surface, + states, + popup_pos.to_physical_precise_round(output_scale_val), + 1.0, + smithay::backend::renderer::element::Kind::Unspecified + ) }); - }); - } - if let Some(toplevel) = window.toplevel() { - toplevel.with_pending_state(|state| { - use smithay::reexports::wayland_protocols::xdg::shell::server::xdg_toplevel::State; - state.states.set(State::Fullscreen); - state.states.set(State::Activated); - state.size = Some((logical_width, logical_height).into()); - }); - toplevel.send_configure(); + if let Ok(Some(e)) = elem { + elements.push(CompositionElements::Surface(e)); + } + } } - } - } - - let use_cpu_explicit = settings.use_cpu || settings.encode_node_index == -1; - let gpu_intent = - settings.output_mode == 1 && !settings.use_openh264 && !use_cpu_explicit; - if use_cpu_explicit && !(settings.output_mode == 1 && settings.use_openh264) { - println!("[Wayland] CPU encoding selected (use_cpu=true or encode_node_index=-1)."); - } - let mut different_gpu = false; - if gpu_intent { - let encode_node_idx = settings.encode_node_index.max(0); - if !state.render_node_path.is_empty() - && !state.render_node_path.contains(&format!("renderD{}", 128 + encode_node_idx)) - { - different_gpu = true; + elements.extend(window.render_elements(renderer, window_loc.to_physical_precise_round(output_scale_val), Scale::from(output_scale_val), 1.0).into_iter().map(CompositionElements::Space)); } - } - if gpu_intent && state.use_gpu && !different_gpu { - let encode_driver = get_gpu_driver(settings.encode_node_index.max(0)); - println!( - "[Wayland] Encode Node Index: {} | Driver: {}", - settings.encode_node_index.max(0), encode_driver - ); + { + let layer_map = layer_map_for_output(&output); + + let draw_layer = |renderer: &mut GlesRenderer, elements: &mut Vec>>, target_layer: smithay::wayland::shell::wlr_layer::Layer| { + for surface in layer_map.layers() { + let current_layer = surface.layer(); + if current_layer == target_layer { + if let Some(geo) = layer_map.layer_geometry(surface) { + let elem = smithay::wayland::compositor::with_states(surface.wl_surface(), |states| { + WaylandSurfaceRenderElement::from_surface( + renderer, surface.wl_surface(), states, + geo.loc.to_physical_precise_round(output_scale_val), 1.0, + smithay::backend::renderer::element::Kind::Unspecified + ) + }); + if let Ok(Some(e)) = elem { + elements.push(CompositionElements::Surface(e)); + } + } + } + } + }; - if encode_driver.contains("nvidia") { - let reused = match state.video_encoder.as_mut() { - Some(GpuEncoder::Nvenc(enc)) => { - match enc.reconfigure_resolution(&settings) { - Ok(()) => { - println!("[Wayland] NVENC session reconfigured in place."); - true + draw_layer(renderer, &mut elements, smithay::wayland::shell::wlr_layer::Layer::Bottom); + draw_layer(renderer, &mut elements, smithay::wayland::shell::wlr_layer::Layer::Background); + } + match node.damage_tracker.render_output(renderer, &mut frame, render_age, &elements, [0.1, 0.1, 0.1, 1.0]) { + Ok(result) => { + render_success = true; + if let Some(damage) = result.damage { + damage_rects = damage.clone(); + } + render_sync = Some(result.sync); + if let Some(c) = cap.as_deref_mut() { + c.needs_full_render = false; + } + }, + Err(e) => eprintln!("Render error: {:?}", e) + } + if let Some(c) = cap.as_deref_mut() { + if !damage_rects.is_empty() { + c.content_gen += 1; + } + if let Some((id, ref mut buf)) = pool_slot { + // No-damage ticks skip the readback, so a pooled buffer can + // lag the offscreen target whenever the encoder held the + // other slot across a tick; one catch-up readback keeps + // every published buffer current. + if render_success && c.pool_content_gen[id] != c.content_gen { + let _ = renderer.with_context(|gl| unsafe { + gl.ReadPixels( + 0, + 0, + width, + height, + smithay::backend::renderer::gles::ffi::RGBA, + smithay::backend::renderer::gles::ffi::UNSIGNED_BYTE, + buf.as_mut_ptr() as *mut std::ffi::c_void, + ); + }); + c.pool_content_gen[id] = c.content_gen; + } + } + } + if pool_slot.is_none() && take_screenshot { + let _ = renderer.with_context(|gl| unsafe { + gl.ReadPixels( + 0, + 0, + width, + height, + smithay::backend::renderer::gles::ffi::RGBA, + smithay::backend::renderer::gles::ffi::UNSIGNED_BYTE, + node.frame_buffer.as_mut_ptr() as *mut std::ffi::c_void, + ); + }); + } + }, + Err(e) => eprintln!("Failed to bind buffer: {:?}", e) + } + } + } + } else { + if let Some(renderer) = state.pixman_renderer.as_mut() { + let mut cap = node.capture.as_mut(); + let (ptr, buf_age) = match pool_slot { + Some((id, ref mut buf)) => { + let age = cap + .as_ref() + .map(|c| { + if c.pool_last_render[id] == 0 { + 0 + } else { + (c.render_seq + 1 - c.pool_last_render[id]) as usize + } + }) + .unwrap_or(0); + (buf.as_mut_ptr() as *mut u32, age) + } + None => (node.frame_buffer.as_mut_ptr() as *mut u32, 0), + }; + let mut image = unsafe { + pixman::Image::from_raw_mut(pixman::FormatCode::A8R8G8B8, width as usize, height as usize, ptr, (width as usize) * 4, false).expect("Failed to create pixman image") + }; + match renderer.bind(&mut image) { + Ok(mut frame) => { + let mut elements: Vec>> = Vec::new(); + + if state.render_cursor_on_framebuffer { + if let Some(pos) = pointer_local { + let scale = Scale::from(output_scale_val); + + if let Some(CursorImageStatus::Named(icon)) = &state.current_cursor_icon { + let name = wayland::frontend::cursor_icon_to_str(icon); + let time = Duration::from_millis(state.clock.now().as_millis() as u64); + if let Some(image) = state.cursor_helper.get_image_by_name(name, output_scale_val.round() as u32, time) { + if let Some(elem) = node.overlay_state.get_cursor_element(renderer, image, pos, output_scale_val) { + elements.push(CompositionElements::Cursor(elem)); + } } - Err(e) => { - eprintln!("[Wayland] NVENC in-place reconfigure unavailable ({e}); rebuilding."); - false + } else if let Some(CursorImageStatus::Surface(surface)) = &state.current_cursor_icon { + let phys_pos = pos.to_physical(scale); + let elem_result = with_states(surface, |states| { + WaylandSurfaceRenderElement::from_surface(renderer, surface, states, phys_pos, 1.0, smithay::backend::renderer::element::Kind::Cursor) + }); + if let Ok(Some(cursor_elem)) = elem_result { + elements.push(CompositionElements::Surface(cursor_elem)); + } + } else if state.current_cursor_icon.is_none() { + let time = Duration::from_millis(state.clock.now().as_millis() as u64); + let image = state.cursor_helper.get_image(output_scale_val.round() as u32, time); + if let Some(elem) = node.overlay_state.get_cursor_element(renderer, image, pos, output_scale_val) { + elements.push(CompositionElements::Cursor(elem)); } } } - _ => false, - }; - if !reused { - state.video_encoder = None; - println!("[Wayland] Nvidia Encoder detected. Initializing NVENC..."); - let egl_display = if let Some(renderer) = state.gles_renderer.as_ref() { - renderer.egl_context().display().get_display_handle().handle - } else { - std::ptr::null() + } + + if let Some(elem) = node.overlay_state.get_watermark_element(renderer) { + elements.push(CompositionElements::Cursor(elem)); + } + + { + let layer_map = layer_map_for_output(&output); + + let draw_layer = |renderer: &mut PixmanRenderer, elements: &mut Vec>>, target_layer: smithay::wayland::shell::wlr_layer::Layer| { + for surface in layer_map.layers() { + let current_layer = surface.layer(); + if current_layer == target_layer { + if let Some(geo) = layer_map.layer_geometry(surface) { + let elem = smithay::wayland::compositor::with_states(surface.wl_surface(), |states| { + WaylandSurfaceRenderElement::from_surface( + renderer, surface.wl_surface(), states, + geo.loc.to_physical_precise_round(output_scale_val), 1.0, + smithay::backend::renderer::element::Kind::Unspecified + ) + }); + if let Ok(Some(e)) = elem { + elements.push(CompositionElements::Surface(e)); + } + } + } + } }; - match NvencEncoder::new(&settings, egl_display) { - Ok(encoder) => { - state.video_encoder = Some(GpuEncoder::Nvenc(encoder)); - println!("[Wayland] NVENC Encoder initialized successfully."); + draw_layer(renderer, &mut elements, smithay::wayland::shell::wlr_layer::Layer::Overlay); + draw_layer(renderer, &mut elements, smithay::wayland::shell::wlr_layer::Layer::Top); + } + + for window in state.space.elements_for_output(&output).collect::>().into_iter().rev() { + let loc = state.space.element_location(window).unwrap_or_default() - origin; + + if let Some(surface) = window.wl_surface() { + let popups = PopupManager::popups_for_surface(&surface); + for (popup, location) in popups { + let popup_surface = popup.wl_surface(); { + let popup_pos = loc + location; + let elem = smithay::wayland::compositor::with_states(popup_surface, |states| { + WaylandSurfaceRenderElement::from_surface( + renderer, + popup_surface, + states, + popup_pos.to_physical_precise_round(output_scale_val), + 1.0, + smithay::backend::renderer::element::Kind::Unspecified + ) + }); + if let Ok(Some(e)) = elem { + elements.push(CompositionElements::Surface(e)); + } + } } - Err(e) => eprintln!( - "[Wayland] Failed to init NVENC: {}. Falling back to CPU.", - e - ), } + + elements.extend(window.render_elements(renderer, loc.to_physical_precise_round(output_scale_val), Scale::from(output_scale_val), 1.0).into_iter().map(CompositionElements::Space)); + } + + { + let layer_map = layer_map_for_output(&output); + + let draw_layer = |renderer: &mut PixmanRenderer, elements: &mut Vec>>, target_layer: smithay::wayland::shell::wlr_layer::Layer| { + for surface in layer_map.layers() { + let current_layer = surface.layer(); + if current_layer == target_layer { + if let Some(geo) = layer_map.layer_geometry(surface) { + let elem = smithay::wayland::compositor::with_states(surface.wl_surface(), |states| { + WaylandSurfaceRenderElement::from_surface( + renderer, surface.wl_surface(), states, + geo.loc.to_physical_precise_round(output_scale_val), 1.0, + smithay::backend::renderer::element::Kind::Unspecified + ) + }); + if let Ok(Some(e)) = elem { + elements.push(CompositionElements::Surface(e)); + } + } + } + } + }; + + draw_layer(renderer, &mut elements, smithay::wayland::shell::wlr_layer::Layer::Bottom); + draw_layer(renderer, &mut elements, smithay::wayland::shell::wlr_layer::Layer::Background); + } + + let render_age = if node.overlay_state.is_animated() || needs_full { 0 } else { buf_age }; + match node.damage_tracker.render_output(renderer, &mut frame, render_age, &elements, [0.1, 0.1, 0.1, 1.0]) { + Ok(result) => { + render_success = true; + if let Some(c) = cap.as_deref_mut() { + c.needs_full_render = false; } - } else { - state.video_encoder = None; - println!("[Wayland] Initializing Unified VAAPI Encoder..."); - if settings.video_fullcolor { - println!("[Wayland] 4:4:4 Fullcolor requested. VAAPI does not support this profile reliably. Falling back to CPU."); + if let Some(damage) = result.damage { damage_rects = damage.clone(); } + }, + Err(e) => eprintln!("Render error: {:?}", e) + } + if let Some(c) = cap.as_deref_mut() { + c.render_seq += 1; + if render_success { + if let Some((id, _)) = pool_slot { + c.pool_last_render[id] = c.render_seq; + } + } + } + }, + Err(e) => eprintln!("Failed to bind pixman image: {:?}", e) + } + } + } + + if render_success { + let time = state.clock.now(); + for window in state.space.elements_for_output(&output).cloned().collect::>() { + window.send_frame(&output, time, Some(Duration::ZERO), |_, _| Some(output.clone())); + } + + if let Some(cap) = node.capture.as_mut() { + if is_memory_throttling { + if cap.encode_pool.is_none() { + cap.frame_counter = cap.frame_counter.wrapping_add(1); + } + } else if cap.encode_pool.is_some() { + if take_screenshot { + if let Some((_, ref buf)) = pool_slot { + let n = buf.len().min(node.frame_buffer.len()); + node.frame_buffer[..n].copy_from_slice(&buf[..n]); + } + } + if let Some((id, buf)) = pool_slot.take() { + let is_animated = node.overlay_state.is_animated(); + cap.encode_pool.as_ref().unwrap().publish(WlFrame { + id, + buf, + frame_id: cap.frame_counter, + damage: std::mem::take(&mut damage_rects), + is_animated, + }); + cap.frame_counter = cap.frame_counter.wrapping_add(1); + } + } else if let Some(ref mut encoder) = cap.video_encoder { + // Deliver the parked frame (if any) first, WITHOUT blocking: this runs + // on the calloop thread, and a blocking send would freeze + // input/command/Wayland dispatch for as long as the Python consumer + // stalls. While a frame stays parked, no new frame is encoded — an + // encoded frame joins the H.264 reference chain and can never be + // dropped — and the tick's damage is latched so the pause never loses + // a change. + let slot_free = match cap.pending_hw_delivery.take() { + None => true, + Some(pending) => match cap.deliver_tx.as_ref() { + None => true, + Some(tx) => match tx.try_send(pending) { + Ok(()) => true, + Err(std::sync::mpsc::TrySendError::Full(p)) => { + cap.pending_hw_delivery = Some(p); + false + } + Err(std::sync::mpsc::TrySendError::Disconnected(_)) => true, + }, + }, + }; + if !slot_free { + if !damage_rects.is_empty() { + cap.pending_hw_damage = true; + } + } else { + let is_animated = node.overlay_state.is_animated(); + let had_damage = !damage_rects.is_empty() + || std::mem::take(&mut cap.pending_hw_damage); + let decision = crate::pipeline::decide_hw_fullframe( + &mut cap.vaapi_state, + &cap.settings, + cap.frame_counter, + had_damage, + is_animated, + requested_idr, + ); + let send_frame = decision.send; + let force_idr = decision.force_idr; + let target_qp = decision.target_qp; + + let mut frame_out = false; + if send_frame { + if let Some(sync) = render_sync.take() { + let _ = sync.wait(); + } + let result = match encoder { + GpuEncoder::Nvenc(enc) => { + if let Some((_, ref dmabuf)) = node.offscreen_buffer { + enc.encode(dmabuf, cap.frame_counter as u64, target_qp, force_idr) } else { - match VaapiEncoder::new(&settings) { - Ok(encoder) => { - state.video_encoder = Some(GpuEncoder::Vaapi(encoder)); - println!( - "[Wayland] VAAPI Encoder initialized successfully." - ); + Err("NVENC ZeroCopy requires offscreen buffer (GPU context)".to_string()) + } + }, + GpuEncoder::Vaapi(enc) => { + if let Some((_, ref dmabuf)) = node.offscreen_buffer { + enc.encode_dmabuf(dmabuf, cap.frame_counter as u64, target_qp, force_idr) + } else { + Err("Vaapi ZeroCopy requires offscreen buffer (GPU context)".to_string()) + } + } + }; + + if let Ok(data) = result { + if !data.is_empty() { + frame_out = true; + cap.encode_stats.frames.fetch_add(1, Ordering::Relaxed); + cap.encode_stats.stripes.fetch_add(1, Ordering::Relaxed); + if let Some(ref tx) = cap.deliver_tx { + let stripes = vec![EncodedStripe { + data: Arc::new(data), data_type: 2, stripe_y_start: 0, + stripe_height: height, frame_id: cap.frame_counter as i32, + }]; + if let Some(ref socket) = cap.recording_sink { + for stripe in &stripes { + socket.write_encoded_frame(stripe); } - Err(e) => eprintln!( - "[Wayland] Failed to init VAAPI: {}. Falling back to CPU.", - e - ), + } + crate::recorder::wayland_tap(node.id, &stripes); + // Non-blocking: a full slot parks the frame (delivered + // ahead of any new encode above). + match tx.try_send(stripes) { + Ok(()) => {} + Err(std::sync::mpsc::TrySendError::Full(s)) => { + cap.pending_hw_delivery = Some(s); + } + Err(std::sync::mpsc::TrySendError::Disconnected(_)) => {} } } } + } else if let Err(e) = result { + eprintln!("HW Encode Error: {}", e); + } + } + // An unserved request stays armed: on an infinite GOP an IDR lost to an + // encode error would never self-heal. + cap.pending_force_idr = requested_idr && !frame_out; + cap.frame_counter = cap.frame_counter.wrapping_add(1); + } + } + } + if take_screenshot { + if let Some((_, resp)) = state.pending_screenshot.take() { + if !node.frame_buffer.is_empty() { + let w = width as u32; + let h = height as u32; + let png = if state.use_gpu { + crate::computer_use::encode_png_rgba(&node.frame_buffer, w, h) } else { - state.video_encoder = None; + let mut rgba = node.frame_buffer.clone(); + for px in rgba.chunks_exact_mut(4) { + px.swap(0, 2); + } + crate::computer_use::encode_png_rgba(&rgba, w, h) + }; + match png { + Ok(data) => { let _ = resp.send(Ok(data)); } + Err(e) => { + let _ = resp.send(Err(format!("PNG encode error: {e}"))); + eprintln!("[ComputerUse] PNG encode error: {}", e); + } } + } else { + let _ = resp.send(Err("Screenshot render produced no pixels".to_string())); + } + } + } + } + if let Some((id, buf)) = pool_slot.take() { + if let Some(cap) = node.capture.as_ref() { + if let Some(ref pool) = cap.encode_pool { + pool.cancel(id, buf); + } + } + } + false +} - if different_gpu { - println!("[Wayland] Decision: Rendering and Encoding GPUs differ -> Forcing Readback (CPU path for pixels)."); - } - if state.video_encoder.is_none() { - println!("[Wayland] Decision: Readback path (encode thread) active."); - } else if !different_gpu { - println!("[Wayland] Decision: Zero-Copy path active."); - } +/// True when the rectangles `(x, y, w, h)` overlap: strict interior intersection, so +/// touching edges do not count and empty (non-positive-dimension) rectangles never +/// overlap anything. Arithmetic is widened so extreme coordinates cannot wrap. +fn rects_overlap(a: (i32, i32, i32, i32), b: (i32, i32, i32, i32)) -> bool { + let (ax, ay, aw, ah) = (a.0 as i64, a.1 as i64, a.2 as i64, a.3 as i64); + let (bx, by, bw, bh) = (b.0 as i64, b.1 as i64, b.2 as i64, b.3 as i64); + aw > 0 && ah > 0 && bw > 0 && bh > 0 + && ax < bx + bw && bx < ax + aw + && ay < by + bh && by < ay + ah +} + +/// The first live output (excluding `skip_id`) whose rectangle overlaps a candidate +/// placement, as `(id, flavor, rect)`. Both rectangle flavors are checked — logical +/// (Space layout, scale-divided) and physical (mode pixels at the same origin) — because +/// input injection and cursor compositing key off the physical rects while window layout +/// keys off the logical ones, and neither may overlap. +fn find_output_overlap( + nodes: &[wayland::frontend::OutputNode], + skip_id: Option, + logical: (i32, i32, i32, i32), + physical: (i32, i32, i32, i32), +) -> Option<(u32, &'static str, (i32, i32, i32, i32))> { + for n in nodes { + if Some(n.id) == skip_id { + continue; + } + if let Some(geo) = n.logical_geometry() { + let other = (geo.loc.x, geo.loc.y, geo.size.w, geo.size.h); + if rects_overlap(logical, other) { + return Some((n.id, "logical", other)); + } + } + if let Some(mode) = n.output.current_mode() { + let other = (n.pos.0, n.pos.1, mode.size.w, mode.size.h); + if rects_overlap(physical, other) { + return Some((n.id, "physical", other)); + } + } + } + None +} + +/// Create an additional output mapped into the layout at `(x, y)`. Fails (false) on a +/// duplicate id, non-positive geometry/scale, a rectangle overlapping a live output, or a +/// GPU render-target allocation failure. Only Create/Reposition placements are validated: +/// a capture reconfigure (StartCapture on an existing output) resizes UNVALIDATED, so +/// keeping a multi-step relayout overlap-free at every step is the caller's ordering +/// responsibility. +fn create_output_on( + state: &mut AppState, + id: u32, + width: i32, + height: i32, + x: i32, + y: i32, + scale: f64, +) -> bool { + if state.node_idx_for_id(id).is_some() || width <= 0 || height <= 0 || scale <= 0.0 { + return false; + } + let logical_size = ( + (width as f64 / scale).round() as i32, + (height as f64 / scale).round() as i32, + ); + if let Some((oid, flavor, other)) = find_output_overlap( + &state.output_nodes, + None, + (x, y, logical_size.0, logical_size.1), + (x, y, width, height), + ) { + eprintln!( + "[Wayland] CreateOutput {id}: rejected, {flavor} rect {}x{}+{x}+{y} overlaps output {oid} at {}x{}+{}+{}.", + if flavor == "logical" { logical_size.0 } else { width }, + if flavor == "logical" { logical_size.1 } else { height }, + other.2, other.3, other.0, other.1, + ); + return false; + } + let output = Output::new( + format!("HEADLESS-{}", id + 1), + PhysicalProperties { + size: (width, height).into(), + subpixel: Subpixel::Unknown, + make: "Pixelflux".into(), + model: "Virtual".into(), + serial_number: format!("{:03}", id + 1), + }, + ); + let mode = OutputMode { size: (width, height).into(), refresh: 60_000 }; + output.change_current_state( + Some(mode), + Some(Transform::Normal), + Some(OutputScale::Fractional(scale)), + Some((x, y).into()), + ); + output.set_preferred(mode); + let mut offscreen = None; + if state.use_gpu { + let Some(gbm) = state.gbm_device.as_mut() else { return false }; + match gbm.create_buffer_object( + width as u32, + height as u32, + GbmFormat::Argb8888, + BufferObjectFlags::RENDERING, + ) { + Ok(bo) => { + let dmabuf = create_dmabuf_from_bo(&bo); + offscreen = Some((bo, dmabuf)); + } + Err(e) => { + eprintln!("[Wayland] CreateOutput {id}: GBM allocation {width}x{height} failed ({e:?})."); + return false; + } + } + } + state.space.map_output(&output, (x, y)); + let global = output.create_global::(&state.dh); + let damage_tracker = OutputDamageTracker::from_output(&output); + println!("[Wayland] Output {id} created: {width}x{height} @ ({x}, {y}) scale {scale:.2}."); + state.output_nodes.push(wayland::frontend::OutputNode { + id, + output, + global, + pos: (x, y), + damage_tracker, + frame_buffer: vec![0u8; (width.max(0) as usize) * (height.max(0) as usize) * 4], + offscreen_buffer: offscreen, + overlay_state: OverlayState::default(), + capture: None, + }); + true +} + +/// Move an existing output (the primary included) to layout offset `(x, y)`. The output's +/// advertised position, its Space mapping, and the windows placed on it (mapped at the +/// output's origin — window positions are output-relative under forced fullscreen) all +/// follow, so absolute input injection and cursor compositing — both keyed off `node.pos` — +/// resolve against the new layout immediately. A destination overlapping another live +/// output is refused (false). As with `CreateOutput`, only the placement itself is +/// validated: a capture reconfigure (StartCapture on an existing output) resizes +/// UNVALIDATED, so keeping a multi-step relayout overlap-free at every step is the +/// caller's ordering responsibility. +fn reposition_output_on(state: &mut AppState, id: u32, x: i32, y: i32) -> bool { + let Some(idx) = state.node_idx_for_id(id) else { return false }; + let output = state.output_nodes[idx].output.clone(); + if state.output_nodes[idx].pos == (x, y) { + return true; + } + let logical_size = state.output_nodes[idx] + .logical_geometry() + .map(|g| (g.size.w, g.size.h)) + .unwrap_or((0, 0)); + let physical_size = output.current_mode().map(|m| (m.size.w, m.size.h)).unwrap_or((0, 0)); + if let Some((oid, flavor, other)) = find_output_overlap( + &state.output_nodes, + Some(id), + (x, y, logical_size.0, logical_size.1), + (x, y, physical_size.0, physical_size.1), + ) { + eprintln!( + "[Wayland] RepositionOutput {id}: rejected, {flavor} rect {}x{}+{x}+{y} overlaps output {oid} at {}x{}+{}+{}.", + if flavor == "logical" { logical_size.0 } else { physical_size.0 }, + if flavor == "logical" { logical_size.1 } else { physical_size.1 }, + other.2, other.3, other.0, other.1, + ); + return false; + } + state.output_nodes[idx].pos = (x, y); + output.change_current_state(None, None, None, Some((x, y).into())); + state.space.map_output(&output, (x, y)); + let windows: Vec = state + .space + .elements() + .filter(|w| wayland::frontend::window_output_id(w) == id) + .cloned() + .collect(); + for window in &windows { + state.space.map_element(window.clone(), (x, y), false); + } + if let Some(cap) = state.output_nodes[idx].capture.as_mut() { + cap.needs_full_render = true; + } + println!("[Wayland] Output {id} repositioned to ({x}, {y})."); + true +} + +/// Destroy a secondary output: end its capture, relocate its windows onto the primary +/// output, unmap it from the space, and retract its global. The primary (id 0) is refused. +fn destroy_output_on(state: &mut AppState, id: u32) -> bool { + if id == 0 { + return false; + } + let Some(_) = state.node_idx_for_id(id) else { return false }; + stop_capture_on_display(state, id); + wayland_owners().lock().unwrap().remove(&id); + // Relocate while the node is still registered so output leave/enter both resolve. + let windows: Vec = state + .space + .elements() + .filter(|w| wayland::frontend::window_output_id(w) == id) + .cloned() + .collect(); + for window in &windows { + state.place_window_on_output(window, 0); + } + for w in &state.pending_windows { + if let Some(meta) = wayland::frontend::window_meta(w) { + if meta.output.load(Ordering::Relaxed) == id { + meta.output.store(0, Ordering::Relaxed); + } + } + } + let idx = state.node_idx_for_id(id).unwrap(); + let node = state.output_nodes.remove(idx); + state.space.unmap_output(&node.output); + state.dh.remove_global::(node.global); + println!( + "[Wayland] Output {id} destroyed; {} window(s) relocated to primary.", + windows.len() + ); + true +} + +/// The main execution loop of the Wayland backend. +/// +/// This function is the central nervous system of the backend. It runs on its own thread and owns +/// the entire lifecycle of the headless Wayland compositor: +/// +/// 1. **Initialization**: builds the `calloop` event loop and the Wayland display, raises +/// libwayland's per-client buffer limit when the newer setter is available (resolved at runtime +/// so the module still loads against older libwayland), and brings up the rendering pipeline — +/// GBM/EGL hardware acceleration on the resolved DRM render node, falling back to software +/// rendering (Pixman) when no node is usable. +/// 2. **State management**: constructs and holds the `AppState` — the Wayland globals (compositor, +/// seat, SHM, shell, dmabuf, selections, and the rest) plus the output registry: the primary +/// virtual `HEADLESS-1` output at layout (0, 0), extended at runtime by CreateOutput with +/// additional outputs at their layout offsets, each `OutputNode` owning its damage tracker, +/// render targets, and (at most one) capture pipeline. +/// 3. **Event dispatch**: +/// - **Command channel**: control messages from the Python thread — per-display start/stop, +/// output lifecycle (create/destroy/list/move-window), input injection routed across the +/// output layout, keymap and clipboard operations, live rate / tunable changes, and the +/// computer-use queries. +/// - **Wayland socket**: accepts client connections and drives the compositor protocol. +/// 4. **StartCapture reconfigure** (per display): reprograms that output's mode / scale / refresh, +/// resizes its framebuffer and offscreen GBM buffer, and fullscreens the toplevels placed on +/// it. The encode device is resolved here: an operator's explicit `encode_node_index` +/// (-1 software, >= 0 a device) always wins, and only the unset `-2` sentinel is filled from +/// the auto-picked render node. H.264 output masks the dimensions even, because 4:2:0 needs +/// even width and height. +/// 5. **Encode-path choice + render loop**: only a same-GPU GLES session encodes zero-copy on this +/// calloop thread, because the dmabuf and its EGL context are calloop-affine; every readback +/// flavor (OpenH264, striped x264/JPEG, Pixman, or a cross-GPU hardware encoder) builds its +/// encoders on that display's dedicated encode thread instead. A shared timer (paced at the +/// fastest active capture) renders each capturing output — its windows, popups and layers made +/// output-local, the cursor only on the pointer's output — applies the shared paint-over / +/// recovery-IDR policy per display, and delivers each display's encoded stripes through its own +/// frame callback. The zero-copy encode waits the GL render fence first, so a hardware encoder +/// reading the dmabuf through CUDA/VA never maps a half-rasterized (torn) frame. +/// 6. **Thread lifecycle**: the Python frame callback runs on a dedicated delivery thread so its +/// GIL never stalls calloop input / control dispatch, and in readback mode the encoders run on +/// the `wl-encode` thread. On a restart or stop the encode thread is torn down before the +/// delivery thread — it feeds the delivery sender and must be gone first — and the retained +/// callbacks are dropped and gated by a process-shutdown flag so nothing fires into a finalizing +/// interpreter. +fn run_wayland_thread( + command_rx: smithay::reexports::calloop::channel::Channel, + wake_rx: smithay::reexports::calloop::channel::Channel<()>, + command_tx: smithay::reexports::calloop::channel::Sender, + initial_width: i32, + initial_height: i32, + explicit_dri_node: String, + auto_gpu_selected: bool, + cursor_size: i32, +) { + let width: i32 = if initial_width > 0 { initial_width } else { 1024 }; + let height: i32 = if initial_height > 0 { initial_height } else { 768 }; + + let mut event_loop = EventLoop::::try_new().expect("Unable to create event_loop"); + let display: Display = Display::new().unwrap(); + let dh: DisplayHandle = display.handle(); + unsafe { + if let Ok(lib) = libloading::Library::new("libwayland-server.so.0") { + if let Ok(set_max) = lib.get::( + b"wl_display_set_default_max_buffer_size\0", + ) { + set_max( + dh.backend_handle().display_ptr() as *mut std::ffi::c_void, + 10 * 1024 * 1024, + ); + } + std::mem::forget(lib); + } + } + + let dri_node = explicit_dri_node; + + let mut use_gpu = !dri_node.is_empty(); + let render_node_path = dri_node.clone(); + + let mut gles_renderer = None; + let mut pixman_renderer = None; + let mut offscreen_buffer: Option<(BufferObject<()>, Dmabuf)> = None; + let mut dmabuf_global = None; + let mut gbm_device_raw = None; + let mut dmabuf_state = DmabufState::new(); + + let mut gpu_success = false; + if use_gpu { + println!("[Wayland] Initializing GL Renderer using device: {}", dri_node); + let init_res: Result<(), String> = (|| { + let device_path = std::path::Path::new(&dri_node); + let file = File::options().read(true).write(true).open(device_path) + .map_err(|e| format!("Failed to open render device: {}", e))?; + let file_for_alloc = file.try_clone() + .map_err(|e| format!("Failed to clone file for GBM Allocator: {}", e))?; + let gbm_allocator = RawGbmDevice::new(file_for_alloc) + .map_err(|_| "Failed to create Raw GBM Device")?; + let gbm = GbmDevice::new(file) + .map_err(|_| "Failed to create GBM device")?; + let egl = unsafe { EGLDisplay::new(gbm) } + .map_err(|_| "Failed to create EGL display")?; + let context = EGLContext::new(&egl) + .map_err(|_| "Failed to create EGL context")?; + let mut renderer = unsafe { GlesRenderer::new(context) } + .map_err(|_| "Failed to init GlesRenderer")?; + + if let Err(e) = renderer.bind_wl_display(&dh) { + println!("[Wayland] Warning: Failed to bind EGL to Wayland Display (Optional): {:?}", e); + } + + let formats = Bind::::supported_formats(&renderer) + .ok_or("Failed to query formats")? + .into_iter() + .collect::>(); + + let node = DrmNode::from_path(device_path) + .map_err(|_| "Failed to create DrmNode")?; + let dmabuf_default_feedback = DmabufFeedbackBuilder::new(node.dev_id(), formats.clone()).build(); + + dmabuf_global = Some(if let Ok(default_feedback) = dmabuf_default_feedback { + dmabuf_state.create_global_with_default_feedback::(&dh, &default_feedback) + } else { + dmabuf_state.create_global::(&dh, formats) + }); + + let bo = gbm_allocator.create_buffer_object( + width as u32, height as u32, GbmFormat::Argb8888, BufferObjectFlags::RENDERING + ).map_err(|_| "Failed to allocate GBM buffer")?; + + let dmabuf = create_dmabuf_from_bo(&bo); + offscreen_buffer = Some((bo, dmabuf)); + gbm_device_raw = Some(gbm_allocator); + gles_renderer = Some(renderer); + Ok(()) + })(); + + match init_res { + Ok(_) => gpu_success = true, + Err(e) => { + println!("[Wayland] GPU Initialization failed: {}. Falling back to Software Renderer (Pixman).", e); + use_gpu = false; + } + } + } + + if !gpu_success { + if !dri_node.is_empty() && !use_gpu { + } else { + println!("[Wayland] No render node. Initializing Software Renderer (Pixman)."); + } + pixman_renderer = Some(PixmanRenderer::new().expect("Failed to init PixmanRenderer")); + use_gpu = false; + } + + let compositor_state = CompositorState::new::(&dh); + let fractional_scale_state = FractionalScaleManagerState::new::(&dh); + let shm_state = ShmState::new::(&dh, vec![]); + let output_state = OutputManagerState::new_with_xdg_output::(&dh); + let mut seat_state = SeatState::new(); + let shell_state = XdgShellState::new::(&dh); + let space = Space::default(); + let layer_shell_state = WlrLayerShellState::new::(&dh); + let data_device_state = DataDeviceState::new::(&dh); + let data_control_state = DataControlState::new::(&dh, None, |_| true); + let _vk_global = dh.create_global::(1, ()); + let pointer_warp_state = PointerWarpManager::new::(&dh); + let relative_pointer_state = RelativePointerManagerState::new::(&dh); + let pointer_constraints_state = PointerConstraintsState::new::(&dh); + + let foreign_toplevel_list = ForeignToplevelListState::new::(&dh); + let xdg_decoration_state = XdgDecorationState::new::(&dh); + let single_pixel_buffer = SinglePixelBufferState::new::(&dh); + let viewporter_state = ViewporterState::new::(&dh); + let presentation_state = PresentationState::new::(&dh, 1); + let xdg_activation_state = XdgActivationState::new::(&dh); + let primary_selection_state = PrimarySelectionState::new::(&dh); + let popups = PopupManager::default(); + + let mut seat = seat_state.new_wl_seat(&dh, "seat0"); + seat.add_keyboard(XkbConfig::default(), 200, 25) + .expect("Failed to init keyboard"); + seat.add_pointer(); + + let mut state = AppState { + compositor_state, + fractional_scale_state, + viewporter_state, + presentation_state, + shm_state, + single_pixel_buffer, + dmabuf_state, + dmabuf_global, + output_state, + seat_state, + shell_state, + layer_shell_state, + space, + data_device_state, + data_control_state, + dh: dh.clone(), + seat, + pointer_warp_state, + relative_pointer_state, + pointer_constraints_state, + output_nodes: Vec::new(), + pending_windows: Vec::new(), + foreign_toplevel_list, + xdg_decoration_state, + xdg_activation_state, + primary_selection_state, + popups, + gles_renderer, + pixman_renderer, + gbm_device: gbm_device_raw, + settings: RustCaptureSettings { + width, + height, + ..RustCaptureSettings::default() + }, + cursor_callback_set: false, + cursor_tx: wayland::cursor::spawn_cursor_worker(cursor_size), + clipboard_callback: None, + pending_clipboard_read: None, + current_selection_mime: None, + last_log_time: Instant::now(), + start_time: Instant::now(), + clock: Clock::new(), + use_gpu, + cursor_helper: Cursor::load(cursor_size), + keymap_policy: wayland::keymap::KeymapPolicy::empty(), + current_cursor_icon: None, + cursor_buffer: None, + render_cursor_on_framebuffer: false, + render_node_path, + auto_gpu_selected, + pending_screenshot: None, + command_rx: None, + }; + // Seed the keymap policy with the seat's initial keymap so overlay binds splice onto + // the exact text clients received. + { + let initial_keymap = if let Some(kb) = state.seat.get_keyboard() { + kb.with_xkb_state(&mut state, |context| match context.xkb().lock() { + Ok(guard) => { + let keymap = unsafe { guard.keymap() }; + keymap.get_as_string(smithay::input::keyboard::xkb::KEYMAP_FORMAT_TEXT_V1) + } + Err(_) => String::new(), + }) + } else { + String::new() + }; + state.keymap_policy.rebuild_base(initial_keymap); + } - if state.recording_sink.is_some() && settings.output_mode == 0 { - eprintln!( - "[recording_sink] WARNING: recording_socket is set but output_mode is JPEG (0). \ - The recording socket requires a single H.264 stream. Please set output_mode=1 \ - on the Python CaptureSettings to produce a recordable output." - ); - } + let output = Output::new( + "HEADLESS-1".into(), + PhysicalProperties { + size: (width, height).into(), + subpixel: Subpixel::Unknown, + make: "Pixelflux".into(), + model: "Virtual".into(), + serial_number: "001".into(), + }, + ); + output.change_current_state( + Some(OutputMode { + size: (width, height).into(), + refresh: 60_000, + }), + Some(Transform::Normal), + Some(OutputScale::Fractional(1.0)), + Some((0, 0).into()), + ); + output.set_preferred(OutputMode { + size: (width, height).into(), + refresh: 60_000, + }); + state.space.map_output(&output, (0, 0)); + let global = output.create_global::(&dh); + let damage_tracker = OutputDamageTracker::from_output(&output); + state.output_nodes.push(wayland::frontend::OutputNode { + id: 0, + output, + global, + pos: (0, 0), + damage_tracker, + frame_buffer: vec![0u8; (width.max(0) as usize) * (height.max(0) as usize) * 4], + offscreen_buffer, + overlay_state: OverlayState::default(), + capture: None, + }); - let watermark_output_scale = state - .outputs - .first() - .map(|o| o.current_scale().fractional_scale()) - .unwrap_or(1.0); - state - .overlay_state - .load_watermark(&settings.watermark_path, watermark_output_scale); - state.callback = Some(cb); - state.is_capturing = true; - WAYLAND_CAPTURE_ALIVE.store(true, Ordering::Release); - state.render_cursor_on_framebuffer = settings.capture_cursor; - state.settings = settings.clone(); - state.encode_stats.frames.store(0, Ordering::Relaxed); - state.encode_stats.stripes.store(0, Ordering::Relaxed); - state.last_log_time = Instant::now(); - state.frame_counter = 0; - state.pending_force_idr = false; - state.vaapi_state = StripeState::default(); - if state.cursor_callback.is_some() { - if let Some(icon) = state.current_cursor_icon.clone() { - state.send_cursor_image(&icon); - } - } - if let Some(p) = state.encode_pool.take() { p.shutdown(); } - let prior_readback_encoder = state - .encode_join - .take() - .and_then(|j| j.join().ok()) - .flatten(); - if let Some(tx) = state.deliver_tx.take() { drop(tx); } - if let Some(j) = state.deliver_join.take() { let _ = j.join(); } - state.pending_hw_delivery = None; - state.pending_hw_damage = false; - if let Some(cb) = state.callback.take() { - let (tx, rx) = std::sync::mpsc::sync_channel::>(1); - let join = thread::spawn(move || { - crate::boost_thread_priority(-10); - while let Ok(stripes) = rx.recv() { - if PY_SHUTDOWN.load(Ordering::Relaxed) { continue; } - Python::attach(|py| { - for s in stripes { - match Py::new(py, StripeFrame::new_owned_meta( - s.data, s.data_type, s.stripe_y_start, - s.stripe_height, s.frame_id, - )) { - Ok(f) => { if let Err(e) = cb.call1(py, (f,)) { e.print(py); } } - Err(e) => eprintln!("[wayland] frame alloc error: {e:?}"), - } - } - }); - } - }); - state.deliver_tx = Some(tx); - state.deliver_join = Some(join); - } + /// Apply every queued control command in FIFO order. Sends wake the loop through the + /// separate wake channel, and the render tick ALSO drains before starting its work, so + /// queued input is applied ahead of a long render/encode instead of waiting it out. + fn drain_thread_commands(state: &mut AppState) { + let Some(rx) = state.command_rx.take() else { return }; + while let Ok(cmd) = rx.try_recv() { + handle_thread_command(state, cmd); + } + state.command_rx = Some(rx); + } - if state.video_encoder.is_none() { - if let Some(ref deliver_tx) = state.deliver_tx { - let pool = Arc::new(WlFramePool::new( - WL_POOL_SURFACES, - (settings.width.max(0) as usize) * (settings.height.max(0) as usize) * 4, - )); - state.pool_last_render = vec![0; WL_POOL_SURFACES]; - state.render_seq = 0; - // u64::MAX marks every slot stale so each one is read back - // before its first publish, whatever the damage says. - state.pool_content_gen = vec![u64::MAX; WL_POOL_SURFACES]; - state.content_gen = 0; - let c = &state.encode_controls; - c.bitrate_kbps.store(settings.video_bitrate_kbps, Ordering::Relaxed); - c.vbv_mult_milli.store( - (settings.video_vbv_multiplier * 1000.0).round() as i32, - Ordering::Relaxed, - ); - c.fps_milli.store( - (settings.target_fps.max(1.0) * 1000.0) as u64, - Ordering::Relaxed, - ); - c.rate_dirty.store(false, Ordering::Relaxed); - c.force_idr.store(false, Ordering::Relaxed); - // Also drop any leftover LiveTunables snapshot: UpdateTunables - // sets it even when no encode thread is consuming (zero-copy - // mode, or after StopCapture), and without this reset the new - // encode thread's first frame would apply that stale snapshot - // OVER the fresh StartCapture settings just established. - c.tunables_dirty.store(false, Ordering::Relaxed); - *c.tunables.lock().unwrap() = None; - let cfg = WlEncodeConfig { - settings: settings.clone(), - use_gpu: state.use_gpu, - try_gpu: gpu_intent && (!state.use_gpu || different_gpu), - prior: prior_readback_encoder, - recording_sink: state.recording_sink.clone(), - deliver_tx: deliver_tx.clone(), - controls: state.encode_controls.clone(), - stats: state.encode_stats.clone(), - }; - let pool2 = pool.clone(); - state.encode_join = Some( - thread::Builder::new() - .name("wl-encode".into()) - .spawn(move || wayland_encode_loop(&pool2, cfg)) - .expect("failed to spawn wl-encode thread"), - ); - state.encode_pool = Some(pool); - } - } else { - state.encode_stats.n_stripes.store(1, Ordering::Relaxed); - *state.encode_stats.desc.lock().unwrap() = - encoder_desc(&settings, state.video_encoder.as_ref(), false, true); - log_stream_settings(&settings, 1, state.video_encoder.as_ref(), false); - } - // Force the keyframe on the now-active path (as RequestIdr does), - // unconditionally: the damage tracker and offscreen buffer stay - // warm across StopCapture/StartCapture, so a restarted capture on - // a static screen otherwise produces no damage, no first frame, - // and no IDR in either the striped or the zero-copy path (a - // stop+start encoder switch froze until the next screen damage). - if state.encode_pool.is_some() { - state.encode_controls.force_idr.store(true, Ordering::Relaxed); - } else { - state.pending_force_idr = true; - } - state.needs_full_render = true; + fn handle_thread_command(state: &mut AppState, cmd: ThreadCommand) { + match cmd { + ThreadCommand::StartCapture { display_id, callback, settings } => { + start_capture_on_display(state, display_id, callback, settings); + } + ThreadCommand::StopCapture { display_id } => { + // Cursor and clipboard callbacks deliberately SURVIVE StopCapture: + // captures cycle on client disconnects and setting restarts, and a + // copy or cursor change during that gap must still reach Python. + // PY_SHUTDOWN gates every use against a finalizing interpreter. + stop_capture_on_display(state, display_id); + } + ThreadCommand::CreateOutput { id, width, height, x, y, scale, reply } => { + let _ = reply.send(create_output_on(state, id, width, height, x, y, scale)); + } + ThreadCommand::DestroyOutput { id, reply } => { + let _ = reply.send(destroy_output_on(state, id)); + } + ThreadCommand::RepositionOutput { id, x, y, reply } => { + let _ = reply.send(reposition_output_on(state, id, x, y)); + } + ThreadCommand::ListOutputs { reply } => { + let list = state + .output_nodes + .iter() + .map(|n| { + let (w, h) = n + .output + .current_mode() + .map(|m| (m.size.w, m.size.h)) + .unwrap_or((0, 0)); + ( + n.id, + n.pos.0, + n.pos.1, + w, + h, + n.output.current_scale().fractional_scale(), + n.capture.is_some(), + ) + }) + .collect(); + let _ = reply.send(list); + } + ThreadCommand::MoveWindowToOutput { window_id, output_id, reply } => { + let window = state + .space + .elements() + .find(|w| { + wayland::frontend::window_meta(w) + .map(|m| m.id == window_id) + .unwrap_or(false) + }) + .cloned(); + let ok = match window { + Some(w) => state.place_window_on_output(&w, output_id), + None => false, + }; + let _ = reply.send(ok); } - CalloopEvent::Msg(ThreadCommand::StopCapture) => { - println!("[Wayland] Capture loop stopped."); - state.is_capturing = false; - WAYLAND_CAPTURE_ALIVE.store(false, Ordering::Release); - state.callback = None; - state.cursor_callback = None; - state.clipboard_callback = None; - state.video_encoder = None; - state.vaapi_state = StripeState::default(); - if let Some(p) = state.encode_pool.take() { p.shutdown(); } - if let Some(j) = state.encode_join.take() { let _ = j.join(); } - state.recording_sink = None; - *state.encode_stats.desc.lock().unwrap() = String::new(); - if let Some(tx) = state.deliver_tx.take() { drop(tx); } - if let Some(j) = state.deliver_join.take() { let _ = j.join(); } - state.pending_hw_delivery = None; - state.pending_hw_damage = false; + ThreadCommand::ListWindows { reply } => { + use smithay::wayland::shell::xdg::XdgToplevelSurfaceData; + let mut list = Vec::new(); + for window in state.space.elements() { + let Some(meta) = wayland::frontend::window_meta(window) else { continue }; + let (title, app_id) = window + .toplevel() + .map(|tl| { + with_states(tl.wl_surface(), |states| { + states + .data_map + .get::() + .map(|d| { + let a = d.lock().unwrap(); + ( + a.title.clone().unwrap_or_default(), + a.app_id.clone().unwrap_or_default(), + ) + }) + .unwrap_or_default() + }) + }) + .unwrap_or_default(); + list.push((meta.id, title, app_id, meta.output.load(Ordering::Relaxed))); + } + let _ = reply.send(list); } - CalloopEvent::Msg(ThreadCommand::SetClipboardCallback(cb)) => { + ThreadCommand::SetClipboardCallback(cb) => { state.clipboard_callback = Some(cb); + // Re-stage a read of the CURRENT selection so a copy made before this + // callback was (re)armed is delivered rather than lost; the post-dispatch + // drain performs the read (a compositor-owned selection is skipped there). + if let Some(mime) = state.current_selection_mime.clone() { + state.pending_clipboard_read = Some(mime); + } } - CalloopEvent::Msg(ThreadCommand::SetClipboard { mime, data }) => { + ThreadCommand::SetClipboard { mime, data } => { let mimes: Vec = if mime.starts_with("text/plain") { ["text/plain;charset=utf-8", "UTF8_STRING", "text/plain", "STRING", "TEXT"].iter().map(|s| s.to_string()).collect() @@ -2087,14 +3162,18 @@ fn run_wayland_thread( mimes, std::sync::Arc::new((mime, data)), ); + // The selection is compositor-owned now; a later SetClipboardCallback + // must not try to re-read a client source that no longer holds it. + state.current_selection_mime = None; } - CalloopEvent::Msg(ThreadCommand::SetCursorCallback(cb)) => { - state.cursor_callback = Some(cb); + ThreadCommand::SetCursorCallback(cb) => { + let _ = state.cursor_tx.send(CursorJob::SetCallback(cb)); + state.cursor_callback_set = true; if let Some(icon) = state.current_cursor_icon.clone() { state.send_cursor_image(&icon); } } - CalloopEvent::Msg(ThreadCommand::KeyboardKey { scancode, state: key_state_val }) => { + ThreadCommand::KeyboardKey { scancode, state: key_state_val } => { let key_state = if key_state_val > 0 { KeyState::Pressed } else { KeyState::Released }; let serial = next_serial(); let time = wayland_time(); @@ -2104,14 +3183,57 @@ fn run_wayland_thread( }); } } - CalloopEvent::Msg(ThreadCommand::SetKeymapString(text)) => { - if let Some(keyboard) = state.seat.get_keyboard() { - if let Err(e) = keyboard.set_keymap_from_string(state, text) { - eprintln!("[Wayland] keymap swap failed: {e:?}"); + ThreadCommand::SetKeymapString(text) => { + // Validate before mutating the policy so a bad string leaves the seat + // keymap untouched. + if crate::wayland::keymap::compile_keymap(&text).is_none() { + eprintln!("[Wayland] set_keymap_string: keymap failed to compile; keeping current keymap."); + } else { + state.keymap_policy.rebuild_base(text); + state.apply_keymap_policy(); + } + } + ThreadCommand::SetXkbLayout { rules, model, layout, variant, options, reply } => { + match crate::wayland::keymap::compile_rmlvo(&rules, &model, &layout, &variant, &options) { + Some(text) => { + state.keymap_policy.rebuild_base(text); + state.apply_keymap_policy(); + let _ = reply.send(true); + } + None => { + eprintln!("[Wayland] set_xkb_layout: RMLVO ({rules:?}, {model:?}, {layout:?}, {variant:?}, {options:?}) failed to compile."); + let _ = reply.send(false); } } } - CalloopEvent::Msg(ThreadCommand::GetXkbKeymap { reply }) => { + ThreadCommand::BindKeysyms { keysyms, reply } => { + let _ = reply.send(state.bind_keysyms(&keysyms)); + } + ThreadCommand::GetKeyboardState { reply } => { + let (pressed, mods) = state + .seat + .get_keyboard() + .map(|kb| { + let pressed: Vec = + kb.pressed_keys().iter().map(|c| c.raw()).collect(); + let m = kb.modifier_state(); + let mask = (m.ctrl as u32) + | (m.shift as u32) << 1 + | (m.alt as u32) << 2 + | (m.logo as u32) << 3 + | (m.caps_lock as u32) << 4 + | (m.num_lock as u32) << 5 + | (m.iso_level3_shift as u32) << 6 + | (m.iso_level5_shift as u32) << 7; + (pressed, mask) + }) + .unwrap_or_default(); + let _ = reply.send((pressed, mods)); + } + ThreadCommand::Barrier { reply } => { + let _ = reply.send(()); + } + ThreadCommand::GetXkbKeymap { reply } => { let mut keymap_str = String::new(); if let Some(keyboard) = state.seat.get_keyboard() { keymap_str = keyboard.with_xkb_state(state, |context| { @@ -2128,35 +3250,44 @@ fn run_wayland_thread( } let _ = reply.send(keymap_str); } - CalloopEvent::Msg(ThreadCommand::PointerMotion { x, y }) => { + ThreadCommand::PointerMotion { x, y } => { let serial = next_serial(); let time = wayland_time(); - let scale = state.settings.scale; - let logical_w = (state.settings.width as f64 / scale).floor(); - let logical_h = (state.settings.height as f64 / scale).floor(); - let logical_x = (x / scale).max(0.0).min(logical_w - 1.0); - let logical_y = (y / scale).max(0.0).min(logical_h - 1.0); + // (x, y) are physical union-layout coordinates: each output occupies + // the physical rectangle at its layout offset, and the point maps + // through the CONTAINING output's scale (clamped into the nearest + // output when outside all of them). + let p = state.layout_physical_to_logical(x, y); if let Some(pointer) = state.seat.get_pointer() { - let p = Point::::from((logical_x, logical_y)); - let mut under = None; - - if let Some(output) = state.outputs.first() { - let layer_map = layer_map_for_output(output); - let pointer_pos = p.to_i32_round(); - + // Layer surfaces live on the output under the point; their + // geometry is output-local, so hit-test with the local point and + // report the global location. + let layer_hit = |state: &AppState, layers: &[smithay::wayland::shell::wlr_layer::Layer]| { + let idx = state.node_idx_under(p)?; + let node = &state.output_nodes[idx]; + let origin = Point::::from(node.pos); + let local = (p - origin.to_f64()).to_i32_round(); + let layer_map = layer_map_for_output(&node.output); for layer in layer_map.layers().rev() { - let state_layer = layer.layer(); - if state_layer == smithay::wayland::shell::wlr_layer::Layer::Overlay || state_layer == smithay::wayland::shell::wlr_layer::Layer::Top { + if layers.contains(&layer.layer()) { if let Some(bbox) = layer_map.layer_geometry(layer) { - if bbox.contains(pointer_pos) { - under = Some((FocusTarget::LayerSurface(layer.clone()), bbox.loc.to_f64())); - break; + if bbox.contains(local) { + return Some(( + FocusTarget::LayerSurface(layer.clone()), + (bbox.loc + origin).to_f64(), + )); } } } } - } + None + }; + + let mut under = layer_hit(state, &[ + smithay::wayland::shell::wlr_layer::Layer::Overlay, + smithay::wayland::shell::wlr_layer::Layer::Top, + ]); if under.is_none() { under = state.space.element_under(p).map(|(window, loc)| { @@ -2165,44 +3296,26 @@ fn run_wayland_thread( } if under.is_none() { - if let Some(output) = state.outputs.first() { - let layer_map = layer_map_for_output(output); - let pointer_pos = p.to_i32_round(); - - for layer in layer_map.layers().rev() { - let state_layer = layer.layer(); - if state_layer == smithay::wayland::shell::wlr_layer::Layer::Bottom || state_layer == smithay::wayland::shell::wlr_layer::Layer::Background { - if let Some(bbox) = layer_map.layer_geometry(layer) { - if bbox.contains(pointer_pos) { - under = Some((FocusTarget::LayerSurface(layer.clone()), bbox.loc.to_f64())); - break; - } - } - } - } - } + under = layer_hit(state, &[ + smithay::wayland::shell::wlr_layer::Layer::Bottom, + smithay::wayland::shell::wlr_layer::Layer::Background, + ]); } pointer.motion(state, under, &MotionEvent { location: p, serial, time }); pointer.frame(state); } } - CalloopEvent::Msg(ThreadCommand::PointerRelativeMotion { dx, dy }) => { + ThreadCommand::PointerRelativeMotion { dx, dy } => { let utime = wayland_utime(); let time = wayland_time(); let serial = next_serial(); if let Some(pointer) = state.seat.get_pointer() { let current_pos = pointer.current_location(); - - let scale = state.settings.scale; - let max_w = (state.settings.width as f64 / scale).floor() - 1.0; - let max_h = (state.settings.height as f64 / scale).floor() - 1.0; - - let new_x = (current_pos.x + dx).max(0.0).min(max_w); - let new_y = (current_pos.y + dy).max(0.0).min(max_h); - - let new_pos = Point::::from((new_x, new_y)); + let new_pos = state.clamp_logical( + (current_pos.x + dx, current_pos.y + dy).into(), + ); let under = state.space.element_under(new_pos).map(|(window, loc)| { (FocusTarget::Window(window.clone()), loc.to_f64()) @@ -2228,7 +3341,7 @@ fn run_wayland_thread( pointer.frame(state); } } - CalloopEvent::Msg(ThreadCommand::PointerButton { btn, state: btn_state_val }) => { + ThreadCommand::PointerButton { btn, state: btn_state_val } => { let serial = next_serial(); let time = wayland_time(); let button_state = if btn_state_val > 0 { smithay::backend::input::ButtonState::Pressed } else { smithay::backend::input::ButtonState::Released }; @@ -2250,7 +3363,7 @@ fn run_wayland_thread( pointer.frame(state); } } - CalloopEvent::Msg(ThreadCommand::PointerAxis { x, y }) => { + ThreadCommand::PointerAxis { x, y } => { let time = wayland_time(); if let Some(pointer) = state.seat.get_pointer() { @@ -2276,668 +3389,236 @@ fn run_wayland_thread( } } } - CalloopEvent::Msg(ThreadCommand::UpdateCursorConfig { render_on_framebuffer }) => { + ThreadCommand::UpdateCursorConfig { render_on_framebuffer } => { state.render_cursor_on_framebuffer = render_on_framebuffer; } - CalloopEvent::Msg(ThreadCommand::RequestIdr) => { - if state.encode_pool.is_some() { - state.encode_controls.force_idr.store(true, Ordering::Relaxed); + ThreadCommand::SetCursorSize { size, reply } => { + if size <= 0 { + let _ = reply.send(false); } else { - state.pending_force_idr = true; - } - state.needs_full_render = true; - } - CalloopEvent::Msg(ThreadCommand::UpdateRate { bitrate_kbps, vbv_multiplier, fps }) => { - if let Some(b) = bitrate_kbps { state.settings.video_bitrate_kbps = b; } - if let Some(v) = vbv_multiplier { state.settings.video_vbv_multiplier = v; } - if let Some(f) = fps { if f > 0.0 { state.settings.target_fps = f; } } - if let Some(GpuEncoder::Nvenc(enc)) = state.video_encoder.as_mut() { - enc.reconfigure_rate(&state.settings); - } - if let Some(GpuEncoder::Vaapi(enc)) = state.video_encoder.as_mut() { - enc.reconfigure_rate(&state.settings); - } - let c = &state.encode_controls; - c.bitrate_kbps.store(state.settings.video_bitrate_kbps, Ordering::Relaxed); - c.vbv_mult_milli.store( - (state.settings.video_vbv_multiplier * 1000.0).round() as i32, - Ordering::Relaxed, - ); - c.fps_milli.store( - (state.settings.target_fps.max(1.0) * 1000.0) as u64, - Ordering::Relaxed, - ); - c.rate_dirty.store(true, Ordering::Release); - } - CalloopEvent::Msg(ThreadCommand::UpdateTunables(t)) => { - t.apply_to(&mut state.settings); - state.render_cursor_on_framebuffer = t.capture_cursor; - *state.encode_controls.tunables.lock().unwrap() = Some(t); - state.encode_controls.tunables_dirty.store(true, Ordering::Release); - } - CalloopEvent::Msg(ThreadCommand::CuScreenshot { resp }) => { - state.pending_screenshot = Some(resp); - } - CalloopEvent::Msg(ThreadCommand::CuCursorPosition { resp }) => { - let pos = state.seat.get_pointer() - .map(|p| p.current_location()) - .unwrap_or_else(|| (0.0f64, 0.0f64).into()); - let scale = state.settings.scale; - let fb_x = pos.x * scale; - let fb_y = pos.y * scale; - let _ = resp.send((fb_x, fb_y)); - } - CalloopEvent::Msg(ThreadCommand::CuGetInfo { resp }) => { - let _ = resp.send(( - state.settings.width, - state.settings.height, - state.settings.scale, - )); - } - CalloopEvent::Closed => {} - } - }) - .unwrap(); - - let source = ListeningSocketSource::new_auto().unwrap(); - let socket_name = source.socket_name().to_string_lossy().into_owned(); - println!("[Wayland] Socket listening on: {:?}", socket_name); - std::env::set_var("WAYLAND_DISPLAY", &socket_name); - - event_loop - .handle() - .insert_source(source, |client_stream, _, state| { - if let Err(err) = state - .dh - .insert_client(client_stream, Arc::new(ClientState::default())) - { - eprintln!("Error adding wayland client: {:?}", err); - } - }) - .expect("Failed to init wayland socket source"); - - let timer = Timer::immediate(); - let mut is_memory_throttling = false; - event_loop - .handle() - .insert_source(timer, move |_, _, state| { - let loop_start_time = Instant::now(); - state.space.refresh(); - - let current_rss = get_process_rss_bytes(); - let shm_usage = get_shm_usage_bytes(); - let memory_threshold = calculate_memory_threshold(state.settings.width, state.settings.height); - - if current_rss > memory_threshold || shm_usage > (4 * 1024 * 1024 * 1024) { - if !is_memory_throttling { - is_memory_throttling = true; - } - } else if is_memory_throttling - && current_rss < (memory_threshold as f64 * 0.75) as usize && shm_usage < (3 * 1024 * 1024 * 1024) { - is_memory_throttling = false; - } - - let now = Instant::now(); - let elapsed = now.duration_since(state.last_log_time).as_secs_f64(); - if elapsed >= 1.0 { - let frames = state.encode_stats.frames.swap(0, Ordering::Relaxed); - let stripes = state.encode_stats.stripes.swap(0, Ordering::Relaxed); - if state.is_capturing && state.settings.debug_logging { - let actual_fps = frames as f64 / elapsed; - let stripes_per_sec = stripes as f64 / elapsed; - let mode_str = state.encode_stats.desc.lock().unwrap().clone(); - let n_stripes = state.encode_stats.n_stripes.load(Ordering::Relaxed); - - let rss_mb = current_rss / 1024 / 1024; - let shm_mb = shm_usage / 1024 / 1024; - let throttle_warn = if is_memory_throttling { " [THROTTLED]" } else { "" }; - - println!("Res: {}x{} Mode: {} Stripes: {} EncFPS: {:.2} EncStripes/s: {:.2} Mem: {}MB SHM: {}MB{}", - state.settings.width, state.settings.height, mode_str, n_stripes, actual_fps, stripes_per_sec, rss_mb, shm_mb, throttle_warn); - } - state.last_log_time = now; - } - - if !state.is_capturing && state.pending_screenshot.is_none() { - return TimeoutAction::ToDuration(Duration::from_millis(16)); - } - - // A recorder connecting counts as a request. Folding it into the pending flags - // lets the decision layer send a decodable frame even when the screen is - // static, and keeps it armed across ticks that skip encoding (parked frame, - // exhausted pool). Both flags are set — like the request-IDR command handlers — - // because only one is consumed depending on which encode path is active. - if state - .recording_sink - .as_ref() - .map(|s| s.should_force_idr()) - .unwrap_or(false) - { - state.pending_force_idr = true; - state.encode_controls.force_idr.store(true, Ordering::Relaxed); - } - let requested_idr = state.pending_force_idr; - - let mut pool_slot: Option<(usize, Vec)> = None; - if let Some(ref pool) = state.encode_pool { - if !is_memory_throttling { - pool_slot = pool.try_begin(); - if pool_slot.is_none() { - return TimeoutAction::ToDuration(Duration::from_millis(1)); - } - } - } - - state.overlay_state.update_position( - state.settings.width, - state.settings.height, - state.settings.watermark_location_enum, - ); - - let mut render_success = false; - let mut render_sync = None; - let mut damage_rects: Vec> = Vec::new(); - let width = state.settings.width; - let height = state.settings.height; - - if let Some(output) = state.outputs.first().cloned() { - let render_age = if state.overlay_state.is_animated() || state.needs_full_render { 0 } else { 1 }; - if state.use_gpu { - if let Some(renderer) = state.gles_renderer.as_mut() { - if let Some((_bo, dmabuf)) = state.offscreen_buffer.as_mut() { - match renderer.bind(dmabuf) { - Ok(mut frame) => { - let mut elements: Vec>> = Vec::new(); - - if state.render_cursor_on_framebuffer { - if let Some(pointer) = state.seat.get_pointer() { - let pos = pointer.current_location(); - let output_scale_val = output.current_scale().fractional_scale(); - let scale = Scale::from(output_scale_val); - - if let Some(CursorImageStatus::Named(icon)) = &state.current_cursor_icon { - let name = wayland::frontend::cursor_icon_to_str(icon); - let time = Duration::from_millis(state.clock.now().as_millis() as u64); - if let Some(image) = state.cursor_helper.get_image_by_name(name, output_scale_val.round() as u32, time) { - if let Some(elem) = state.overlay_state.get_cursor_element(renderer, image, pos.to_i32_round()) { - elements.push(CompositionElements::Cursor(elem)); - } - } - } else if let Some(CursorImageStatus::Surface(surface)) = &state.current_cursor_icon { - let phys_pos = pos.to_physical(scale); - let elem_result = with_states(surface, |states| { - WaylandSurfaceRenderElement::from_surface(renderer, surface, states, phys_pos, 1.0, smithay::backend::renderer::element::Kind::Cursor) - }); - if let Ok(Some(cursor_elem)) = elem_result { - elements.push(CompositionElements::Surface(cursor_elem)); - } - } else if state.current_cursor_icon.is_none() { - let time = Duration::from_millis(state.clock.now().as_millis() as u64); - let image = state.cursor_helper.get_image(output_scale_val.round() as u32, time); - if let Some(elem) = state.overlay_state.get_cursor_element(renderer, image, pos.to_i32_round()) { - elements.push(CompositionElements::Cursor(elem)); - } - } - } - } - - if let Some(elem) = state.overlay_state.get_watermark_element(renderer) { - elements.push(CompositionElements::Cursor(elem)); - } - - { - let layer_map = layer_map_for_output(&output); - - let draw_layer = |renderer: &mut GlesRenderer, elements: &mut Vec>>, target_layer: smithay::wayland::shell::wlr_layer::Layer| { - for surface in layer_map.layers().rev() { - let current_layer = surface.layer(); - if current_layer == target_layer { - if let Some(geo) = layer_map.layer_geometry(surface) { - let elem = smithay::wayland::compositor::with_states(surface.wl_surface(), |states| { - WaylandSurfaceRenderElement::from_surface( - renderer, surface.wl_surface(), states, - geo.loc.to_physical_precise_round(1), 1.0, - smithay::backend::renderer::element::Kind::Unspecified - ) - }); - if let Ok(Some(e)) = elem { - elements.push(CompositionElements::Surface(e)); - } - } - } - } - }; - - draw_layer(renderer, &mut elements, smithay::wayland::shell::wlr_layer::Layer::Overlay); - draw_layer(renderer, &mut elements, smithay::wayland::shell::wlr_layer::Layer::Top); - } - - for window in state.space.elements_for_output(&output).collect::>().into_iter().rev() { - let window_loc = state.space.element_location(window).unwrap_or_default(); - - if let Some(surface) = window.wl_surface() { - let popups = PopupManager::popups_for_surface(&surface); - for (popup, location) in popups { - let popup_surface = popup.wl_surface(); - let popup_pos = window_loc + location; - let elem = smithay::wayland::compositor::with_states(popup_surface, |states| { - WaylandSurfaceRenderElement::from_surface( - renderer, - popup_surface, - states, - popup_pos.to_physical_precise_round(1), - 1.0, - smithay::backend::renderer::element::Kind::Unspecified - ) - }); - if let Ok(Some(e)) = elem { - elements.push(CompositionElements::Surface(e)); - } - } - } - - elements.extend(window.render_elements(renderer, window_loc.to_physical_precise_round(1), Scale::from(1.0), 1.0).into_iter().map(CompositionElements::Space)); - } - - { - let layer_map = layer_map_for_output(&output); - - let draw_layer = |renderer: &mut GlesRenderer, elements: &mut Vec>>, target_layer: smithay::wayland::shell::wlr_layer::Layer| { - for surface in layer_map.layers() { - let current_layer = surface.layer(); - if current_layer == target_layer { - if let Some(geo) = layer_map.layer_geometry(surface) { - let elem = smithay::wayland::compositor::with_states(surface.wl_surface(), |states| { - WaylandSurfaceRenderElement::from_surface( - renderer, surface.wl_surface(), states, - geo.loc.to_physical_precise_round(1), 1.0, - smithay::backend::renderer::element::Kind::Unspecified - ) - }); - if let Ok(Some(e)) = elem { - elements.push(CompositionElements::Surface(e)); - } - } - } - } - }; - - draw_layer(renderer, &mut elements, smithay::wayland::shell::wlr_layer::Layer::Bottom); - draw_layer(renderer, &mut elements, smithay::wayland::shell::wlr_layer::Layer::Background); - } - match damage_tracker.render_output(renderer, &mut frame, render_age, &elements, [0.1, 0.1, 0.1, 1.0]) { - Ok(result) => { - render_success = true; - if let Some(damage) = result.damage { - damage_rects = damage.clone(); - } - render_sync = Some(result.sync); - state.needs_full_render = false; - }, - Err(e) => eprintln!("Render error: {:?}", e) - } - if !damage_rects.is_empty() { - state.content_gen += 1; - } - if let Some((id, ref mut buf)) = pool_slot { - // No-damage ticks skip the readback, so a pooled - // buffer can lag the offscreen target whenever the - // encoder held the other slot across a tick. Publishing - // a lagging buffer would let a paint-over / burst / - // recovery send re-encode pre-damage pixels; one - // catch-up readback keeps every published buffer - // current while idle readbacks stay skipped. - if render_success && state.pool_content_gen[id] != state.content_gen { - let _ = renderer.with_context(|gl| unsafe { - gl.ReadPixels( - 0, - 0, - width, - height, - smithay::backend::renderer::gles::ffi::RGBA, - smithay::backend::renderer::gles::ffi::UNSIGNED_BYTE, - buf.as_mut_ptr() as *mut std::ffi::c_void, - ); - }); - state.pool_content_gen[id] = state.content_gen; - } - } else if state.pending_screenshot.is_some() { - let _ = renderer.with_context(|gl| unsafe { - gl.ReadPixels( - 0, - 0, - width, - height, - smithay::backend::renderer::gles::ffi::RGBA, - smithay::backend::renderer::gles::ffi::UNSIGNED_BYTE, - state.frame_buffer.as_mut_ptr() as *mut std::ffi::c_void, - ); - }); - } - }, - Err(e) => eprintln!("Failed to bind buffer: {:?}", e) - } - } - } - } else { - if let Some(renderer) = state.pixman_renderer.as_mut() { - let (ptr, buf_age) = match pool_slot { - Some((id, ref mut buf)) => { - let age = if state.pool_last_render[id] == 0 { - 0 - } else { - (state.render_seq + 1 - state.pool_last_render[id]) as usize - }; - (buf.as_mut_ptr() as *mut u32, age) - } - None => (state.frame_buffer.as_mut_ptr() as *mut u32, 0), - }; - let mut image = unsafe { - pixman::Image::from_raw_mut(pixman::FormatCode::A8R8G8B8, width as usize, height as usize, ptr, (width as usize) * 4, false).expect("Failed to create pixman image") - }; - match renderer.bind(&mut image) { - Ok(mut frame) => { - let mut elements: Vec>> = Vec::new(); - - if state.render_cursor_on_framebuffer { - if let Some(pointer) = state.seat.get_pointer() { - let pos = pointer.current_location(); - let output_scale_val = output.current_scale().fractional_scale(); - let scale = Scale::from(output_scale_val); - - if let Some(CursorImageStatus::Named(icon)) = &state.current_cursor_icon { - let name = wayland::frontend::cursor_icon_to_str(icon); - let time = Duration::from_millis(state.clock.now().as_millis() as u64); - if let Some(image) = state.cursor_helper.get_image_by_name(name, output_scale_val.round() as u32, time) { - if let Some(elem) = state.overlay_state.get_cursor_element(renderer, image, pos.to_i32_round()) { - elements.push(CompositionElements::Cursor(elem)); - } - } - } else if let Some(CursorImageStatus::Surface(surface)) = &state.current_cursor_icon { - let phys_pos = pos.to_physical(scale); - let elem_result = with_states(surface, |states| { - WaylandSurfaceRenderElement::from_surface(renderer, surface, states, phys_pos, 1.0, smithay::backend::renderer::element::Kind::Cursor) - }); - if let Ok(Some(cursor_elem)) = elem_result { - elements.push(CompositionElements::Surface(cursor_elem)); - } - } else if state.current_cursor_icon.is_none() { - let time = Duration::from_millis(state.clock.now().as_millis() as u64); - let image = state.cursor_helper.get_image(output_scale_val.round() as u32, time); - if let Some(elem) = state.overlay_state.get_cursor_element(renderer, image, pos.to_i32_round()) { - elements.push(CompositionElements::Cursor(elem)); - } - } - } - } - - if let Some(elem) = state.overlay_state.get_watermark_element(renderer) { - elements.push(CompositionElements::Cursor(elem)); - } - - { - let layer_map = layer_map_for_output(&output); - - let draw_layer = |renderer: &mut PixmanRenderer, elements: &mut Vec>>, target_layer: smithay::wayland::shell::wlr_layer::Layer| { - for surface in layer_map.layers() { - let current_layer = surface.layer(); - if current_layer == target_layer { - if let Some(geo) = layer_map.layer_geometry(surface) { - let elem = smithay::wayland::compositor::with_states(surface.wl_surface(), |states| { - WaylandSurfaceRenderElement::from_surface( - renderer, surface.wl_surface(), states, - geo.loc.to_physical_precise_round(1), 1.0, - smithay::backend::renderer::element::Kind::Unspecified - ) - }); - if let Ok(Some(e)) = elem { - elements.push(CompositionElements::Surface(e)); - } - } - } - } - }; - - draw_layer(renderer, &mut elements, smithay::wayland::shell::wlr_layer::Layer::Overlay); - draw_layer(renderer, &mut elements, smithay::wayland::shell::wlr_layer::Layer::Top); - } - - for window in state.space.elements_for_output(&output).collect::>().into_iter().rev() { - let loc = state.space.element_location(window).unwrap_or_default(); - - if let Some(surface) = window.wl_surface() { - let popups = PopupManager::popups_for_surface(&surface); - for (popup, location) in popups { - let popup_surface = popup.wl_surface(); { - let popup_pos = loc + location; - let elem = smithay::wayland::compositor::with_states(popup_surface, |states| { - WaylandSurfaceRenderElement::from_surface( - renderer, - popup_surface, - states, - popup_pos.to_physical_precise_round(1), - 1.0, - smithay::backend::renderer::element::Kind::Unspecified - ) - }); - if let Ok(Some(e)) = elem { - elements.push(CompositionElements::Surface(e)); - } - } - } - } - - elements.extend(window.render_elements(renderer, loc.to_physical_precise_round(1), Scale::from(1.0), 1.0).into_iter().map(CompositionElements::Space)); - } - - { - let layer_map = layer_map_for_output(&output); - - let draw_layer = |renderer: &mut PixmanRenderer, elements: &mut Vec>>, target_layer: smithay::wayland::shell::wlr_layer::Layer| { - for surface in layer_map.layers() { - let current_layer = surface.layer(); - if current_layer == target_layer { - if let Some(geo) = layer_map.layer_geometry(surface) { - let elem = smithay::wayland::compositor::with_states(surface.wl_surface(), |states| { - WaylandSurfaceRenderElement::from_surface( - renderer, surface.wl_surface(), states, - geo.loc.to_physical_precise_round(1), 1.0, - smithay::backend::renderer::element::Kind::Unspecified - ) - }); - if let Ok(Some(e)) = elem { - elements.push(CompositionElements::Surface(e)); - } - } - } - } - }; - - draw_layer(renderer, &mut elements, smithay::wayland::shell::wlr_layer::Layer::Bottom); - draw_layer(renderer, &mut elements, smithay::wayland::shell::wlr_layer::Layer::Background); - } - - let render_age = if state.overlay_state.is_animated() || state.needs_full_render { 0 } else { buf_age }; - match damage_tracker.render_output(renderer, &mut frame, render_age, &elements, [0.1, 0.1, 0.1, 1.0]) { - Ok(result) => { - render_success = true; - state.needs_full_render = false; - if let Some(damage) = result.damage { damage_rects = damage.clone(); } - }, - Err(e) => eprintln!("Render error: {:?}", e) - } - state.render_seq += 1; - if render_success { - if let Some((id, _)) = pool_slot { - state.pool_last_render[id] = state.render_seq; - } - } - }, - Err(e) => eprintln!("Failed to bind pixman image: {:?}", e) + state.cursor_helper = Cursor::load(size); + let _ = state.cursor_tx.send(CursorJob::SetSize(size)); + // The burned-in cursor changed size; force a repaint everywhere so + // a static screen doesn't keep showing the old sprite. + for node in state.output_nodes.iter_mut() { + if let Some(cap) = node.capture.as_mut() { + cap.needs_full_render = true; + } } + let _ = reply.send(true); } } - - if render_success { - let time = state.clock.now(); - for window in state.space.elements() { - window.send_frame(&output, time, Some(Duration::ZERO), |_, _| Some(output.clone())); - } - - if is_memory_throttling { - if state.encode_pool.is_none() { - state.frame_counter = state.frame_counter.wrapping_add(1); - } - } else if state.encode_pool.is_some() { - if state.pending_screenshot.is_some() { - if let Some((_, ref buf)) = pool_slot { - let n = buf.len().min(state.frame_buffer.len()); - state.frame_buffer[..n].copy_from_slice(&buf[..n]); + ThreadCommand::RequestIdr { display_id } => { + if let Some(idx) = state.node_idx_for_id(display_id) { + if let Some(cap) = state.output_nodes[idx].capture.as_mut() { + if cap.encode_pool.is_some() { + cap.encode_controls.force_idr.store(true, Ordering::Relaxed); + } else { + cap.pending_force_idr = true; } + cap.needs_full_render = true; } - if let Some((id, buf)) = pool_slot.take() { - let is_animated = state.overlay_state.is_animated(); - state.encode_pool.as_ref().unwrap().publish(WlFrame { - id, - buf, - frame_id: state.frame_counter, - damage: std::mem::take(&mut damage_rects), - is_animated, - }); - state.frame_counter = state.frame_counter.wrapping_add(1); - } - } else if let Some(ref mut encoder) = state.video_encoder { - // Deliver the parked frame (if any) first, WITHOUT blocking: - // this runs on the calloop thread, and a blocking send would - // freeze input/command/Wayland dispatch for as long as the - // Python consumer stalls (the readback path offloads its - // sends to the wl-encode thread for exactly this reason). - // While a frame stays parked, no new frame is encoded — an - // encoded frame joins the H.264 reference chain and can - // never be dropped — and the tick's damage is latched so - // the pause never loses a change. - let slot_free = match state.pending_hw_delivery.take() { - None => true, - Some(pending) => match state.deliver_tx.as_ref() { - None => true, - Some(tx) => match tx.try_send(pending) { - Ok(()) => true, - Err(std::sync::mpsc::TrySendError::Full(p)) => { - state.pending_hw_delivery = Some(p); - false - } - Err(std::sync::mpsc::TrySendError::Disconnected(_)) => true, - }, - }, - }; - if !slot_free { - if !damage_rects.is_empty() { - state.pending_hw_damage = true; + } + } + ThreadCommand::UpdateRate { display_id, bitrate_kbps, vbv_multiplier, fps } => { + if let Some(idx) = state.node_idx_for_id(display_id) { + if let Some(cap) = state.output_nodes[idx].capture.as_mut() { + if let Some(b) = bitrate_kbps { cap.settings.video_bitrate_kbps = b; } + if let Some(v) = vbv_multiplier { cap.settings.video_vbv_multiplier = v; } + if let Some(f) = fps { if f > 0.0 { cap.settings.target_fps = f; } } + if let Some(GpuEncoder::Nvenc(enc)) = cap.video_encoder.as_mut() { + enc.reconfigure_rate(&cap.settings); } - } else { - let is_animated = state.overlay_state.is_animated(); - let had_damage = !damage_rects.is_empty() - || std::mem::take(&mut state.pending_hw_damage); - let decision = crate::pipeline::decide_hw_fullframe( - &mut state.vaapi_state, - &state.settings, - state.frame_counter, - had_damage, - is_animated, - requested_idr, - ); - let send_frame = decision.send; - let force_idr = decision.force_idr; - let target_qp = decision.target_qp; - - let mut frame_out = false; - if send_frame { - if let Some(sync) = render_sync.take() { - let _ = sync.wait(); + if let Some(GpuEncoder::Vaapi(enc)) = cap.video_encoder.as_mut() { + enc.reconfigure_rate(&cap.settings); } - let result = match encoder { - GpuEncoder::Nvenc(enc) => { - if let Some((_, ref dmabuf)) = state.offscreen_buffer { - enc.encode(dmabuf, state.frame_counter as u64, target_qp, force_idr) - } else { - Err("NVENC ZeroCopy requires offscreen buffer (GPU context)".to_string()) - } - }, - GpuEncoder::Vaapi(enc) => { - if let Some((_, ref dmabuf)) = state.offscreen_buffer { - enc.encode_dmabuf(dmabuf, state.frame_counter as u64, target_qp, force_idr) - } else { - Err("Vaapi ZeroCopy requires offscreen buffer (GPU context)".to_string()) - } - } - }; - - if let Ok(data) = result { - if !data.is_empty() { - frame_out = true; - state.encode_stats.frames.fetch_add(1, Ordering::Relaxed); - state.encode_stats.stripes.fetch_add(1, Ordering::Relaxed); - if let Some(ref tx) = state.deliver_tx { - let stripes = vec![EncodedStripe { - data: Arc::new(data), data_type: 2, stripe_y_start: 0, - stripe_height: height, frame_id: state.frame_counter as i32, - }]; - if let Some(ref socket) = state.recording_sink { - for stripe in &stripes { - socket.write_encoded_frame(stripe); - } - } - // Non-blocking: a full slot parks the frame - // (delivered ahead of any new encode above). - match tx.try_send(stripes) { - Ok(()) => {} - Err(std::sync::mpsc::TrySendError::Full(s)) => { - state.pending_hw_delivery = Some(s); - } - Err(std::sync::mpsc::TrySendError::Disconnected(_)) => {} - } - } - } - } else if let Err(e) = result { - eprintln!("HW Encode Error: {}", e); + let c = &cap.encode_controls; + c.bitrate_kbps.store(cap.settings.video_bitrate_kbps, Ordering::Relaxed); + c.vbv_mult_milli.store( + (cap.settings.video_vbv_multiplier * 1000.0).round() as i32, + Ordering::Relaxed, + ); + c.fps_milli.store( + (cap.settings.target_fps.max(1.0) * 1000.0) as u64, + Ordering::Relaxed, + ); + c.rate_dirty.store(true, Ordering::Release); + if display_id == 0 { + state.settings.video_bitrate_kbps = cap.settings.video_bitrate_kbps; + state.settings.video_vbv_multiplier = cap.settings.video_vbv_multiplier; + state.settings.target_fps = cap.settings.target_fps; } } - // An unserved request stays armed: on an infinite GOP an IDR lost - // to an encode error would never self-heal. - state.pending_force_idr = requested_idr && !frame_out; - state.frame_counter = state.frame_counter.wrapping_add(1); + } + } + ThreadCommand::UpdateTunables { display_id, tunables: t } => { + state.render_cursor_on_framebuffer = t.capture_cursor; + if display_id == 0 { + t.apply_to(&mut state.settings); + } + if let Some(idx) = state.node_idx_for_id(display_id) { + if let Some(cap) = state.output_nodes[idx].capture.as_mut() { + t.apply_to(&mut cap.settings); + *cap.encode_controls.tunables.lock().unwrap() = Some(t); + cap.encode_controls.tunables_dirty.store(true, Ordering::Release); } } - if let Some(resp) = state.pending_screenshot.take() { - if !state.frame_buffer.is_empty() { - let w = state.settings.width as u32; - let h = state.settings.height as u32; - let png = if state.use_gpu { - crate::computer_use::encode_png_rgba(&state.frame_buffer, w, h) - } else { - let mut rgba = state.frame_buffer.clone(); - for px in rgba.chunks_exact_mut(4) { - px.swap(0, 2); - } - crate::computer_use::encode_png_rgba(&rgba, w, h) - }; - match png { - Ok(data) => { let _ = resp.send(data); } - Err(e) => { let _ = resp.send(Vec::new()); eprintln!("[ComputerUse] PNG encode error: {}", e); } + } + ThreadCommand::CuScreenshot { display_id, resp } => { + if state.node_idx_for_id(display_id).is_some() { + state.pending_screenshot = Some((display_id, resp)); + } else { + let _ = resp.send(Err(format!("Unknown display: {display_id}"))); + } + } + ThreadCommand::CuCursorPosition { resp } => { + let pos = state.seat.get_pointer() + .map(|p| p.current_location()) + .unwrap_or_else(|| (0.0f64, 0.0f64).into()); + let _ = resp.send(state.layout_logical_to_physical(pos)); + } + ThreadCommand::CuGetInfo { display_id, resp } => { + let info = state + .node_idx_for_id(display_id) + .map(|idx| { + let node = &state.output_nodes[idx]; + match node.capture.as_ref() { + Some(c) => (c.settings.width, c.settings.height, c.settings.scale), + None => node + .output + .current_mode() + .map(|m| { + ( + m.size.w, + m.size.h, + node.output.current_scale().fractional_scale(), + ) + }) + .unwrap_or((0, 0, 0.0)), } - } else { - let _ = resp.send(Vec::new()); - } + }) + .unwrap_or((0, 0, 0.0)); + let _ = resp.send(info); + } + } + } + + state.command_rx = Some(command_rx); + event_loop + .handle() + .insert_source(wake_rx, |_, _, state| { + drain_thread_commands(state); + }) + .unwrap(); + + let source = ListeningSocketSource::new_auto().unwrap(); + let socket_name = source.socket_name().to_string_lossy().into_owned(); + println!("[Wayland] Socket listening on: {:?}", socket_name); + std::env::set_var("WAYLAND_DISPLAY", &socket_name); + publish_socket_name(&socket_name); + + event_loop + .handle() + .insert_source(source, |client_stream, _, state| { + if let Err(err) = state + .dh + .insert_client(client_stream, Arc::new(ClientState::default())) + { + eprintln!("Error adding wayland client: {:?}", err); + } + }) + .expect("Failed to init wayland socket source"); + + let timer = Timer::immediate(); + let mut is_memory_throttling = false; + event_loop + .handle() + .insert_source(timer, move |_, _, state| { + // Apply queued commands (input above all) BEFORE the render/encode work: a + // command that raced the timer wakeup would otherwise wait out the whole tick. + drain_thread_commands(state); + let loop_start_time = Instant::now(); + state.space.refresh(); + + let current_rss = get_process_rss_bytes(); + let shm_usage = get_shm_usage_bytes(); + // The threshold follows the largest active capture (fallback: primary settings). + let (max_w, max_h) = state + .output_nodes + .iter() + .filter_map(|n| n.capture.as_ref().map(|c| (c.settings.width, c.settings.height))) + .fold((state.settings.width, state.settings.height), |acc, (w, h)| { + (acc.0.max(w), acc.1.max(h)) + }); + let memory_threshold = calculate_memory_threshold(max_w, max_h); + + if current_rss > memory_threshold || shm_usage > (4 * 1024 * 1024 * 1024) { + if !is_memory_throttling { + is_memory_throttling = true; + } + } else if is_memory_throttling + && current_rss < (memory_threshold as f64 * 0.75) as usize && shm_usage < (3 * 1024 * 1024 * 1024) { + is_memory_throttling = false; + } + + let now = Instant::now(); + let elapsed = now.duration_since(state.last_log_time).as_secs_f64(); + if elapsed >= 1.0 { + for node in &state.output_nodes { + let Some(cap) = node.capture.as_ref() else { continue }; + let frames = cap.encode_stats.frames.swap(0, Ordering::Relaxed); + let stripes = cap.encode_stats.stripes.swap(0, Ordering::Relaxed); + if cap.settings.debug_logging { + let actual_fps = frames as f64 / elapsed; + let stripes_per_sec = stripes as f64 / elapsed; + let mode_str = cap.encode_stats.desc.lock().unwrap().clone(); + let n_stripes = cap.encode_stats.n_stripes.load(Ordering::Relaxed); + + let rss_mb = current_rss / 1024 / 1024; + let shm_mb = shm_usage / 1024 / 1024; + let throttle_warn = if is_memory_throttling { " [THROTTLED]" } else { "" }; + + println!("Display: {} Res: {}x{} Mode: {} Stripes: {} EncFPS: {:.2} EncStripes/s: {:.2} Mem: {}MB SHM: {}MB{}", + node.id, cap.settings.width, cap.settings.height, mode_str, n_stripes, actual_fps, stripes_per_sec, rss_mb, shm_mb, throttle_warn); } } + state.last_log_time = now; + } + + let any_capturing = state.output_nodes.iter().any(|n| n.capture.is_some()); + if !any_capturing && state.pending_screenshot.is_none() { + return TimeoutAction::ToDuration(Duration::from_millis(16)); } - if let Some((id, buf)) = pool_slot.take() { - if let Some(ref pool) = state.encode_pool { - pool.cancel(id, buf); + + // Render every output that needs it. The nodes are taken out of the state so + // each per-output render can borrow the shared renderer/space alongside its + // own damage tracker and buffers. + let mut nodes = std::mem::take(&mut state.output_nodes); + let mut any_pool_busy = false; + for node in nodes.iter_mut() { + if render_node_tick(state, node, is_memory_throttling) { + any_pool_busy = true; } } + state.output_nodes = nodes; + + if any_pool_busy { + return TimeoutAction::ToDuration(Duration::from_millis(1)); + } let work_elapsed = loop_start_time.elapsed(); - let fps = (if is_memory_throttling { 5.0 } else { state.settings.target_fps }).max(1.0); + let max_fps = state + .output_nodes + .iter() + .filter_map(|n| n.capture.as_ref().map(|c| c.settings.target_fps)) + .fold(0.0f64, f64::max); + let fps = (if is_memory_throttling { + 5.0 + } else if max_fps > 0.0 { + max_fps + } else { + state.settings.target_fps + }) + .max(1.0); let target_frame_duration = Duration::from_secs_f64(1.0 / fps); let wait_duration = target_frame_duration.saturating_sub(work_elapsed); let final_wait = if wait_duration.as_millis() < 1 { Duration::from_millis(1) } else { wait_duration }; @@ -2953,17 +3634,8 @@ fn run_wayland_thread( }) .unwrap(); - if let Ok(port_str) = std::env::var("PIXELFLUX_CU") { - if let Ok(port) = port_str.parse::() { - println!("[ComputerUse] Starting server on port {} (PIXELFLUX_CU={})", port, port); - let cu_tx = command_tx.clone(); - thread::spawn(move || { - crate::computer_use::run_cu_server(cu_tx, port); - }); - } else { - println!("[ComputerUse] Invalid PIXELFLUX_CU value: '{}' - expected a port number", port_str); - } - } + crate::computer_use::register_wayland_backend(command_tx.clone()); + crate::computer_use::spawn_cu_from_env(); event_loop.run(None, &mut state, |state| { state.process_pending_clipboard_read(); @@ -3045,6 +3717,18 @@ impl StripeFrame { #[pyclass] struct WaylandBackend { tx: smithay::reexports::calloop::channel::Sender, + /// Wakes the calloop after each command send: the command channel itself is drained in + /// place by the compositor thread (render tick and wake handler), not registered as its + /// own source, so input never waits behind an in-flight render tick's timer wakeup. + wake_tx: smithay::reexports::calloop::channel::Sender<()>, +} + +impl WaylandBackend { + fn send(&self, cmd: ThreadCommand) -> Result<(), String> { + self.tx.send(cmd).map_err(|e| e.to_string())?; + let _ = self.wake_tx.send(()); + Ok(()) + } } #[pymethods] @@ -3063,61 +3747,155 @@ impl WaylandBackend { cursor_size: i32, ) -> Self { let (tx, rx) = smithay::reexports::calloop::channel::channel(); + let (wake_tx, wake_rx) = smithay::reexports::calloop::channel::channel(); let cu_tx = tx.clone(); thread::spawn(move || { crate::boost_thread_priority(-15); - run_wayland_thread(rx, cu_tx, width, height, dri_node, auto_gpu_selected, cursor_size); + run_wayland_thread(rx, wake_rx, cu_tx, width, height, dri_node, auto_gpu_selected, cursor_size); }); - WaylandBackend { tx } + WaylandBackend { tx, wake_tx } } - /// Begin a capture with the given frame callback and settings. + /// Begin a capture with the given frame callback and settings. The target display is + /// the settings' `display_id` attribute (absent = 0, the primary); each display id runs + /// at most one capture, independent of every other display's. /// /// Issuing the start also clears the interpreter-teardown gate: starting from Python proves the /// interpreter is live again after a manual atexit sweep. fn start_capture(&self, callback: Py, settings: &Bound<'_, PyAny>) -> PyResult<()> { let rust_settings = extract_settings(settings)?; + let display_id = read_display_id(settings); PY_SHUTDOWN.store(false, Ordering::Relaxed); - self.tx - .send(ThreadCommand::StartCapture(callback, rust_settings)) + self.send(ThreadCommand::StartCapture { display_id, callback: Some(callback), settings: rust_settings }) .map_err(|e| PyErr::new::(format!("Failed to send start command: {}", e)))?; Ok(()) } - fn stop_capture(&self) -> PyResult<()> { - self.tx - .send(ThreadCommand::StopCapture) + /// Stop the capture bound to `display_id` (default: the primary display). + #[pyo3(signature = (display_id = 0))] + fn stop_capture(&self, display_id: u32) -> PyResult<()> { + self.send(ThreadCommand::StopCapture { display_id }) .map_err(|e| PyErr::new::(format!("Failed to send stop command: {}", e)))?; Ok(()) } + /// Create an additional output (`WxH` physical pixels at fractional `scale`) mapped into + /// the layout at offset `(x, y)`; `id` is the display key used by every per-display API. + /// False when the id is taken, the geometry/scale is invalid, the rectangle overlaps a + /// live output, or the GPU render target cannot be allocated. Capture reconfigures + /// (`start_capture` on an existing output) resize without this validation, so a + /// multi-step relayout must stay overlap-free at every step by caller ordering. + #[pyo3(signature = (id, width, height, x = 0, y = 0, scale = 1.0))] + fn create_output( + &self, + py: Python<'_>, + id: u32, + width: i32, + height: i32, + x: i32, + y: i32, + scale: f64, + ) -> PyResult { + let (reply_tx, reply_rx) = std::sync::mpsc::channel::(); + self.send(ThreadCommand::CreateOutput { id, width, height, x, y, scale, reply: reply_tx }) + .map_err(|e| PyErr::new::(format!("Failed to create output: {}", e)))?; + Ok(py + .detach(move || reply_rx.recv_timeout(Duration::from_secs(2))) + .unwrap_or(false)) + } + + /// Destroy a secondary output: its capture ends cleanly and its windows relocate to the + /// primary output. False for the primary (id 0) or an unknown id. + fn destroy_output(&self, py: Python<'_>, id: u32) -> PyResult { + let (reply_tx, reply_rx) = std::sync::mpsc::channel::(); + self.send(ThreadCommand::DestroyOutput { id, reply: reply_tx }) + .map_err(|e| PyErr::new::(format!("Failed to destroy output: {}", e)))?; + Ok(py + .detach(move || reply_rx.recv_timeout(Duration::from_secs(2))) + .unwrap_or(false)) + } + + /// Move an existing output (the primary, id 0, included) to layout offset `(x, y)`; + /// its windows, absolute input injection, and cursor compositing follow. False for an + /// unknown id or a destination overlapping a live output. Capture reconfigures + /// (`start_capture` on an existing output) resize without this validation, so a + /// multi-step relayout must stay overlap-free at every step by caller ordering. + fn reposition_output(&self, py: Python<'_>, id: u32, x: i32, y: i32) -> PyResult { + let (reply_tx, reply_rx) = std::sync::mpsc::channel::(); + self.send(ThreadCommand::RepositionOutput { id, x, y, reply: reply_tx }) + .map_err(|e| PyErr::new::(format!("Failed to reposition output: {}", e)))?; + Ok(py + .detach(move || reply_rx.recv_timeout(Duration::from_secs(2))) + .unwrap_or(false)) + } + + /// Every live output as `(id, x, y, width, height, scale, capturing)` — width/height in + /// physical pixels, `(x, y)` the layout offset. + fn list_outputs(&self, py: Python<'_>) -> PyResult> { + let (reply_tx, reply_rx) = std::sync::mpsc::channel(); + self.send(ThreadCommand::ListOutputs { reply: reply_tx }) + .map_err(|e| PyErr::new::(format!("Failed to list outputs: {}", e)))?; + Ok(py + .detach(move || reply_rx.recv_timeout(Duration::from_secs(2))) + .unwrap_or_default()) + } + + /// Move the window with the given id onto output `output_id`, fullscreened at that + /// output's logical size. False for an unknown window or output id. + fn move_window_to_output(&self, py: Python<'_>, window_id: u32, output_id: u32) -> PyResult { + let (reply_tx, reply_rx) = std::sync::mpsc::channel::(); + self.send(ThreadCommand::MoveWindowToOutput { window_id, output_id, reply: reply_tx }) + .map_err(|e| PyErr::new::(format!("Failed to move window: {}", e)))?; + Ok(py + .detach(move || reply_rx.recv_timeout(Duration::from_secs(2))) + .unwrap_or(false)) + } + + /// Every mapped window as `(window_id, title, app_id, output_id)`. + fn list_windows(&self, py: Python<'_>) -> PyResult> { + let (reply_tx, reply_rx) = std::sync::mpsc::channel(); + self.send(ThreadCommand::ListWindows { reply: reply_tx }) + .map_err(|e| PyErr::new::(format!("Failed to list windows: {}", e)))?; + Ok(py + .detach(move || reply_rx.recv_timeout(Duration::from_secs(2))) + .unwrap_or_default()) + } + fn set_cursor_callback(&self, callback: Py) -> PyResult<()> { - self.tx - .send(ThreadCommand::SetCursorCallback(callback)) + self.send(ThreadCommand::SetCursorCallback(callback)) .map_err(|e| PyErr::new::(format!("Failed to set cursor callback: {}", e)))?; Ok(()) } + /// Recreate the cursor theme at `size` pixels — no restart: subsequent named-cursor + /// callbacks and the burned-in cursor overlay render at the new size. False for a + /// non-positive size. + fn set_cursor_size(&self, py: Python<'_>, size: i32) -> PyResult { + let (reply_tx, reply_rx) = std::sync::mpsc::channel::(); + self.send(ThreadCommand::SetCursorSize { size, reply: reply_tx }) + .map_err(|e| PyErr::new::(format!("Failed to set cursor size: {}", e)))?; + Ok(py + .detach(move || reply_rx.recv_timeout(Duration::from_secs(2))) + .unwrap_or(false)) + } + /// cb(mime: str, data: bytes) fires when a client app copies to the clipboard. fn set_clipboard_callback(&self, callback: Py) -> PyResult<()> { - self.tx - .send(ThreadCommand::SetClipboardCallback(callback)) + self.send(ThreadCommand::SetClipboardCallback(callback)) .map_err(|e| PyErr::new::(format!("Failed to set clipboard callback: {}", e)))?; Ok(()) } /// Compositor-side clipboard offer: serve `data` as `mime` to pasting clients. fn set_clipboard(&self, mime: String, data: Vec) -> PyResult<()> { - self.tx - .send(ThreadCommand::SetClipboard { mime, data }) + self.send(ThreadCommand::SetClipboard { mime, data }) .map_err(|e| PyErr::new::(format!("Failed to set clipboard: {}", e)))?; Ok(()) } fn inject_key(&self, scancode: u32, state: u32) -> PyResult<()> { - self.tx - .send(ThreadCommand::KeyboardKey { scancode, state }) + self.send(ThreadCommand::KeyboardKey { scancode, state }) .map_err(|e| PyErr::new::(format!("Failed to inject key: {}", e)))?; Ok(()) } @@ -3126,8 +3904,7 @@ impl WaylandBackend { /// owns keysym-to-keycode policy: define keycodes here, then press them via /// `inject_key`. Ordered with key events on the one compositor channel. fn set_keymap_string(&self, text: String) -> PyResult<()> { - self.tx - .send(ThreadCommand::SetKeymapString(text)) + self.send(ThreadCommand::SetKeymapString(text)) .map_err(|e| PyErr::new::(format!("Failed to set keymap: {}", e)))?; Ok(()) } @@ -3140,8 +3917,7 @@ impl WaylandBackend { /// and an empty string is returned when the keymap cannot be read in time. fn get_xkb_keymap_string(&self, py: Python<'_>) -> PyResult { let (reply_tx, reply_rx) = std::sync::mpsc::channel::(); - self.tx - .send(ThreadCommand::GetXkbKeymap { reply: reply_tx }) + self.send(ThreadCommand::GetXkbKeymap { reply: reply_tx }) .map_err(|e| PyErr::new::(format!("Failed to request keymap: {}", e)))?; let result = py.detach(move || reply_rx.recv_timeout(Duration::from_secs(2))); match result { @@ -3151,36 +3927,31 @@ impl WaylandBackend { } fn inject_mouse_move(&self, x: f64, y: f64) -> PyResult<()> { - self.tx - .send(ThreadCommand::PointerMotion { x, y }) + self.send(ThreadCommand::PointerMotion { x, y }) .map_err(|e| PyErr::new::(format!("Failed to inject motion: {}", e)))?; Ok(()) } fn inject_relative_mouse_move(&self, dx: f64, dy: f64) -> PyResult<()> { - self.tx - .send(ThreadCommand::PointerRelativeMotion { dx, dy }) + self.send(ThreadCommand::PointerRelativeMotion { dx, dy }) .map_err(|e| PyErr::new::(format!("Failed to inject relative motion: {}", e)))?; Ok(()) } fn inject_mouse_button(&self, btn: u32, state: u32) -> PyResult<()> { - self.tx - .send(ThreadCommand::PointerButton { btn, state }) + self.send(ThreadCommand::PointerButton { btn, state }) .map_err(|e| PyErr::new::(format!("Failed to inject button: {}", e)))?; Ok(()) } fn inject_mouse_scroll(&self, x: f64, y: f64) -> PyResult<()> { - self.tx - .send(ThreadCommand::PointerAxis { x, y }) + self.send(ThreadCommand::PointerAxis { x, y }) .map_err(|e| PyErr::new::(format!("Failed to inject axis: {}", e)))?; Ok(()) } fn set_cursor_rendering(&self, enabled: bool) -> PyResult<()> { - self.tx - .send(ThreadCommand::UpdateCursorConfig { render_on_framebuffer: enabled }) + self.send(ThreadCommand::UpdateCursorConfig { render_on_framebuffer: enabled }) .map_err(|e| PyErr::new::(format!("Failed to set cursor config: {}", e)))?; Ok(()) } @@ -3189,31 +3960,100 @@ impl WaylandBackend { /// or a decoder reset can resume immediately. With the default infinite GOP this /// is the only recovery path, so every consumer that can lose decoder state must /// call it. No-op cost on the JPEG/software path (keyframes are N/A). - fn request_idr_frame(&self) -> PyResult<()> { - self.tx - .send(ThreadCommand::RequestIdr) + #[pyo3(signature = (display_id = 0))] + fn request_idr_frame(&self, display_id: u32) -> PyResult<()> { + self.send(ThreadCommand::RequestIdr { display_id }) .map_err(|e| PyErr::new::(format!("Failed to request IDR: {}", e)))?; Ok(()) } - /// Apply a live bitrate (kbps) / VBV (kb) / framerate change to the running capture. - fn update_rate(&self, bitrate_kbps: Option, vbv_multiplier: Option, fps: Option) -> PyResult<()> { - self.tx - .send(ThreadCommand::UpdateRate { bitrate_kbps, vbv_multiplier, fps }) + /// Apply a live bitrate (kbps) / VBV (kb) / framerate change to the given display's + /// running capture. + #[pyo3(signature = (bitrate_kbps = None, vbv_multiplier = None, fps = None, display_id = 0))] + fn update_rate(&self, bitrate_kbps: Option, vbv_multiplier: Option, fps: Option, display_id: u32) -> PyResult<()> { + self.send(ThreadCommand::UpdateRate { display_id, bitrate_kbps, vbv_multiplier, fps }) .map_err(|e| PyErr::new::(format!("Failed to update rate: {}", e)))?; Ok(()) } + + /// Set the seat's BASE xkb layout from RMLVO names at runtime (empty strings select the + /// xkbcommon defaults). Returns whether the layout compiled and was applied; overlay binds + /// rebuild on top with their keycodes unchanged. + #[pyo3(signature = (layout, variant = String::new(), options = String::new(), model = String::new(), rules = String::new()))] + fn set_xkb_layout( + &self, + py: Python<'_>, + layout: String, + variant: String, + options: String, + model: String, + rules: String, + ) -> PyResult { + let (reply_tx, reply_rx) = std::sync::mpsc::channel::(); + self.send(ThreadCommand::SetXkbLayout { rules, model, layout, variant, options, reply: reply_tx }) + .map_err(|e| PyErr::new::(format!("Failed to set layout: {}", e)))?; + Ok(py + .detach(move || reply_rx.recv_timeout(Duration::from_secs(2))) + .unwrap_or(false)) + } + + /// Resolve `keysyms` to `(keycode, level)` pairs against the seat keymap, overlay-binding + /// every keysym the base cannot produce. Binding N new keysyms costs ONE keymap swap, and a + /// keycode currently held down is never rebound. `(0, 0)` marks an unbindable keysym. + fn bind_keysyms(&self, py: Python<'_>, keysyms: Vec) -> PyResult> { + let n = keysyms.len(); + let (reply_tx, reply_rx) = std::sync::mpsc::channel::>(); + self.send(ThreadCommand::BindKeysyms { keysyms, reply: reply_tx }) + .map_err(|e| PyErr::new::(format!("Failed to bind keysyms: {}", e)))?; + Ok(py + .detach(move || reply_rx.recv_timeout(Duration::from_secs(2))) + .unwrap_or_else(|_| vec![(0, 0); n])) + } + + /// Debug/verification readback of the seat keyboard: `(pressed_keycodes, modifier_mask)` + /// with mask bits 1 ctrl, 2 shift, 4 alt, 8 logo, 16 caps, 32 num, 64 altgr, 128 level5. + fn get_keyboard_state(&self, py: Python<'_>) -> PyResult<(Vec, u32)> { + let (reply_tx, reply_rx) = std::sync::mpsc::channel::<(Vec, u32)>(); + self.send(ThreadCommand::GetKeyboardState { reply: reply_tx }) + .map_err(|e| PyErr::new::(format!("Failed to read keyboard state: {}", e)))?; + Ok(py + .detach(move || reply_rx.recv_timeout(Duration::from_secs(2))) + .unwrap_or_default()) + } + + /// The capture geometry actually live on the given display: `(width, height, scale)` in + /// physical pixels. Reflects any degrade a `start_capture` performed (H.264 even-masking, + /// GBM allocation failure keeping the previous mode), and the command channel is FIFO, so + /// calling this after `start_capture` returns what that start realized. + #[pyo3(signature = (display_id = 0))] + fn get_realized_geometry(&self, py: Python<'_>, display_id: u32) -> PyResult<(i32, i32, f64)> { + let (reply_tx, reply_rx) = std::sync::mpsc::channel::<(i32, i32, f64)>(); + self.send(ThreadCommand::CuGetInfo { display_id, resp: reply_tx }) + .map_err(|e| PyErr::new::(format!("Failed to read geometry: {}", e)))?; + Ok(py + .detach(move || reply_rx.recv_timeout(Duration::from_secs(2))) + .unwrap_or((0, 0, 0.0))) + } } impl WaylandBackend { - fn update_tunables(&self, t: LiveTunables) -> PyResult<()> { - self.tx - .send(ThreadCommand::UpdateTunables(t)) + fn update_tunables(&self, display_id: u32, t: LiveTunables) -> PyResult<()> { + self.send(ThreadCommand::UpdateTunables { display_id, tunables: t }) .map_err(|e| PyErr::new::(format!("Failed to update tunables: {}", e)))?; Ok(()) } } +/// The optional `display_id` attribute on a settings object (absent/invalid = 0, the +/// primary display). +fn read_display_id(settings: &Bound<'_, PyAny>) -> u32 { + settings + .getattr("display_id") + .ok() + .and_then(|v| v.extract::().ok()) + .unwrap_or(0) +} + use std::sync::atomic::{AtomicBool, AtomicI32, AtomicU32, AtomicU64, Ordering}; use std::sync::{Condvar, Mutex, OnceLock}; @@ -3241,6 +4081,8 @@ fn stripe_frame_from_buffer( /// can stash extra attributes not listed here. #[pyclass(dict)] struct CaptureSettings { + /// Wayland display key this capture binds to (0 = primary output); ignored on X11. + #[pyo3(get, set)] display_id: u32, #[pyo3(get, set)] capture_width: i32, #[pyo3(get, set)] capture_height: i32, #[pyo3(get, set)] scale: f64, @@ -3298,6 +4140,7 @@ impl CaptureSettings { #[new] fn new(py: Python<'_>) -> Self { Self { + display_id: 0, capture_width: 1920, capture_height: 1080, scale: 1.0, capture_x: 0, capture_y: 0, target_fps: 60.0, jpeg_quality: 85, paint_over_jpeg_quality: 95, use_paint_over_quality: false, paint_over_trigger_frames: 10, @@ -3319,6 +4162,34 @@ impl CaptureSettings { /// Process-wide Wayland backend: input and capture share ONE compositor (constructed lazily). static WAYLAND_BACKEND: OnceLock>>> = OnceLock::new(); +/// The compositor's auto-picked socket name (e.g. "wayland-1"), published by the compositor +/// thread once its listening socket exists. `ListeningSocketSource::new_auto` binds the first +/// FREE wayland-N, which need not match any configured index — consumers must read the real +/// name from here instead of assuming one. +static WAYLAND_SOCKET_NAME: Mutex> = Mutex::new(None); +static WAYLAND_SOCKET_CV: Condvar = Condvar::new(); + +fn publish_socket_name(name: &str) { + *WAYLAND_SOCKET_NAME.lock().unwrap() = Some(name.to_string()); + WAYLAND_SOCKET_CV.notify_all(); +} + +/// Wait (bounded) for the compositor thread to publish its socket name. +fn wait_socket_name(timeout: Duration) -> Option { + let deadline = Instant::now() + timeout; + let mut g = WAYLAND_SOCKET_NAME.lock().unwrap(); + loop { + if let Some(name) = g.as_ref() { + return Some(name.clone()); + } + let now = Instant::now(); + if now >= deadline { + return None; + } + let (gg, _) = WAYLAND_SOCKET_CV.wait_timeout(g, deadline - now).unwrap(); + g = gg; + } +} /// Cursor callback registered before the backend exists (selkies registers it pre-start); /// applied when the backend is created, which is deferred to capture start so the real /// render node (not a placeholder) reaches the compositor. @@ -3327,16 +4198,25 @@ static PENDING_CURSOR_CALLBACK: Mutex>> = Mutex::new(None); /// threads must never attach to a finalizing interpreter (aborts the process pre-3.13). /// Cleared by a fresh capture start (only a live interpreter can start one). pub(crate) static PY_SHUTDOWN: AtomicBool = AtomicBool::new(false); -/// ScreenCapture id that owns the active shared Wayland capture (0 = none). Only the owner may -/// stop it, so an input-only or stale instance can't tear down a live capture. -static WAYLAND_OWNER: AtomicU64 = AtomicU64::new(0); -/// Real Wayland capture liveness (StartCapture -> true, StopCapture -> false), mirrored -/// from AppState.is_capturing so the Python-facing is_capturing() reports whether the -/// pipeline is actually running, not merely which ScreenCapture owns the backend. -static WAYLAND_CAPTURE_ALIVE: std::sync::atomic::AtomicBool = - std::sync::atomic::AtomicBool::new(false); -/// Hands each `ScreenCapture` a unique, monotonic id — the token `WAYLAND_OWNER` compares to -/// decide which instance is allowed to stop the one shared compositor capture. Starts at 1 so 0 can +/// Per-display capture ownership: display id -> the ScreenCapture id that owns that +/// display's capture. Only the owner may stop it, so an input-only or stale instance can't +/// tear down a live capture. +static WAYLAND_OWNERS: OnceLock>> = OnceLock::new(); + +fn wayland_owners() -> &'static Mutex> { + WAYLAND_OWNERS.get_or_init(|| Mutex::new(std::collections::HashMap::new())) +} + +/// Display ids whose capture pipeline is actually running (StartCapture inserts, +/// StopCapture/DestroyOutput remove), so the Python-facing is_capturing() reports pipeline +/// liveness, not merely which ScreenCapture owns the backend. +static WAYLAND_ALIVE_DISPLAYS: OnceLock>> = OnceLock::new(); + +fn wayland_alive() -> &'static Mutex> { + WAYLAND_ALIVE_DISPLAYS.get_or_init(|| Mutex::new(std::collections::HashSet::new())) +} +/// Hands each `ScreenCapture` a unique, monotonic id — the token `WAYLAND_OWNERS` compares to +/// decide which instance is allowed to stop a display's capture. Starts at 1 so 0 can /// mean "no owner". static NEXT_CAPTURE_ID: AtomicU64 = AtomicU64::new(1); /// Registry of every live X11 capture's `Controls`, so the atexit sweep can flag them all to @@ -3380,22 +4260,23 @@ pub(crate) fn boost_thread_priority(nice: libc::c_int) { /// Forward a live rate change to the shared Wayland backend (no-op if none is running). fn wayland_update_rate( py: Python<'_>, + display_id: u32, bitrate_kbps: Option, vbv_multiplier: Option, fps: Option, ) { if let Some(slot) = WAYLAND_BACKEND.get() { if let Some(be) = slot.lock().unwrap().as_ref() { - let _ = be.bind(py).borrow().update_rate(bitrate_kbps, vbv_multiplier, fps); + let _ = be.bind(py).borrow().update_rate(bitrate_kbps, vbv_multiplier, fps, display_id); } } } /// Forward live per-frame tunables to the shared Wayland backend (no-op if none is running). -fn wayland_update_tunables(py: Python<'_>, t: LiveTunables) { +fn wayland_update_tunables(py: Python<'_>, display_id: u32, t: LiveTunables) { if let Some(slot) = WAYLAND_BACKEND.get() { if let Some(be) = slot.lock().unwrap().as_ref() { - let _ = be.bind(py).borrow().update_tunables(t); + let _ = be.bind(py).borrow().update_tunables(display_id, t); } } } @@ -3500,6 +4381,8 @@ struct ScState { /// self-joining. deliver_handle: Option>, deliver_thread_id: Option, + /// The Wayland display id this instance's capture is bound to (backend == 2). + wl_display: u32, } /// Unified capture handle exposed to Python. Drives the X11 capture directly or delegates to the @@ -3522,7 +4405,7 @@ impl ScreenCapture { /// GIL across the joins would deadlock. A re-entrant stop arriving on the capture, encode, or /// deliver thread cannot join itself, so it detaches and lets the threads exit on the stop flag. fn stop_internal(&self, py: Python<'_>) -> PyResult<()> { - let (handle, deliver_handle, same_thread, backend, controls) = { + let (handle, deliver_handle, same_thread, backend, controls, wl_display) = { let mut st = self.inner.lock().unwrap(); if let Some(c) = &st.controls { c.stop.store(true, Ordering::Relaxed); @@ -3542,12 +4425,14 @@ impl ScreenCapture { let handle = st.handle.take(); let deliver_handle = st.deliver_handle.take(); let backend = st.backend; + let wl_display = st.wl_display; st.backend = 0; st.cap_thread_id = None; st.encode_thread_id = None; st.encode_tid_rx = None; st.deliver_thread_id = None; - (handle, deliver_handle, same, backend, controls) + st.wl_display = 0; + (handle, deliver_handle, same, backend, controls, wl_display) }; if let Some(c) = &controls { live_x11().lock().unwrap().retain(|x| !Arc::ptr_eq(x, c)); @@ -3556,13 +4441,20 @@ impl ScreenCapture { crate::x11::cursor::release(py); } if backend == 2 { - if WAYLAND_OWNER - .compare_exchange(self.id, 0, Ordering::AcqRel, Ordering::Relaxed) - .is_ok() - { + let did = wl_display; + let owned = { + let mut owners = wayland_owners().lock().unwrap(); + if owners.get(&did) == Some(&self.id) { + owners.remove(&did); + true + } else { + false + } + }; + if owned { if let Some(slot) = WAYLAND_BACKEND.get() { if let Some(be) = slot.lock().unwrap().as_ref() { - let _ = be.bind(py).borrow().stop_capture(); + let _ = be.bind(py).borrow().stop_capture(did); } } } @@ -3604,6 +4496,7 @@ impl ScreenCapture { encode_tid_rx: None, deliver_handle: None, deliver_thread_id: None, + wl_display: 0, }), } } @@ -3625,10 +4518,14 @@ impl ScreenCapture { callback: Py, settings: &Bound<'_, PyAny>, ) -> PyResult<()> { + let display_id = read_display_id(settings); let live_wayland_restart = want_wayland(settings) - && self.inner.lock().unwrap().backend == 2 - && WAYLAND_OWNER.load(Ordering::Relaxed) == self.id - && WAYLAND_CAPTURE_ALIVE.load(Ordering::Acquire); + && { + let st = self.inner.lock().unwrap(); + st.backend == 2 && st.wl_display == display_id + } + && wayland_owners().lock().unwrap().get(&display_id) == Some(&self.id) + && wayland_alive().lock().unwrap().contains(&display_id); if !live_wayland_restart { self.stop_internal(py)?; } @@ -3660,8 +4557,12 @@ impl ScreenCapture { cursor_size, )?; be.bind(py).borrow().start_capture(callback, settings)?; - WAYLAND_OWNER.store(self.id, Ordering::Relaxed); - self.inner.lock().unwrap().backend = 2; + wayland_owners().lock().unwrap().insert(display_id, self.id); + { + let mut st = self.inner.lock().unwrap(); + st.backend = 2; + st.wl_display = display_id; + } return Ok(()); } @@ -3809,9 +4710,9 @@ impl ScreenCapture { } fn request_idr_frame(&self, py: Python<'_>) -> PyResult<()> { - let (backend, controls) = { + let (backend, controls, did) = { let st = self.inner.lock().unwrap(); - (st.backend, st.controls.clone()) + (st.backend, st.controls.clone(), st.wl_display) }; match backend { 1 => { @@ -3822,7 +4723,7 @@ impl ScreenCapture { 2 => { if let Some(slot) = WAYLAND_BACKEND.get() { if let Some(be) = slot.lock().unwrap().as_ref() { - let _ = be.bind(py).borrow().request_idr_frame(); + let _ = be.bind(py).borrow().request_idr_frame(did); } } } @@ -3836,9 +4737,9 @@ impl ScreenCapture { /// On the X11 path the dirty flag is Release-published after the payload store, so the encode /// thread's Acquire read can never observe the flag set against a stale bitrate. fn update_video_bitrate(&self, py: Python<'_>, kbps: i32) -> PyResult<()> { - let (backend, controls) = { + let (backend, controls, did) = { let st = self.inner.lock().unwrap(); - (st.backend, st.controls.clone()) + (st.backend, st.controls.clone(), st.wl_display) }; match backend { 1 => { @@ -3847,16 +4748,16 @@ impl ScreenCapture { c.rate_dirty.store(true, Ordering::Release); } } - 2 => wayland_update_rate(py, Some(kbps), None, None), + 2 => wayland_update_rate(py, did, Some(kbps), None, None), _ => {} } Ok(()) } fn update_framerate(&self, py: Python<'_>, fps: f64) -> PyResult<()> { - let (backend, controls) = { + let (backend, controls, did) = { let st = self.inner.lock().unwrap(); - (st.backend, st.controls.clone()) + (st.backend, st.controls.clone(), st.wl_display) }; match backend { 1 => { @@ -3865,7 +4766,7 @@ impl ScreenCapture { c.rate_dirty.store(true, Ordering::Release); } } - 2 => wayland_update_rate(py, None, None, Some(fps)), + 2 => wayland_update_rate(py, did, None, None, Some(fps)), _ => {} } Ok(()) @@ -3873,9 +4774,9 @@ impl ScreenCapture { /// Live CBR VBV change, as a multiple of one frame's bit budget (<= 0 = policy default). fn update_vbv_multiplier(&self, py: Python<'_>, multiplier: f64) -> PyResult<()> { - let (backend, controls) = { + let (backend, controls, did) = { let st = self.inner.lock().unwrap(); - (st.backend, st.controls.clone()) + (st.backend, st.controls.clone(), st.wl_display) }; match backend { 1 => { @@ -3885,7 +4786,7 @@ impl ScreenCapture { c.rate_dirty.store(true, Ordering::Release); } } - 2 => wayland_update_rate(py, None, Some(multiplier), None), + 2 => wayland_update_rate(py, did, None, Some(multiplier), None), _ => {} } Ok(()) @@ -3897,9 +4798,9 @@ impl ScreenCapture { fn update_tunables(&self, py: Python<'_>, settings: &Bound<'_, PyAny>) -> PyResult<()> { let rs = extract_settings(settings)?; let t = LiveTunables::from_settings(&rs); - let (backend, controls) = { + let (backend, controls, did) = { let st = self.inner.lock().unwrap(); - (st.backend, st.controls.clone()) + (st.backend, st.controls.clone(), st.wl_display) }; match backend { 1 => { @@ -3910,7 +4811,7 @@ impl ScreenCapture { } crate::x11::cursor::set_size_cap(rs.cursor_size_cap); } - 2 => wayland_update_tunables(py, t), + 2 => wayland_update_tunables(py, did, t), _ => {} } Ok(()) @@ -3947,8 +4848,8 @@ impl ScreenCapture { .as_ref() .map(|c| !c.stop.load(Ordering::Relaxed)) .unwrap_or(false), - 2 => WAYLAND_OWNER.load(Ordering::Relaxed) == self.id - && WAYLAND_CAPTURE_ALIVE.load(Ordering::Acquire), + 2 => wayland_owners().lock().unwrap().get(&st.wl_display) == Some(&self.id) + && wayland_alive().lock().unwrap().contains(&st.wl_display), _ => false, } } @@ -4027,6 +4928,93 @@ impl ScreenCapture { )), } } + /// Set the seat's BASE xkb layout from RMLVO names; false when no backend runs or the + /// layout fails to compile. + #[pyo3(signature = (layout, variant = String::new(), options = String::new(), model = String::new(), rules = String::new()))] + fn set_xkb_layout( + &self, + py: Python<'_>, + layout: String, + variant: String, + options: String, + model: String, + rules: String, + ) -> PyResult { + wayland_backend_running(py).map_or(Ok(false), |be| { + be.bind(py).borrow().set_xkb_layout(py, layout, variant, options, model, rules) + }) + } + /// Resolve keysyms to `(keycode, level)` with batched overlay binding (one keymap swap + /// per call, held keycodes never rebound); all `(0, 0)` when no backend runs. + fn bind_keysyms(&self, py: Python<'_>, keysyms: Vec) -> PyResult> { + let n = keysyms.len(); + wayland_backend_running(py) + .map_or(Ok(vec![(0, 0); n]), |be| be.bind(py).borrow().bind_keysyms(py, keysyms)) + } + /// Seat keyboard readback: `(pressed_keycodes, modifier_mask)`; empty when no backend runs. + fn get_keyboard_state(&self, py: Python<'_>) -> PyResult<(Vec, u32)> { + wayland_backend_running(py) + .map_or(Ok((Vec::new(), 0)), |be| be.bind(py).borrow().get_keyboard_state(py)) + } + /// The capture geometry actually live on the given display `(width, height, scale)`; + /// `(0, 0, 0.0)` when no backend runs. + #[pyo3(signature = (display_id = 0))] + fn get_realized_geometry(&self, py: Python<'_>, display_id: u32) -> PyResult<(i32, i32, f64)> { + wayland_backend_running(py) + .map_or(Ok((0, 0, 0.0)), |be| be.bind(py).borrow().get_realized_geometry(py, display_id)) + } + /// Create an additional Wayland output (see `WaylandBackend.create_output`); false when + /// no backend runs. + #[pyo3(signature = (id, width, height, x = 0, y = 0, scale = 1.0))] + fn create_output( + &self, + py: Python<'_>, + id: u32, + width: i32, + height: i32, + x: i32, + y: i32, + scale: f64, + ) -> PyResult { + wayland_backend_running(py).map_or(Ok(false), |be| { + be.bind(py).borrow().create_output(py, id, width, height, x, y, scale) + }) + } + /// Destroy a secondary Wayland output; false when no backend runs. + fn destroy_output(&self, py: Python<'_>, id: u32) -> PyResult { + wayland_backend_running(py) + .map_or(Ok(false), |be| be.bind(py).borrow().destroy_output(py, id)) + } + /// Move a Wayland output (the primary included) to layout offset `(x, y)`; false when + /// no backend runs or the id is unknown. + fn reposition_output(&self, py: Python<'_>, id: u32, x: i32, y: i32) -> PyResult { + wayland_backend_running(py) + .map_or(Ok(false), |be| be.bind(py).borrow().reposition_output(py, id, x, y)) + } + /// Recreate the Wayland cursor theme at `size` pixels (named-cursor callbacks and the + /// burned-in overlay); false when no backend runs or the size is non-positive. + fn set_cursor_size(&self, py: Python<'_>, size: i32) -> PyResult { + wayland_backend_running(py) + .map_or(Ok(false), |be| be.bind(py).borrow().set_cursor_size(py, size)) + } + /// Every live Wayland output as `(id, x, y, width, height, scale, capturing)`; empty + /// when no backend runs. + fn list_outputs(&self, py: Python<'_>) -> PyResult> { + wayland_backend_running(py) + .map_or(Ok(Vec::new()), |be| be.bind(py).borrow().list_outputs(py)) + } + /// Move a window onto an output (fullscreened there); false when no backend runs. + fn move_window_to_output(&self, py: Python<'_>, window_id: u32, output_id: u32) -> PyResult { + wayland_backend_running(py).map_or(Ok(false), |be| { + be.bind(py).borrow().move_window_to_output(py, window_id, output_id) + }) + } + /// Every mapped window as `(window_id, title, app_id, output_id)`; empty when no + /// backend runs. + fn list_windows(&self, py: Python<'_>) -> PyResult> { + wayland_backend_running(py) + .map_or(Ok(Vec::new()), |be| be.bind(py).borrow().list_windows(py)) + } } /// Best-effort teardown: flag the capture thread to exit without joining. @@ -4044,11 +5032,96 @@ impl Drop for ScreenCapture { } } +/// Build a Python dict from a recorder status snapshot (one shape for status and stop). +fn recording_status_dict(py: Python<'_>, s: &crate::recorder::RecordingStatus) -> PyResult> { + let d = pyo3::types::PyDict::new(py); + d.set_item("active", s.active)?; + d.set_item("path", &s.path)?; + d.set_item("backend", s.backend)?; + d.set_item("mode", s.mode)?; + d.set_item("frames", s.frames)?; + d.set_item("sync_frames", s.sync_frames)?; + d.set_item("dropped", s.dropped)?; + d.set_item("skipped_non_h264", s.skipped_non_h264)?; + d.set_item("bytes", s.bytes)?; + d.set_item("duration_s", s.duration_s)?; + d.set_item("width", s.width)?; + d.set_item("height", s.height)?; + d.set_item("error", s.error.as_deref())?; + Ok(d.into_any().unbind()) +} + +/// Start the built-in MP4 recorder. Works with no capture and no client running: the +/// recorder owns an independent capture (X11 root, or a Wayland output of the in-process +/// compositor) and taps a live streaming session instead of restarting it. `settings` is an +/// optional `CaptureSettings` for a recorder-owned capture (H.264 only; `display_id` +/// selects the Wayland output); when omitted, `PIXELFLUX_RECORD_*` environment variables +/// and full-screen defaults apply. +#[pyfunction] +#[pyo3(signature = (path, settings = None))] +fn start_recording( + py: Python<'_>, + path: String, + settings: Option<&Bound<'_, PyAny>>, +) -> PyResult> { + let mut opts = crate::recorder::RecordOptions::from_env(path); + if let Some(s) = settings { + let rs = extract_settings(s)?; + if rs.output_mode != 1 { + return Err(PyErr::new::( + "recording requires H.264 capture settings (output_mode=1); JPEG cannot be recorded", + )); + } + opts.display_id = read_display_id(s); + if let Some(explicit) = s + .getattr("use_wayland") + .ok() + .and_then(|v| v.extract::().ok()) + { + opts.backend = Some(if explicit { + crate::recorder::PreferredBackend::Wayland + } else { + crate::recorder::PreferredBackend::X11 + }); + } + // Explicit settings are authoritative over the env knobs they subsume. + opts.fps = 0.0; + opts.bitrate_kbps = 0; + opts.capture = Some(rs); + } + let status = py + .detach(|| crate::recorder::start(opts)) + .map_err(PyErr::new::)?; + recording_status_dict(py, &status) +} + +/// Stop the active recording, finalize the MP4, and return the final status dict. Raises +/// when no recording is active or nothing recordable was captured. +#[pyfunction] +fn stop_recording(py: Python<'_>) -> PyResult> { + let status = py + .detach(crate::recorder::stop) + .map_err(PyErr::new::)?; + recording_status_dict(py, &status) +} + +/// Status of the live recording (or the last finished one); `None` if this process has +/// never recorded. +#[pyfunction] +fn recording_status(py: Python<'_>) -> PyResult> { + match crate::recorder::status() { + Some(s) => recording_status_dict(py, &s), + None => Ok(py.None()), + } +} + /// Bring the Wayland compositor socket up before any capture so apps launched early can /// connect (sets WAYLAND_DISPLAY for children of this process). Idempotent; the running /// backend keeps its node/dimensions on later calls. `render_node` is an explicit /// /dev/dri/renderD* path; `auto_gpu` is a truthy string or vendor/driver token; empty -/// values mean software rendering. +/// values mean software rendering. Returns the compositor's actual socket name (the +/// auto-picked `wayland-N`, which need not match any configured index); empty string only +/// if the socket did not come up in time. #[pyfunction] #[pyo3(signature = (width = 0, height = 0, render_node = String::new(), auto_gpu = String::new(), cursor_size = -1))] fn ensure_wayland_display( @@ -4058,9 +5131,19 @@ fn ensure_wayland_display( render_node: String, auto_gpu: String, cursor_size: i32, -) -> PyResult<()> { - ensure_wayland_backend(py, width, height, render_node, auto_gpu, String::new(), cursor_size) - .map(|_| ()) +) -> PyResult { + ensure_wayland_backend(py, width, height, render_node, auto_gpu, String::new(), cursor_size)?; + Ok(py + .detach(|| wait_socket_name(Duration::from_secs(5))) + .unwrap_or_default()) +} + +/// The running compositor's Wayland socket name (e.g. "wayland-1"), or None when no +/// compositor thread has been started (this never creates one). +#[pyfunction] +fn get_wayland_display_name(py: Python<'_>) -> Option { + wayland_backend_running(py)?; + py.detach(|| wait_socket_name(Duration::from_secs(2))) } /// Stop every live capture (registered with atexit) before interpreter finalization. @@ -4074,6 +5157,9 @@ fn ensure_wayland_display( #[pyfunction] fn _stop_all_captures(py: Python<'_>) { PY_SHUTDOWN.store(true, Ordering::Relaxed); + // Finalize any active recording first so its last buffered MP4 sample is flushed and + // its own capture (if any) is stopped through the normal path. + py.detach(crate::recorder::finalize_on_exit); *PENDING_CURSOR_CALLBACK.lock().unwrap() = None; crate::x11::cursor::shutdown(); if let Some(slot) = LIVE_X11.get() { @@ -4083,11 +5169,26 @@ fn _stop_all_captures(py: Python<'_>) { } if let Some(slot) = WAYLAND_BACKEND.get() { if let Some(be) = slot.lock().unwrap().as_ref() { - let _ = be.bind(py).borrow().stop_capture(); + let be = be.bind(py).borrow(); + let mut displays: Vec = + wayland_alive().lock().unwrap().iter().copied().collect(); + if !displays.contains(&0) { + displays.push(0); + } + for did in displays { + let _ = be.stop_capture(did); + } + // Wait (bounded) until the calloop finished processing the stops: dropping a + // hardware encoder session (NVENC/CUDA) mid-process-exit segfaults, so the + // interpreter must not finalize while that teardown is still running. + let (ack_tx, ack_rx) = std::sync::mpsc::channel::<()>(); + if be.send(ThreadCommand::Barrier { reply: ack_tx }).is_ok() { + let _ = py.detach(move || ack_rx.recv_timeout(Duration::from_secs(2))); + } } } - WAYLAND_OWNER.store(0, Ordering::Relaxed); - WAYLAND_CAPTURE_ALIVE.store(false, Ordering::Relaxed); + wayland_owners().lock().unwrap().clear(); + wayland_alive().lock().unwrap().clear(); py.detach(|| std::thread::sleep(Duration::from_millis(50))); } @@ -4103,15 +5204,81 @@ fn pixelflux(m: &Bound<'_, PyModule>) -> PyResult<()> { m.add("X11_CURSOR_CALLBACK", true)?; m.add_function(wrap_pyfunction!(stripe_frame_from_buffer, m)?)?; m.add_function(wrap_pyfunction!(ensure_wayland_display, m)?)?; + m.add_function(wrap_pyfunction!(get_wayland_display_name, m)?)?; + m.add_function(wrap_pyfunction!(start_recording, m)?)?; + m.add_function(wrap_pyfunction!(stop_recording, m)?)?; + m.add_function(wrap_pyfunction!(recording_status, m)?)?; m.add_function(wrap_pyfunction!(_stop_all_captures, m)?)?; if let Ok(atexit) = m.py().import("atexit") { let _ = atexit.call_method1("register", (m.getattr("_stop_all_captures")?,)); } + // Standalone CU entry point: with PIXELFLUX_CU set the server binds at import, serving + // X11 (via DISPLAY) until a Wayland compositor registers itself as the backend. + crate::computer_use::spawn_cu_from_env(); + // PIXELFLUX_RECORD=: start recording from process start (X11 immediately, or as + // soon as the in-process Wayland compositor comes up). + crate::recorder::autostart_from_env(); Ok(()) } +#[cfg(test)] +mod output_overlap_tests { + //! Invariants of the output-placement intersection predicate: strict interior + //! intersection (touching edges never overlap), containment and identity overlap, + //! empty/negative rectangles never overlap, and extreme coordinates do not wrap. + use super::rects_overlap; + + #[test] + fn disjoint_rects_do_not_overlap() { + assert!(!rects_overlap((0, 0, 100, 100), (200, 0, 100, 100))); + assert!(!rects_overlap((0, 0, 100, 100), (0, 200, 100, 100))); + } + + #[test] + fn touching_edges_do_not_overlap() { + // Right edge of a meets left edge of b, and bottom meets top. + assert!(!rects_overlap((0, 0, 100, 100), (100, 0, 100, 100))); + assert!(!rects_overlap((0, 0, 100, 100), (0, 100, 100, 100))); + // Corner touch only. + assert!(!rects_overlap((0, 0, 100, 100), (100, 100, 50, 50))); + } + + #[test] + fn one_pixel_intrusion_overlaps() { + assert!(rects_overlap((0, 0, 100, 100), (99, 0, 100, 100))); + assert!(rects_overlap((0, 0, 100, 100), (0, 99, 100, 100))); + } + + #[test] + fn containment_and_identity_overlap() { + assert!(rects_overlap((0, 0, 100, 100), (25, 25, 10, 10))); + assert!(rects_overlap((25, 25, 10, 10), (0, 0, 100, 100))); + assert!(rects_overlap((5, 5, 50, 50), (5, 5, 50, 50))); + } + + #[test] + fn empty_or_negative_rects_never_overlap() { + assert!(!rects_overlap((10, 10, 0, 50), (0, 0, 100, 100))); + assert!(!rects_overlap((10, 10, 50, 0), (0, 0, 100, 100))); + assert!(!rects_overlap((10, 10, -5, 5), (0, 0, 100, 100))); + assert!(!rects_overlap((0, 0, 100, 100), (10, 10, 0, 0))); + } + + #[test] + fn negative_origins_overlap_correctly() { + assert!(rects_overlap((-50, -50, 100, 100), (0, 0, 100, 100))); + assert!(!rects_overlap((-100, -100, 100, 100), (0, 0, 100, 100))); + } + + #[test] + fn extreme_coordinates_do_not_wrap() { + assert!(!rects_overlap((i32::MAX - 10, 0, 10, 10), (i32::MIN, 0, 10, 10))); + assert!(rects_overlap((i32::MAX - 10, 0, 10, 10), (i32::MAX - 5, 0, 10, 10))); + } +} + #[cfg(test)] mod wl_frame_pool_tests { //! Invariants under test: try_begin is non-blocking and hands out a buffer ONLY while the diff --git a/pixelflux/src/pipeline.rs b/pixelflux/src/pipeline.rs index f62a47f..26afaee 100644 --- a/pixelflux/src/pipeline.rs +++ b/pixelflux/src/pipeline.rs @@ -14,7 +14,6 @@ use crate::encoders::nvenc::NvencEncoder; use crate::encoders::oh264::Openh264Encoder; use crate::encoders::software::{encode_cpu, EncodedStripe, StripeState}; use crate::encoders::vaapi::VaapiEncoder; -use crate::recording_sink::RecordingSink; use crate::RustCaptureSettings; use std::sync::Arc; @@ -164,7 +163,8 @@ enum X11Encoder { /// content frame-to-frame. Full-frame H.264 runs through `decide_hw_fullframe`; striped JPEG/x264 /// runs through `encode_cpu` with `hash_damage=true`. /// -/// Recording sink fan-out is handled at the delivery layer. +/// Recording fan-out (socket sink and MP4 recorder) is handled at the delivery layer; a +/// consumer needing a keyframe goes through [`X11Pipeline::request_idr`] like everyone else. pub struct X11Pipeline { settings: RustCaptureSettings, stripes: Vec, @@ -172,7 +172,6 @@ pub struct X11Pipeline { hw_state: StripeState, frame_counter: u16, pending_force_idr: bool, - recording_sink: Option>, } impl X11Pipeline { @@ -182,8 +181,6 @@ impl X11Pipeline { /// /// * `settings` - Capture configuration. The `output_mode`, `use_openh264`, `use_cpu`, /// `encode_node_index`, and `video_fullcolor` fields drive encoder selection. - /// * `recording_sink` - Optional Unix-socket H.264 fan-out. Owned by the caller; not - /// rebound on auto-adjust resizes to avoid tearing down the socket listener. /// /// # Encoder selection /// @@ -191,7 +188,7 @@ impl X11Pipeline { /// 2. **NVENC** — on an NVIDIA driver (or no detectable GPU, since the attempt is cheap). /// 3. **VA-API** — on any other GPU driver (except 4:4:4 full-color, which falls to x264). /// 4. **Software** (`X11Encoder::None`) — on any hardware init failure or explicit software. - pub fn new(settings: RustCaptureSettings, recording_sink: Option>) -> Self { + pub fn new(settings: RustCaptureSettings) -> Self { let hw = if settings.output_mode == 1 && settings.use_openh264 { println!("[x11] OpenH264 software encoder selected."); match Openh264Encoder::new(&settings) { @@ -248,7 +245,6 @@ impl X11Pipeline { hw_state: StripeState::default(), frame_counter: 0, pending_force_idr: false, - recording_sink, } } @@ -340,14 +336,7 @@ impl X11Pipeline { pub fn process(&mut self, argb: &[u8], stride: usize) -> Vec { let width = self.settings.width; let height = self.settings.height; - // A recorder connecting is a first-class IDR request: folding it in here lets the - // decision layer send a decodable frame promptly even when the screen is static. - let requested = self.pending_force_idr - || self - .recording_sink - .as_ref() - .map(|s| s.should_force_idr()) - .unwrap_or(false); + let requested = self.pending_force_idr; let threshold = self.settings.damage_block_threshold; let duration = self.settings.damage_block_duration as i32; @@ -450,7 +439,7 @@ mod tests { use_paint_over_quality: false, ..Default::default() }; - let mut p = X11Pipeline::new(s, None); + let mut p = X11Pipeline::new(s); let stride = 128 * 4; let frame_a = vec![10u8; stride * 128]; let mut frame_b = frame_a.clone(); @@ -483,7 +472,7 @@ mod tests { target_fps: 60.0, ..Default::default() }; - let mut p = X11Pipeline::new(s, None); + let mut p = X11Pipeline::new(s); let stride = 128 * 4; let frame = vec![10u8; stride * 128]; assert!(!p.process(&frame, stride).is_empty(), "first frame emits"); diff --git a/pixelflux/src/recorder/mod.rs b/pixelflux/src/recorder/mod.rs new file mode 100644 index 0000000..893657e --- /dev/null +++ b/pixelflux/src/recorder/mod.rs @@ -0,0 +1,744 @@ +/* + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at https://mozilla.org/MPL/2.0/. + */ + +//! Built-in MP4 recorder: an independent pipeline that captures the desktop to a +//! ready-to-play fragmented MP4 without needing any connected client. +//! +//! One implementation serves three control surfaces — the Python module functions +//! (`start_recording` / `stop_recording` / `recording_status`), the `PIXELFLUX_RECORD*` +//! environment variables (record from process start), and the `record_start` / +//! `record_stop` / `record_status` endpoints on the Computer Use HTTP server — so their +//! behavior cannot drift. +//! +//! The recorder is a consumer of the existing capture machinery, never a hook inside an +//! encoder: +//! +//! * **X11**: it always runs its own [`crate::x11::run_capture`] instance against the root +//! window (a second capture of the same root is independent of any streaming session), with +//! the encoded frames delivered straight into the recorder's queue. +//! * **Wayland**: the compositor lives in this process and allows one capture per output, so +//! the recorder attaches at the delivery layer. When the output is already being captured +//! for a streaming client it taps that stream (and paces recovery keyframes through the +//! standard `RequestIdr` command); when the output is idle it starts its own capture with no +//! Python callback and taps the identical delivery point. +//! +//! Frames cross into the recorder through one bounded queue, mirroring the Unix-socket sink's +//! isolation contract: the delivery thread only clones an `Arc` and `try_send`s, an overflowing +//! queue drops frames (never blocks the pipeline), and all muxing happens on the recorder's own +//! writer thread. Timestamps are wall-clock, so the damage-driven, variable-rate frame flow +//! lands at its true times in the MP4. + +pub mod mp4; + +use std::sync::atomic::{AtomicBool, AtomicU32, AtomicU64, Ordering}; +use std::sync::{mpsc, Arc, Mutex}; +use std::thread; +use std::time::{Duration, Instant}; + +use crossbeam_channel::{bounded, Receiver, Sender, TrySendError}; + +use crate::encoders::software::EncodedStripe; +use crate::RustCaptureSettings; +use crate::ThreadCommand; + +/// Recorder queue bound, matching the socket sink's stalled-consumer policy: a writer that +/// falls this far behind loses frames instead of growing memory or blocking the encoder. +const QUEUE_CAP: usize = 256; + +/// A queued frame: `Arc`-shared encoded payload, byte offset where the Annex-B stream +/// starts (past the wire header when present), and the wall-clock capture time. +type TapFrame = (Arc>, usize, u64); + +/// Which capture system feeds the recording. +#[derive(Clone, Copy, PartialEq, Eq, Debug)] +pub enum PreferredBackend { + X11, + Wayland, +} + +/// Resolved recording parameters. Environment variables provide the defaults; explicit +/// Python/REST arguments override them. +#[derive(Clone)] +pub struct RecordOptions { + pub path: String, + pub display_id: u32, + /// Capture fps cap for a recorder-owned capture (`<= 0` = default 30). An attached + /// recording follows the live session's rate. + pub fps: f64, + /// Bitrate override in kbps for a recorder-owned capture (`<= 0` = settings default). + pub bitrate_kbps: i32, + /// Recovery-keyframe cadence in seconds (`<= 0` = default 2.0). Recorder-owned captures + /// carry it in their settings (scheduled IDRs); attached recordings pace standard + /// request-IDR commands at this interval. + pub keyframe_interval_s: f64, + pub backend: Option, + /// Full capture settings for a recorder-owned capture; `None` derives them (full root / + /// current output geometry, H.264 full-frame). + pub capture: Option, +} + +impl RecordOptions { + /// Options seeded entirely from `PIXELFLUX_RECORD_*` environment variables. + pub fn from_env(path: String) -> Self { + let f = |k: &str| std::env::var(k).ok().and_then(|v| v.parse::().ok()); + let backend = match std::env::var("PIXELFLUX_RECORD_BACKEND").ok().as_deref() { + Some("x11") => Some(PreferredBackend::X11), + Some("wayland") => Some(PreferredBackend::Wayland), + _ => None, + }; + Self { + path, + display_id: f("PIXELFLUX_RECORD_DISPLAY").map(|v| v as u32).unwrap_or(0), + fps: f("PIXELFLUX_RECORD_FPS").unwrap_or(0.0), + bitrate_kbps: f("PIXELFLUX_RECORD_BITRATE").map(|v| v as i32).unwrap_or(0), + keyframe_interval_s: f("PIXELFLUX_RECORD_KEYFRAME_S").unwrap_or(0.0), + backend, + capture: None, + } + } + + fn effective_fps(&self) -> f64 { + if self.fps > 0.0 { self.fps } else { 30.0 } + } + + fn effective_keyframe_s(&self) -> f64 { + if self.keyframe_interval_s > 0.0 { self.keyframe_interval_s } else { 2.0 } + } +} + +/// Live counters shared between the feeding side (delivery threads), the writer thread and +/// the status surfaces. +struct RecShared { + start: Instant, + enqueued: AtomicU64, + dropped: AtomicU64, + skipped_non_h264: AtomicU64, + muxed: AtomicU64, + sync_frames: AtomicU64, + bytes: AtomicU64, + width: AtomicU32, + height: AtomicU32, + last_idr_req_us: AtomicU64, + error: Mutex>, +} + +impl RecShared { + fn new() -> Arc { + Arc::new(Self { + start: Instant::now(), + enqueued: AtomicU64::new(0), + dropped: AtomicU64::new(0), + skipped_non_h264: AtomicU64::new(0), + muxed: AtomicU64::new(0), + sync_frames: AtomicU64::new(0), + bytes: AtomicU64::new(0), + width: AtomicU32::new(0), + height: AtomicU32::new(0), + last_idr_req_us: AtomicU64::new(0), + error: Mutex::new(None), + }) + } + + fn set_error(&self, msg: String) { + let mut g = self.error.lock().unwrap(); + if g.is_none() { + eprintln!("[recorder] {msg}"); + *g = Some(msg); + } + } +} + +/// Point-in-time view of the recorder, identical across the Python, env and REST surfaces. +#[derive(Clone, Debug)] +pub struct RecordingStatus { + pub active: bool, + pub path: String, + pub backend: &'static str, + pub mode: &'static str, + pub frames: u64, + pub sync_frames: u64, + pub dropped: u64, + pub skipped_non_h264: u64, + pub bytes: u64, + pub duration_s: f64, + pub width: u32, + pub height: u32, + pub error: Option, +} + +/// The wayland-side tap handle consulted by [`wayland_tap`]; cloneable so the delivery +/// thread can drop the registry lock before doing any work. +#[derive(Clone)] +struct WlTap { + display_id: u32, + tx: Sender, + shared: Arc, + /// Standard request-IDR path for attached recordings (`None` when the recorder owns the + /// capture and scheduled keyframes ride in its settings). + idr: Option, + keyframe_interval_us: u64, +} + +/// Sends `ThreadCommand::RequestIdr` for one display over the compositor's command channel. +#[derive(Clone)] +struct IdrRequester { + tx: smithay::reexports::calloop::channel::Sender, + display_id: u32, +} + +impl IdrRequester { + fn request(&self) { + let _ = self.tx.send(ThreadCommand::RequestIdr { display_id: self.display_id }); + } +} + +/// How the active recording is fed and what must be torn down on stop. +enum RecordingMode { + /// Recorder-owned X11 capture: its controls and capture thread. + X11Own { + controls: Arc, + join: thread::JoinHandle<()>, + }, + /// Recorder-owned Wayland capture on `display_id` (stopped unless a streaming client has + /// since taken ownership of the display). + WaylandOwn { display_id: u32 }, + /// Tap on a streaming client's live Wayland capture; nothing to stop. + WaylandAttached, +} + +struct ActiveRecording { + path: String, + backend: &'static str, + mode_name: &'static str, + mode: RecordingMode, + tx: Sender, + shared: Arc, + writer: thread::JoinHandle<()>, +} + +static ACTIVE: Mutex> = Mutex::new(None); +static LAST_FINISHED: Mutex> = Mutex::new(None); +/// Fast idle guard for [`wayland_tap`]: the delivery threads pay one relaxed atomic load +/// per frame while no recording is armed. +static WL_TAP_ARMED: AtomicBool = AtomicBool::new(false); +static WL_TAP: Mutex> = Mutex::new(None); + +/// Delivery-layer feed from the Wayland pipelines (readback encode loop and zero-copy +/// tick). Near-zero cost when idle; while recording it clones an `Arc` into the bounded +/// queue and never blocks. +pub(crate) fn wayland_tap(display_id: u32, stripes: &[EncodedStripe]) { + if !WL_TAP_ARMED.load(Ordering::Relaxed) { + return; + } + let Some(tap) = WL_TAP.lock().unwrap().clone() else { return }; + if tap.display_id != display_id { + return; + } + offer_frame(&tap.shared, &tap.tx, stripes); + if let Some(ref idr) = tap.idr { + pace_idr_requests(&tap.shared, idr, tap.keyframe_interval_us); + } +} + +/// Issue a standard request-IDR when the cadence interval has elapsed. Compare-and-swap on +/// the last-request stamp so concurrent delivery threads emit one request per interval. +fn pace_idr_requests(shared: &RecShared, idr: &IdrRequester, interval_us: u64) { + if interval_us == 0 { + return; + } + let now = shared.start.elapsed().as_micros() as u64; + let last = shared.last_idr_req_us.load(Ordering::Relaxed); + if now.saturating_sub(last) >= interval_us + && shared + .last_idr_req_us + .compare_exchange(last, now, Ordering::Relaxed, Ordering::Relaxed) + .is_ok() + { + idr.request(); + } +} + +/// Enqueue one delivered frame for muxing. Only a single full-frame H.264 stripe is +/// recordable; JPEG or striped output is counted and skipped so the surfaces can report a +/// clear "nothing recordable" error instead of writing a corrupt file. +fn offer_frame(shared: &RecShared, tx: &Sender, stripes: &[EncodedStripe]) { + if stripes.is_empty() { + return; + } + if stripes.len() != 1 || stripes[0].data_type != 2 || stripes[0].stripe_y_start != 0 { + shared.skipped_non_h264.fetch_add(1, Ordering::Relaxed); + return; + } + let s = &stripes[0]; + if s.data.is_empty() { + return; + } + let offset = if s.data.len() > 10 && s.data[0] == 0x04 { 10 } else { 0 }; + let pts_us = shared.start.elapsed().as_micros() as u64; + match tx.try_send((s.data.clone(), offset, pts_us)) { + Ok(()) => { + shared.enqueued.fetch_add(1, Ordering::Relaxed); + } + Err(TrySendError::Full(_)) => { + shared.dropped.fetch_add(1, Ordering::Relaxed); + } + Err(TrySendError::Disconnected(_)) => {} + } +} + +/// Writer-thread body: drain the queue, convert Annex-B to AVCC, and mux. Output starts at +/// the first IDR with parameter sets; earlier frames are discarded. Runs until every sender +/// is dropped (stop) or a write error occurs, then finalizes the file and publishes the +/// final status. +fn writer_thread( + rx: Receiver, + file: std::fs::File, + path: String, + backend: &'static str, + mode_name: &'static str, + shared: Arc, +) { + let mut writer = mp4::FragmentWriter::new(std::io::BufWriter::new(file)); + let mut builder = mp4::H264SampleBuilder::new(); + + 'recv: for (buf, offset, pts_us) in rx.iter() { + let Some(sample) = builder.build_sample(&buf[offset..]) else { continue }; + if !writer.init_written() { + if !sample.sync || !builder.have_parameter_sets() { + continue; + } + let Some(cfg) = builder.track_config() else { + shared.set_error("failed to parse H.264 SPS for MP4 init".to_string()); + break 'recv; + }; + shared.width.store(cfg.width, Ordering::Relaxed); + shared.height.store(cfg.height, Ordering::Relaxed); + if let Err(e) = writer.write_init(&cfg) { + shared.set_error(format!("MP4 init write failed: {e}")); + break 'recv; + } + } + if let Err(e) = writer.push_sample(sample.data, sample.sync, pts_us) { + shared.set_error(format!("MP4 write failed: {e}")); + break 'recv; + } + let st = writer.stats(); + shared.muxed.store(st.samples + 1, Ordering::Relaxed); // +1: one sample is buffered + shared.sync_frames.store(st.sync_samples, Ordering::Relaxed); + shared.bytes.store(st.bytes, Ordering::Relaxed); + } + + let final_stats = match writer.finish() { + Ok(st) => st, + Err(e) => { + shared.set_error(format!("MP4 finalize failed: {e}")); + mp4::Mp4Stats::default() + } + }; + if final_stats.samples == 0 { + shared.set_error( + "no recordable H.264 frames received (the session must produce a single \ + full-frame H.264 stream; JPEG and striped modes cannot be recorded)" + .to_string(), + ); + let _ = std::fs::remove_file(&path); + } + let status = RecordingStatus { + active: false, + path, + backend, + mode: mode_name, + frames: final_stats.samples, + sync_frames: final_stats.sync_samples, + dropped: shared.dropped.load(Ordering::Relaxed), + skipped_non_h264: shared.skipped_non_h264.load(Ordering::Relaxed), + bytes: final_stats.bytes, + duration_s: final_stats.duration_us as f64 / 1e6, + width: shared.width.load(Ordering::Relaxed), + height: shared.height.load(Ordering::Relaxed), + error: shared.error.lock().unwrap().clone(), + }; + println!( + "[recorder] finished {}: {} frames ({} sync), {:.2}s, {} bytes", + status.path, status.frames, status.sync_frames, status.duration_s, status.bytes + ); + *LAST_FINISHED.lock().unwrap() = Some(status); +} + +/// Capture settings for a recorder-owned capture: explicit settings when given (validated +/// H.264), otherwise derived defaults, always forced to the one recordable shape (full-frame +/// H.264, no socket rebind, scheduled recovery keyframes). +fn own_capture_settings(opts: &RecordOptions) -> Result { + let mut s = match &opts.capture { + Some(explicit) => { + if explicit.output_mode != 1 { + return Err( + "recording requires H.264 capture settings (output_mode=1); JPEG cannot be recorded" + .to_string(), + ); + } + explicit.clone() + } + None => RustCaptureSettings { + width: 0, + height: 0, + output_mode: 1, + capture_cursor: true, + target_fps: opts.effective_fps(), + ..Default::default() + }, + }; + s.video_fullframe = true; + s.recording_socket = String::new(); + if opts.fps > 0.0 { + s.target_fps = opts.fps; + } + if opts.bitrate_kbps > 0 { + s.video_bitrate_kbps = opts.bitrate_kbps; + } + if s.keyframe_interval_s <= 0.0 { + s.keyframe_interval_s = opts.effective_keyframe_s(); + } + Ok(s) +} + +/// Start a recording. Exactly one may be active per process; returns the initial status. +pub fn start(opts: RecordOptions) -> Result { + let mut guard = ACTIVE.lock().unwrap(); + if guard.is_some() { + return Err("a recording is already active".to_string()); + } + if opts.path.is_empty() { + return Err("recording path is empty".to_string()); + } + if let Some(cap) = &opts.capture { + if cap.output_mode != 1 { + return Err( + "recording requires H.264 capture settings (output_mode=1); JPEG cannot be recorded" + .to_string(), + ); + } + } + + let wl_tx = crate::computer_use::wayland_command_sender(); + let backend = match opts.backend { + Some(PreferredBackend::X11) => PreferredBackend::X11, + Some(PreferredBackend::Wayland) => { + if wl_tx.is_none() { + return Err("no Wayland compositor is running in this process".to_string()); + } + PreferredBackend::Wayland + } + None => { + if wl_tx.is_some() { + PreferredBackend::Wayland + } else if std::env::var("DISPLAY").map(|v| !v.is_empty()).unwrap_or(false) { + PreferredBackend::X11 + } else { + return Err( + "no capture backend available: no in-process Wayland compositor and DISPLAY is unset" + .to_string(), + ); + } + } + }; + + let file = std::fs::File::create(&opts.path) + .map_err(|e| format!("cannot create {}: {e}", opts.path))?; + let shared = RecShared::new(); + let (tx, rx) = bounded::(QUEUE_CAP); + + let (mode, mode_name, backend_name) = match backend { + PreferredBackend::Wayland => { + let cmd_tx = wl_tx.unwrap(); + let display_id = opts.display_id; + let attached = crate::wayland_alive().lock().unwrap().contains(&display_id); + if attached { + *WL_TAP.lock().unwrap() = Some(WlTap { + display_id, + tx: tx.clone(), + shared: shared.clone(), + idr: Some(IdrRequester { tx: cmd_tx.clone(), display_id }), + keyframe_interval_us: (opts.effective_keyframe_s() * 1e6) as u64, + }); + WL_TAP_ARMED.store(true, Ordering::Relaxed); + // The stream is mid-GOP: a standard request-IDR gives the file its first + // decodable frame promptly. + let _ = cmd_tx.send(ThreadCommand::RequestIdr { display_id }); + (RecordingMode::WaylandAttached, "attached", "wayland") + } else { + let mut settings = own_capture_settings(&opts)?; + if settings.width <= 0 || settings.height <= 0 { + let (reply_tx, reply_rx) = mpsc::channel(); + cmd_tx + .send(ThreadCommand::ListOutputs { reply: reply_tx }) + .map_err(|_| "wayland compositor is not accepting commands".to_string())?; + let outputs = reply_rx + .recv_timeout(Duration::from_secs(5)) + .map_err(|_| "wayland compositor did not report its outputs".to_string())?; + let out = outputs + .iter() + .find(|o| o.0 == display_id) + .ok_or_else(|| format!("no wayland output with display id {display_id}"))?; + settings.width = out.3; + settings.height = out.4; + settings.scale = out.5; + } + *WL_TAP.lock().unwrap() = Some(WlTap { + display_id, + tx: tx.clone(), + shared: shared.clone(), + idr: None, + keyframe_interval_us: 0, + }); + WL_TAP_ARMED.store(true, Ordering::Relaxed); + cmd_tx + .send(ThreadCommand::StartCapture { display_id, callback: None, settings }) + .map_err(|_| { + disarm_tap(); + "wayland compositor is not accepting commands".to_string() + })?; + // StartCapture has no reply; liveness shows up in the alive set. + let deadline = Instant::now() + Duration::from_secs(3); + loop { + if crate::wayland_alive().lock().unwrap().contains(&display_id) { + break; + } + if Instant::now() >= deadline { + disarm_tap(); + return Err(format!( + "wayland capture on display {display_id} failed to start" + )); + } + thread::sleep(Duration::from_millis(20)); + } + (RecordingMode::WaylandOwn { display_id }, "own-capture", "wayland") + } + } + PreferredBackend::X11 => { + let settings = own_capture_settings(&opts)?; + let controls = Arc::new(crate::x11::Controls::new(&settings)); + crate::live_x11().lock().unwrap().push(controls.clone()); + let err_slot: Arc>> = Arc::new(Mutex::new(None)); + let err_slot2 = err_slot.clone(); + let (etid_tx, etid_rx) = mpsc::channel(); + let cap_controls = controls.clone(); + let feed_tx = tx.clone(); + let feed_shared = shared.clone(); + let join = thread::spawn(move || { + let on_frame = move |stripes: Vec| { + offer_frame(&feed_shared, &feed_tx, &stripes); + }; + if let Err(e) = crate::x11::run_capture(settings, cap_controls, etid_tx, on_frame) { + eprintln!("[recorder] x11 capture error: {e}"); + *err_slot2.lock().unwrap() = Some(e); + } + }); + match etid_rx.recv_timeout(Duration::from_secs(3)) { + Ok(_) => {} + Err(mpsc::RecvTimeoutError::Disconnected) => { + let _ = join.join(); + crate::live_x11().lock().unwrap().retain(|c| !Arc::ptr_eq(c, &controls)); + let msg = err_slot + .lock() + .unwrap() + .clone() + .unwrap_or_else(|| "X11 capture exited during start".to_string()); + return Err(msg); + } + // Slow X server setup: the capture is still coming up; proceed. + Err(mpsc::RecvTimeoutError::Timeout) => {} + } + (RecordingMode::X11Own { controls, join }, "own-capture", "x11") + } + }; + + let writer = { + let shared = shared.clone(); + let path = opts.path.clone(); + thread::Builder::new() + .name("pf-recorder".to_string()) + .spawn(move || writer_thread(rx, file, path, backend_name, mode_name, shared)) + .map_err(|e| format!("failed to spawn recorder writer thread: {e}"))? + }; + + println!( + "[recorder] recording to {} ({} {})", + opts.path, backend_name, mode_name + ); + let status = RecordingStatus { + active: true, + path: opts.path.clone(), + backend: backend_name, + mode: mode_name, + frames: 0, + sync_frames: 0, + dropped: 0, + skipped_non_h264: 0, + bytes: 0, + duration_s: 0.0, + width: 0, + height: 0, + error: None, + }; + *guard = Some(ActiveRecording { + path: opts.path, + backend: backend_name, + mode_name, + mode, + tx, + shared, + writer, + }); + Ok(status) +} + +fn disarm_tap() { + WL_TAP_ARMED.store(false, Ordering::Relaxed); + *WL_TAP.lock().unwrap() = None; +} + +/// Stop the active recording, finalize the MP4, and return the final status. Errors when no +/// recording is active or when nothing recordable was ever received. +pub fn stop() -> Result { + let active = ACTIVE + .lock() + .unwrap() + .take() + .ok_or_else(|| "no recording is active".to_string())?; + disarm_tap(); + match active.mode { + RecordingMode::X11Own { controls, join } => { + controls.stop.store(true, Ordering::Relaxed); + let _ = join.join(); + crate::live_x11().lock().unwrap().retain(|c| !Arc::ptr_eq(c, &controls)); + } + RecordingMode::WaylandOwn { display_id } => { + // A streaming client that reconfigured this display now owns it; the capture + // must survive the recorder's exit. + let client_owns = crate::wayland_owners().lock().unwrap().contains_key(&display_id); + if !client_owns { + if let Some(tx) = crate::computer_use::wayland_command_sender() { + let _ = tx.send(ThreadCommand::StopCapture { display_id }); + // The next start classifies attached-vs-own against wayland_alive: + // wait for the compositor to drain the stop, or an immediate restart + // attaches to the dying capture and records nothing. + let (ack_tx, ack_rx) = mpsc::channel(); + if tx.send(ThreadCommand::Barrier { reply: ack_tx }).is_ok() { + let _ = ack_rx.recv_timeout(Duration::from_secs(2)); + } + } + } + } + RecordingMode::WaylandAttached => {} + } + drop(active.tx); + let _ = active.writer.join(); + let finished = LAST_FINISHED.lock().unwrap().clone().ok_or_else(|| { + format!("recording of {} produced no final status", active.path) + })?; + match finished.error { + Some(ref e) => Err(e.clone()), + None => Ok(finished), + } +} + +/// The current status: the live recording when one is active, otherwise the last finished +/// one; `None` when the process has never recorded. +pub fn status() -> Option { + let guard = ACTIVE.lock().unwrap(); + if let Some(a) = guard.as_ref() { + return Some(RecordingStatus { + active: true, + path: a.path.clone(), + backend: a.backend, + mode: a.mode_name, + frames: a.shared.muxed.load(Ordering::Relaxed), + sync_frames: a.shared.sync_frames.load(Ordering::Relaxed), + dropped: a.shared.dropped.load(Ordering::Relaxed), + skipped_non_h264: a.shared.skipped_non_h264.load(Ordering::Relaxed), + bytes: a.shared.bytes.load(Ordering::Relaxed), + duration_s: a.shared.start.elapsed().as_secs_f64(), + width: a.shared.width.load(Ordering::Relaxed), + height: a.shared.height.load(Ordering::Relaxed), + error: a.shared.error.lock().unwrap().clone(), + }); + } + drop(guard); + LAST_FINISHED.lock().unwrap().clone() +} + +/// Finalize any active recording at interpreter shutdown so the MP4's last buffered sample +/// is flushed. Best-effort; the fragmented layout keeps even an unflushed file playable. +pub fn finalize_on_exit() { + if ACTIVE.lock().unwrap().is_some() { + let _ = stop(); + } +} + +/// `PIXELFLUX_RECORD=`: record from process start. X11 (via `DISPLAY`) starts +/// immediately; otherwise a background thread waits for the in-process Wayland compositor to +/// come up and then starts. Retries transient failures until the deadline. +pub fn autostart_from_env() { + use std::sync::OnceLock; + static STARTED: OnceLock<()> = OnceLock::new(); + let Ok(path) = std::env::var("PIXELFLUX_RECORD") else { return }; + if path.is_empty() { + return; + } + let mut first = false; + STARTED.get_or_init(|| first = true); + if !first { + return; + } + thread::spawn(move || { + let opts = RecordOptions::from_env(path); + let deadline = Instant::now() + Duration::from_secs(300); + let mut last_err = String::new(); + loop { + let wayland_up = crate::computer_use::wayland_command_sender().is_some(); + let x11_up = std::env::var("DISPLAY").map(|v| !v.is_empty()).unwrap_or(false); + let ready = match opts.backend { + Some(PreferredBackend::X11) => x11_up, + Some(PreferredBackend::Wayland) => wayland_up, + None => wayland_up || x11_up, + }; + if ready { + match start(opts.clone()) { + Ok(_) => return, + Err(e) => last_err = e, + } + } + if Instant::now() >= deadline { + eprintln!( + "[recorder] PIXELFLUX_RECORD autostart gave up: {}", + if last_err.is_empty() { "no capture backend appeared" } else { &last_err } + ); + return; + } + thread::sleep(Duration::from_millis(250)); + } + }); +} + +/// Serialize a status into the JSON shape shared by the REST endpoints. +pub fn status_to_json(s: &RecordingStatus) -> serde_json::Value { + serde_json::json!({ + "active": s.active, + "path": s.path, + "backend": s.backend, + "mode": s.mode, + "frames": s.frames, + "sync_frames": s.sync_frames, + "dropped": s.dropped, + "skipped_non_h264": s.skipped_non_h264, + "bytes": s.bytes, + "duration_s": s.duration_s, + "width": s.width, + "height": s.height, + "error": s.error, + }) +} diff --git a/pixelflux/src/recorder/mp4.rs b/pixelflux/src/recorder/mp4.rs new file mode 100644 index 0000000..5e96c13 --- /dev/null +++ b/pixelflux/src/recorder/mp4.rs @@ -0,0 +1,806 @@ +/* + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at https://mozilla.org/MPL/2.0/. + */ + +//! Pure-Rust fragmented-MP4 (fMP4) muxer for the built-in recorder. +//! +//! Hand-rolled rather than pulled in as a dependency for two reasons: `ffmpeg-sys-next`'s +//! `avformat` feature would add libavformat as a hard runtime dependency of every build, and the +//! pure-Rust mp4 crates only write moov-trailing progressive files, which lose everything on a +//! crash. Fragmented MP4 needs no trailer and no seeking — each `moof`+`mdat` pair is +//! self-contained — so a file truncated by a crash or SIGKILL stays playable up to the last +//! fragment, and the writer works on any `Write` sink. +//! +//! Timestamps are caller-supplied wall-clock microseconds (damage-driven capture emits sparse, +//! irregular frames), carried at a 90 kHz track timescale with one sample per fragment: `tfdt` +//! anchors every sample at its true capture time, so variable framerate needs no constant-rate +//! lie. The sample duration is only known once the NEXT frame arrives, so one sample is always +//! buffered and flushed a frame behind (a clean stop closes it with the median observed +//! duration); on SIGKILL at most that one buffered frame is lost — every fragment already +//! written remains playable. +//! +//! Codec support is deliberately split: [`annexb_to_sample`] and the H.264-specific parameter-set +//! capture live in [`H264SampleBuilder`], while the fragment/box writer below is codec-agnostic +//! (bytes + sync flag + timestamps + a ready-made `stsd` sample entry). HEVC or AV1 recording +//! later means a new sample builder emitting an `hvc1`/`av01` entry, not a new muxer. + +use std::io::Write; + +/// 90 kHz: the conventional H.264 track timescale, exactly representing common frame intervals. +const TIMESCALE: u32 = 90_000; + +/// Fallback duration for the final buffered sample when only one frame was ever written +/// (no observed inter-frame delta to take a median of): 1/30 s. +const DEFAULT_LAST_DURATION: u32 = TIMESCALE / 30; + +/// Split an Annex-B elementary stream into NAL payloads (start codes removed, emulation +/// prevention bytes kept — the RBSP layer is only unescaped where a parser needs it). +pub fn split_annexb(data: &[u8]) -> Vec<&[u8]> { + let mut nals = Vec::new(); + let mut i = 0usize; + let mut nal_start: Option = None; + while i + 2 < data.len() { + if data[i] == 0 && data[i + 1] == 0 && data[i + 2] == 1 { + let code_start = if i > 0 && data[i - 1] == 0 { i - 1 } else { i }; + if let Some(s) = nal_start { + if code_start > s { + nals.push(&data[s..code_start]); + } + } + i += 3; + nal_start = Some(i); + } else if data[i + 2] == 0 { + // A zero at i+2 can begin the next start code; only advance one byte. + i += 1; + } else { + i += 3; + } + } + if let Some(s) = nal_start { + if data.len() > s { + nals.push(&data[s..]); + } + } + nals +} + +/// Unescape an H.264 RBSP: drop the emulation-prevention byte from every `00 00 03` run. +fn unescape_rbsp(data: &[u8]) -> Vec { + let mut out = Vec::with_capacity(data.len()); + let mut zeros = 0u32; + for &b in data { + if zeros >= 2 && b == 3 { + zeros = 0; + continue; + } + if b == 0 { + zeros += 1; + } else { + zeros = 0; + } + out.push(b); + } + out +} + +/// MSB-first bit reader over an unescaped RBSP, with Exp-Golomb decode for SPS parsing. +struct BitReader<'a> { + data: &'a [u8], + pos: usize, +} + +impl<'a> BitReader<'a> { + fn new(data: &'a [u8]) -> Self { + Self { data, pos: 0 } + } + + fn bit(&mut self) -> Option { + let byte = *self.data.get(self.pos / 8)?; + let bit = (byte >> (7 - (self.pos % 8))) & 1; + self.pos += 1; + Some(bit as u32) + } + + fn bits(&mut self, n: u32) -> Option { + let mut v = 0u32; + for _ in 0..n { + v = (v << 1) | self.bit()?; + } + Some(v) + } + + /// ue(v): count leading zeros, then read that many bits after the marker one. + fn ue(&mut self) -> Option { + let mut zeros = 0u32; + while self.bit()? == 0 { + zeros += 1; + if zeros > 31 { + return None; + } + } + let rest = self.bits(zeros)?; + Some((1u32 << zeros) - 1 + rest) + } + + fn se(&mut self) -> Option { + let k = self.ue()? as i64; + Some(if k % 2 == 0 { -(k / 2) as i32 } else { ((k + 1) / 2) as i32 }) + } +} + +/// Coded frame dimensions parsed out of an H.264 SPS NAL (with its NAL header byte). +/// +/// Walks every field ahead of `pic_width_in_mbs_minus1` — including the high-profile +/// chroma/bit-depth/scaling-list block — and applies the frame-cropping rectangle with the +/// chroma-format-dependent crop units, so 4:2:0, 4:2:2 and 4:4:4 streams from any of the +/// project's encoders all report their true display size. +pub fn parse_sps_dimensions(sps_nal: &[u8]) -> Option<(u32, u32)> { + if sps_nal.len() < 4 || sps_nal[0] & 0x1f != 7 { + return None; + } + let profile_idc = sps_nal[1]; + let rbsp = unescape_rbsp(&sps_nal[4..]); + let mut r = BitReader::new(&rbsp); + r.ue()?; // seq_parameter_set_id + + let mut chroma_format_idc = 1u32; + if matches!( + profile_idc, + 100 | 110 | 122 | 244 | 44 | 83 | 86 | 118 | 128 | 138 | 139 | 134 | 135 + ) { + chroma_format_idc = r.ue()?; + if chroma_format_idc == 3 { + r.bit()?; // separate_colour_plane_flag + } + r.ue()?; // bit_depth_luma_minus8 + r.ue()?; // bit_depth_chroma_minus8 + r.bit()?; // qpprime_y_zero_transform_bypass_flag + if r.bit()? == 1 { + // seq_scaling_matrix_present_flag + let lists = if chroma_format_idc == 3 { 12 } else { 8 }; + for i in 0..lists { + if r.bit()? == 1 { + let size = if i < 6 { 16 } else { 64 }; + let mut next_scale = 8i32; + let mut last_scale = 8i32; + for _ in 0..size { + if next_scale != 0 { + let delta = r.se()?; + next_scale = (last_scale + delta + 256) % 256; + } + if next_scale != 0 { + last_scale = next_scale; + } + } + } + } + } + } + + r.ue()?; // log2_max_frame_num_minus4 + let pic_order_cnt_type = r.ue()?; + if pic_order_cnt_type == 0 { + r.ue()?; // log2_max_pic_order_cnt_lsb_minus4 + } else if pic_order_cnt_type == 1 { + r.bit()?; // delta_pic_order_always_zero_flag + r.se()?; // offset_for_non_ref_pic + r.se()?; // offset_for_top_to_bottom_field + let n = r.ue()?; + for _ in 0..n { + r.se()?; + } + } + r.ue()?; // max_num_ref_frames + r.bit()?; // gaps_in_frame_num_value_allowed_flag + let pic_width_in_mbs = r.ue()? + 1; + let pic_height_in_map_units = r.ue()? + 1; + let frame_mbs_only = r.bit()?; + if frame_mbs_only == 0 { + r.bit()?; // mb_adaptive_frame_field_flag + } + r.bit()?; // direct_8x8_inference_flag + + let (mut crop_l, mut crop_r, mut crop_t, mut crop_b) = (0u32, 0u32, 0u32, 0u32); + if r.bit()? == 1 { + crop_l = r.ue()?; + crop_r = r.ue()?; + crop_t = r.ue()?; + crop_b = r.ue()?; + } + + let (sub_w, sub_h) = match chroma_format_idc { + 0 | 3 => (1u32, 1u32), + 2 => (2, 1), + _ => (2, 2), + }; + let crop_unit_x = sub_w; + let crop_unit_y = sub_h * (2 - frame_mbs_only); + let width = pic_width_in_mbs * 16 - crop_unit_x * (crop_l + crop_r); + let height = (2 - frame_mbs_only) * pic_height_in_map_units * 16 - crop_unit_y * (crop_t + crop_b); + Some((width, height)) +} + +fn mk_box(fourcc: &[u8; 4], payload: &[u8]) -> Vec { + let mut b = Vec::with_capacity(8 + payload.len()); + b.extend_from_slice(&((8 + payload.len()) as u32).to_be_bytes()); + b.extend_from_slice(fourcc); + b.extend_from_slice(payload); + b +} + +fn mk_full_box(fourcc: &[u8; 4], version: u8, flags: u32, payload: &[u8]) -> Vec { + let mut p = Vec::with_capacity(4 + payload.len()); + p.push(version); + p.extend_from_slice(&flags.to_be_bytes()[1..]); + p.extend_from_slice(payload); + mk_box(fourcc, &p) +} + +const MATRIX_IDENTITY: [u8; 36] = { + let mut m = [0u8; 36]; + m[1] = 0x01; // 0x00010000 + m[17] = 0x01; + m[32] = 0x40; // 0x40000000 + m +}; + +/// A codec's contribution to the init segment: its `stsd` sample entry (with decoder +/// configuration record) plus the display dimensions for `tkhd`. +pub struct TrackConfig { + pub sample_entry: Vec, + pub width: u32, + pub height: u32, +} + +/// One buffered access unit awaiting its duration (known when the next one arrives). +struct PendingSample { + data: Vec, + sync: bool, + dts: u64, +} + +/// Aggregate counters reported when the writer finishes. +#[derive(Clone, Copy, Debug, Default)] +pub struct Mp4Stats { + pub samples: u64, + pub sync_samples: u64, + pub bytes: u64, + pub duration_us: u64, +} + +/// Codec-agnostic fMP4 fragment writer: `ftyp`+`moov` once, then one `moof`+`mdat` pair +/// per sample, each anchored at its wall-clock decode time via `tfdt`. +pub struct FragmentWriter { + out: W, + seq: u32, + wrote_init: bool, + pending: Option, + last_dts: Option, + /// Every flushed inter-frame duration, kept so the final buffered sample (whose + /// successor never arrives) can close on the MEDIAN — damage-driven capture makes the + /// last observed delta an outlier as often as not (a long static gap, a burst pair). + durations: Vec, + stats: Mp4Stats, +} + +impl FragmentWriter { + pub fn new(out: W) -> Self { + Self { + out, + seq: 0, + wrote_init: false, + pending: None, + last_dts: None, + durations: Vec::new(), + stats: Mp4Stats::default(), + } + } + + pub fn init_written(&self) -> bool { + self.wrote_init + } + + /// Counters for the fragments written so far (the buffered pending sample is not yet + /// included; `finish` folds it in). + pub fn stats(&self) -> Mp4Stats { + self.stats + } + + /// Write `ftyp` + `moov` (track 1, `mvex`/`trex` marking the movie fragmented). Must be + /// called once, before the first sample. + pub fn write_init(&mut self, cfg: &TrackConfig) -> std::io::Result<()> { + let mut ftyp_p = Vec::new(); + ftyp_p.extend_from_slice(b"isom"); + ftyp_p.extend_from_slice(&0x200u32.to_be_bytes()); + for brand in [b"isom", b"iso5", b"iso6", b"avc1", b"mp41"] { + ftyp_p.extend_from_slice(brand); + } + let ftyp = mk_box(b"ftyp", &ftyp_p); + + let mut mvhd_p = Vec::new(); + mvhd_p.extend_from_slice(&[0u8; 8]); // creation/modification time + mvhd_p.extend_from_slice(&TIMESCALE.to_be_bytes()); + mvhd_p.extend_from_slice(&0u32.to_be_bytes()); // duration: unknown (fragmented) + mvhd_p.extend_from_slice(&0x00010000u32.to_be_bytes()); // rate 1.0 + mvhd_p.extend_from_slice(&0x0100u16.to_be_bytes()); // volume 1.0 + mvhd_p.extend_from_slice(&[0u8; 10]); // reserved + mvhd_p.extend_from_slice(&MATRIX_IDENTITY); + mvhd_p.extend_from_slice(&[0u8; 24]); // pre_defined + mvhd_p.extend_from_slice(&2u32.to_be_bytes()); // next_track_ID + let mvhd = mk_full_box(b"mvhd", 0, 0, &mvhd_p); + + let mut tkhd_p = Vec::new(); + tkhd_p.extend_from_slice(&[0u8; 8]); + tkhd_p.extend_from_slice(&1u32.to_be_bytes()); // track_ID + tkhd_p.extend_from_slice(&[0u8; 4]); // reserved + tkhd_p.extend_from_slice(&0u32.to_be_bytes()); // duration + tkhd_p.extend_from_slice(&[0u8; 16]); // reserved, layer, group, volume, reserved + tkhd_p.extend_from_slice(&MATRIX_IDENTITY); + tkhd_p.extend_from_slice(&(cfg.width << 16).to_be_bytes()); + tkhd_p.extend_from_slice(&(cfg.height << 16).to_be_bytes()); + let tkhd = mk_full_box(b"tkhd", 0, 3, &tkhd_p); // enabled | in_movie + + let mut mdhd_p = Vec::new(); + mdhd_p.extend_from_slice(&[0u8; 8]); + mdhd_p.extend_from_slice(&TIMESCALE.to_be_bytes()); + mdhd_p.extend_from_slice(&0u32.to_be_bytes()); + mdhd_p.extend_from_slice(&0x55c4u16.to_be_bytes()); // language: und + mdhd_p.extend_from_slice(&[0u8; 2]); + let mdhd = mk_full_box(b"mdhd", 0, 0, &mdhd_p); + + let mut hdlr_p = Vec::new(); + hdlr_p.extend_from_slice(&[0u8; 4]); // pre_defined + hdlr_p.extend_from_slice(b"vide"); + hdlr_p.extend_from_slice(&[0u8; 12]); + hdlr_p.extend_from_slice(b"pixelflux\0"); + let hdlr = mk_full_box(b"hdlr", 0, 0, &hdlr_p); + + let vmhd = mk_full_box(b"vmhd", 0, 1, &[0u8; 8]); + let url = mk_full_box(b"url ", 0, 1, &[]); // self-contained + let mut dref_p = 1u32.to_be_bytes().to_vec(); + dref_p.extend_from_slice(&url); + let dref = mk_full_box(b"dref", 0, 0, &dref_p); + let dinf = mk_box(b"dinf", &dref); + + let mut stsd_p = 1u32.to_be_bytes().to_vec(); + stsd_p.extend_from_slice(&cfg.sample_entry); + let stsd = mk_full_box(b"stsd", 0, 0, &stsd_p); + let stts = mk_full_box(b"stts", 0, 0, &0u32.to_be_bytes()); + let stsc = mk_full_box(b"stsc", 0, 0, &0u32.to_be_bytes()); + let stsz = mk_full_box(b"stsz", 0, 0, &[0u8; 8]); + let stco = mk_full_box(b"stco", 0, 0, &0u32.to_be_bytes()); + let stbl = mk_box( + b"stbl", + &[stsd, stts, stsc, stsz, stco].concat(), + ); + + let minf = mk_box(b"minf", &[vmhd, dinf, stbl].concat()); + let mdia = mk_box(b"mdia", &[mdhd, hdlr, minf].concat()); + let trak = mk_box(b"trak", &[tkhd, mdia].concat()); + + let mut trex_p = Vec::new(); + trex_p.extend_from_slice(&1u32.to_be_bytes()); // track_ID + trex_p.extend_from_slice(&1u32.to_be_bytes()); // default_sample_description_index + trex_p.extend_from_slice(&[0u8; 12]); // default duration/size/flags + let trex = mk_full_box(b"trex", 0, 0, &trex_p); + let mvex = mk_box(b"mvex", &trex); + + let moov = mk_box(b"moov", &[mvhd, trak, mvex].concat()); + + self.out.write_all(&ftyp)?; + self.out.write_all(&moov)?; + self.stats.bytes += (ftyp.len() + moov.len()) as u64; + self.wrote_init = true; + Ok(()) + } + + /// Queue one sample at `pts_us` (wall-clock microseconds since recording start), + /// flushing the previously buffered sample with its now-known duration. Timestamps are + /// clamped strictly monotonic so a repeated or reordered clock can never emit a + /// zero/negative duration. + pub fn push_sample(&mut self, data: Vec, sync: bool, pts_us: u64) -> std::io::Result<()> { + let mut dts = pts_us * (TIMESCALE as u64 / 1000) / 1000; + if let Some(last) = self.last_dts { + if dts <= last { + dts = last + 1; + } + } + self.last_dts = Some(dts); + if let Some(prev) = self.pending.take() { + let duration = (dts - prev.dts).min(u32::MAX as u64) as u32; + self.durations.push(duration); + self.write_fragment(&prev, duration)?; + } + self.pending = Some(PendingSample { data, sync, dts }); + Ok(()) + } + + /// Median of the observed inter-frame durations (default 1/30 s when fewer than two + /// frames were pushed), used to close the final sample. + fn median_duration(&self) -> u32 { + if self.durations.is_empty() { + return DEFAULT_LAST_DURATION; + } + let mut sorted = self.durations.clone(); + sorted.sort_unstable(); + sorted[sorted.len() / 2] + } + + fn write_fragment(&mut self, s: &PendingSample, duration: u32) -> std::io::Result<()> { + self.seq += 1; + + let mut tfhd_p = Vec::new(); + tfhd_p.extend_from_slice(&1u32.to_be_bytes()); // track_ID + let tfhd = mk_full_box(b"tfhd", 0, 0x020000, &tfhd_p); // default-base-is-moof + + let mut tfdt_p = Vec::new(); + tfdt_p.extend_from_slice(&s.dts.to_be_bytes()); + let tfdt = mk_full_box(b"tfdt", 1, 0, &tfdt_p); + + // sample flags: sync = "depends on nothing"; non-sync also sets the non-sync bit. + let sample_flags: u32 = if s.sync { 0x0200_0000 } else { 0x0101_0000 }; + let mut trun_p = Vec::new(); + trun_p.extend_from_slice(&1u32.to_be_bytes()); // sample_count + trun_p.extend_from_slice(&0i32.to_be_bytes()); // data_offset placeholder + trun_p.extend_from_slice(&duration.to_be_bytes()); + trun_p.extend_from_slice(&(s.data.len() as u32).to_be_bytes()); + trun_p.extend_from_slice(&sample_flags.to_be_bytes()); + // data-offset | sample-duration | sample-size | sample-flags + let mut trun = mk_full_box(b"trun", 0, 0x000701, &trun_p); + + let traf_len = 8 + tfhd.len() + tfdt.len() + trun.len(); + let moof_len = 8 + 16 + traf_len; // mfhd is 16 bytes + // First sample byte sits just past the mdat header, relative to moof start. + let data_offset = (moof_len + 8) as i32; + let off_pos = trun.len() - trun_p.len() + 4; // into trun payload: after sample_count + trun[off_pos..off_pos + 4].copy_from_slice(&data_offset.to_be_bytes()); + + let mfhd = mk_full_box(b"mfhd", 0, 0, &self.seq.to_be_bytes()); + let traf = mk_box(b"traf", &[tfhd, tfdt, trun].concat()); + let moof = mk_box(b"moof", &[mfhd, traf].concat()); + debug_assert_eq!(moof.len(), moof_len); + + self.out.write_all(&moof)?; + self.out.write_all(&((8 + s.data.len()) as u32).to_be_bytes())?; + self.out.write_all(b"mdat")?; + self.out.write_all(&s.data)?; + self.out.flush()?; + + self.stats.samples += 1; + if s.sync { + self.stats.sync_samples += 1; + } + self.stats.bytes += (moof.len() + 8 + s.data.len()) as u64; + self.stats.duration_us = (s.dts + duration as u64) * 1000 / (TIMESCALE as u64 / 1000); + Ok(()) + } + + /// Flush the final buffered sample, closed with the MEDIAN observed inter-frame + /// duration (its successor never arrives), and return the aggregate counters. + pub fn finish(mut self) -> std::io::Result { + if let Some(prev) = self.pending.take() { + let d = self.median_duration(); + self.write_fragment(&prev, d)?; + } + self.out.flush()?; + Ok(self.stats) + } +} + +/// H.264-specific front end: captures SPS/PPS from the stream, gates output on the first +/// IDR, converts Annex-B access units to AVCC samples, and builds the `avc1` sample entry. +pub struct H264SampleBuilder { + sps: Option>, + pps: Option>, +} + +/// One converted access unit ready for the fragment writer. +pub struct BuiltSample { + pub data: Vec, + pub sync: bool, +} + +impl Default for H264SampleBuilder { + fn default() -> Self { + Self::new() + } +} + +impl H264SampleBuilder { + pub fn new() -> Self { + Self { sps: None, pps: None } + } + + pub fn have_parameter_sets(&self) -> bool { + self.sps.is_some() && self.pps.is_some() + } + + /// Convert one Annex-B access unit into a length-prefixed AVCC sample, harvesting + /// SPS/PPS on the way. Returns `None` for an AU with no slice data (e.g. bare parameter + /// sets). `sync` is true when the AU contains an IDR slice. + pub fn build_sample(&mut self, annexb: &[u8]) -> Option { + let nals = split_annexb(annexb); + let mut data = Vec::with_capacity(annexb.len() + 8); + let mut sync = false; + let mut has_slice = false; + for nal in nals { + if nal.is_empty() { + continue; + } + match nal[0] & 0x1f { + 7 => { + if self.sps.as_deref() != Some(nal) { + self.sps = Some(nal.to_vec()); + } + } + 8 => { + if self.pps.as_deref() != Some(nal) { + self.pps = Some(nal.to_vec()); + } + } + 5 => { + sync = true; + has_slice = true; + } + 1 => has_slice = true, + _ => {} + } + data.extend_from_slice(&(nal.len() as u32).to_be_bytes()); + data.extend_from_slice(nal); + } + if !has_slice { + return None; + } + Some(BuiltSample { data, sync }) + } + + /// Build the `avc1` sample entry + `avcC` record from the captured parameter sets, with + /// the display dimensions parsed from the SPS. + pub fn track_config(&self) -> Option { + let sps = self.sps.as_deref()?; + let pps = self.pps.as_deref()?; + let (width, height) = parse_sps_dimensions(sps)?; + + let mut avcc_p = Vec::new(); + avcc_p.push(1); // configurationVersion + avcc_p.push(sps[1]); // AVCProfileIndication + avcc_p.push(sps[2]); // profile_compatibility + avcc_p.push(sps[3]); // AVCLevelIndication + avcc_p.push(0xff); // lengthSizeMinusOne = 3 + avcc_p.push(0xe1); // numOfSequenceParameterSets = 1 + avcc_p.extend_from_slice(&(sps.len() as u16).to_be_bytes()); + avcc_p.extend_from_slice(sps); + avcc_p.push(1); // numOfPictureParameterSets + avcc_p.extend_from_slice(&(pps.len() as u16).to_be_bytes()); + avcc_p.extend_from_slice(pps); + let avcc = mk_box(b"avcC", &avcc_p); + + let mut entry_p = Vec::new(); + entry_p.extend_from_slice(&[0u8; 6]); // reserved + entry_p.extend_from_slice(&1u16.to_be_bytes()); // data_reference_index + entry_p.extend_from_slice(&[0u8; 16]); // pre_defined / reserved + entry_p.extend_from_slice(&(width as u16).to_be_bytes()); + entry_p.extend_from_slice(&(height as u16).to_be_bytes()); + entry_p.extend_from_slice(&0x0048_0000u32.to_be_bytes()); // horizresolution 72 dpi + entry_p.extend_from_slice(&0x0048_0000u32.to_be_bytes()); // vertresolution + entry_p.extend_from_slice(&[0u8; 4]); // reserved + entry_p.extend_from_slice(&1u16.to_be_bytes()); // frame_count + entry_p.extend_from_slice(&[0u8; 32]); // compressorname + entry_p.extend_from_slice(&0x0018u16.to_be_bytes()); // depth 24 + entry_p.extend_from_slice(&(-1i16).to_be_bytes()); // pre_defined + entry_p.extend_from_slice(&avcc); + let entry = mk_box(b"avc1", &entry_p); + + Some(TrackConfig { sample_entry: entry, width, height }) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + // x264 SPS at 1284x722 (High 4:2:0: cropping in both axes on odd-macroblock dims). + const SPS_HIGH_1284X722: &str = "67640020acd9405105de788c0440000003004000000f03c60c6580"; + // x264 SPS at 640x360 (Constrained Baseline). + const SPS_BASE_640X360: &str = "6742c01ed900a02ff970110000030001000003003c0f162e48"; + // x264 SPS at 1920x1080 (High 4:4:4 Predictive: chroma_format_idc == 3 path). + const SPS_444_1920X1080: &str = "67f40028919b280f0044fc4e0220000003002000000781e30632c0"; + + fn hex(s: &str) -> Vec { + (0..s.len()) + .step_by(2) + .map(|i| u8::from_str_radix(&s[i..i + 2], 16).unwrap()) + .collect() + } + + /// Both start-code lengths delimit NALs; payloads come back exactly, with no framing. + #[test] + fn split_annexb_handles_mixed_start_codes() { + let mut data = Vec::new(); + data.extend_from_slice(&[0, 0, 0, 1, 0x67, 0xAA, 0xBB]); + data.extend_from_slice(&[0, 0, 1, 0x68, 0xCC]); + data.extend_from_slice(&[0, 0, 0, 1, 0x65, 0x00, 0x00, 0x03, 0x01, 0xDD]); + let nals = split_annexb(&data); + assert_eq!(nals.len(), 3); + assert_eq!(nals[0], &[0x67, 0xAA, 0xBB]); + assert_eq!(nals[1], &[0x68, 0xCC]); + assert_eq!(nals[2], &[0x65, 0x00, 0x00, 0x03, 0x01, 0xDD]); + } + + /// Dimensions from real x264 SPS across the profiles the project's encoders emit, + /// including the frame-cropping and 4:4:4 chroma paths. + #[test] + fn sps_dimensions_across_profiles() { + assert_eq!(parse_sps_dimensions(&hex(SPS_HIGH_1284X722)), Some((1284, 722))); + assert_eq!(parse_sps_dimensions(&hex(SPS_BASE_640X360)), Some((640, 360))); + assert_eq!(parse_sps_dimensions(&hex(SPS_444_1920X1080)), Some((1920, 1080))); + } + + /// Annex-B -> AVCC: every NAL is length-prefixed, IDR marks sync, parameter sets are + /// harvested, and a parameter-set-only AU yields no sample. + #[test] + fn annexb_to_avcc_sample() { + let sps = hex(SPS_BASE_640X360); + let mut au = Vec::new(); + au.extend_from_slice(&[0, 0, 0, 1]); + au.extend_from_slice(&sps); + au.extend_from_slice(&[0, 0, 0, 1, 0x68, 0xCE, 0x38, 0x80]); + au.extend_from_slice(&[0, 0, 0, 1, 0x65, 1, 2, 3, 4]); + + let mut b = H264SampleBuilder::new(); + assert!(b.build_sample(&[0u8, 0, 0, 1, 0x67, 0x42, 0xc0, 0x1e, 0xd9]).is_none()); + let s = b.build_sample(&au).expect("IDR AU builds a sample"); + assert!(s.sync); + assert!(b.have_parameter_sets()); + // Sample = 3 length-prefixed NALs, sizes preserved. + let mut off = 0usize; + let mut sizes = Vec::new(); + while off < s.data.len() { + let n = u32::from_be_bytes(s.data[off..off + 4].try_into().unwrap()) as usize; + sizes.push(n); + off += 4 + n; + } + assert_eq!(off, s.data.len()); + assert_eq!(sizes, vec![sps.len(), 4, 5]); + + let p = b.build_sample(&[0u8, 0, 0, 1, 0x41, 9, 9]).unwrap(); + assert!(!p.sync); + } + + /// Walk the top-level boxes of a finished two-sample stream: init once, then one + /// moof+mdat pair per sample, with sizes that exactly tile the buffer. + #[test] + fn fragment_stream_box_layout() { + let mut b = H264SampleBuilder::new(); + let mut au = Vec::new(); + au.extend_from_slice(&[0, 0, 0, 1]); + au.extend_from_slice(&hex(SPS_BASE_640X360)); + au.extend_from_slice(&[0, 0, 0, 1, 0x68, 0xCE, 0x38, 0x80]); + au.extend_from_slice(&[0, 0, 0, 1, 0x65, 1, 2, 3, 4]); + let s1 = b.build_sample(&au).unwrap(); + let s2 = b.build_sample(&[0u8, 0, 0, 1, 0x41, 5, 6, 7]).unwrap(); + + let mut buf = Vec::new(); + let mut w = FragmentWriter::new(&mut buf); + w.write_init(&b.track_config().unwrap()).unwrap(); + w.push_sample(s1.data, s1.sync, 0).unwrap(); + w.push_sample(s2.data, s2.sync, 33_000).unwrap(); + let stats = w.finish().unwrap(); + assert_eq!(stats.samples, 2); + assert_eq!(stats.sync_samples, 1); + assert_eq!(stats.bytes as usize, buf.len()); + + let mut kinds = Vec::new(); + let mut off = 0usize; + while off < buf.len() { + let size = u32::from_be_bytes(buf[off..off + 4].try_into().unwrap()) as usize; + kinds.push(buf[off + 4..off + 8].to_vec()); + assert!(size >= 8 && off + size <= buf.len()); + off += size; + } + assert_eq!(off, buf.len()); + let names: Vec<&str> = kinds.iter().map(|k| std::str::from_utf8(k).unwrap()).collect(); + assert_eq!(names, vec!["ftyp", "moov", "moof", "mdat", "moof", "mdat"]); + } + + /// Every trun sample_duration in stream order (the writer emits one sample per trun). + fn trun_durations(buf: &[u8]) -> Vec { + let mut out = Vec::new(); + let mut off = 0usize; + while off + 24 <= buf.len() { + if &buf[off + 4..off + 8] == b"trun" { + // [size][fourcc][ver+flags][sample_count][data_offset][duration] + out.push(u32::from_be_bytes(buf[off + 20..off + 24].try_into().unwrap())); + } + off += 1; + } + out + } + + /// Each flushed sample's duration is the pts delta to its successor, and the final + /// buffered sample closes with the MEDIAN observed duration — not the last delta, which + /// under damage-driven capture is an outlier as often as not. + #[test] + fn sample_durations_are_pts_deltas_with_median_tail() { + let mut buf = Vec::new(); + let mut w = FragmentWriter::new(&mut buf); + let cfg = { + let mut b = H264SampleBuilder::new(); + let mut au = vec![0, 0, 0, 1]; + au.extend_from_slice(&hex(SPS_BASE_640X360)); + au.extend_from_slice(&[0, 0, 0, 1, 0x68, 0xCE, 0x38, 0x80]); + au.extend_from_slice(&[0, 0, 0, 1, 0x65, 1]); + b.build_sample(&au); + b.track_config().unwrap() + }; + w.write_init(&cfg).unwrap(); + // 33 ms, 33 ms, then a 300 ms static gap before the final frame. + w.push_sample(vec![0, 0, 0, 1, 0x65], true, 0).unwrap(); + w.push_sample(vec![0, 0, 0, 1, 0x41], false, 33_000).unwrap(); + w.push_sample(vec![0, 0, 0, 1, 0x41], false, 66_000).unwrap(); + w.push_sample(vec![0, 0, 0, 1, 0x41], false, 366_000).unwrap(); + let stats = w.finish().unwrap(); + assert_eq!(stats.samples, 4); + // 90 kHz ticks: 33 ms = 2970. The tail closes at the median (2970), not 27000. + assert_eq!(trun_durations(&buf), vec![2970, 2970, 27_000, 2970]); + } + + /// A single-frame recording still closes with a sane nonzero duration. + #[test] + fn single_sample_uses_default_duration() { + let mut buf = Vec::new(); + let mut w = FragmentWriter::new(&mut buf); + let cfg = { + let mut b = H264SampleBuilder::new(); + let mut au = vec![0, 0, 0, 1]; + au.extend_from_slice(&hex(SPS_BASE_640X360)); + au.extend_from_slice(&[0, 0, 0, 1, 0x68, 0xCE, 0x38, 0x80]); + au.extend_from_slice(&[0, 0, 0, 1, 0x65, 1]); + b.build_sample(&au); + b.track_config().unwrap() + }; + w.write_init(&cfg).unwrap(); + w.push_sample(vec![0, 0, 0, 1, 0x65], true, 0).unwrap(); + let stats = w.finish().unwrap(); + assert_eq!(stats.samples, 1); + assert_eq!(trun_durations(&buf), vec![DEFAULT_LAST_DURATION]); + } + + /// Non-monotonic wall-clock input is clamped to strictly increasing decode times, so no + /// fragment can carry a zero or negative duration. + #[test] + fn pts_clamped_strictly_monotonic() { + let mut buf = Vec::new(); + let mut w = FragmentWriter::new(&mut buf); + let cfg = { + let mut b = H264SampleBuilder::new(); + let mut au = vec![0, 0, 0, 1]; + au.extend_from_slice(&hex(SPS_BASE_640X360)); + au.extend_from_slice(&[0, 0, 0, 1, 0x68, 0xCE, 0x38, 0x80]); + au.extend_from_slice(&[0, 0, 0, 1, 0x65, 1]); + b.build_sample(&au); + b.track_config().unwrap() + }; + w.write_init(&cfg).unwrap(); + w.push_sample(vec![0, 0, 0, 1, 0x65], true, 1000).unwrap(); + w.push_sample(vec![0, 0, 0, 1, 0x41], false, 1000).unwrap(); // repeated clock + w.push_sample(vec![0, 0, 0, 1, 0x41], false, 500).unwrap(); // clock went backwards + let stats = w.finish().unwrap(); + assert_eq!(stats.samples, 3); + + // Extract each tfdt baseMediaDecodeTime and check strict monotonicity. + let mut times = Vec::new(); + let mut off = 0usize; + while off + 8 <= buf.len() { + if &buf[off + 4..off + 8] == b"tfdt" { + let t = u64::from_be_bytes(buf[off + 12..off + 20].try_into().unwrap()); + times.push(t); + } + off += 1; + } + assert_eq!(times.len(), 3); + assert!(times.windows(2).all(|w| w[1] > w[0]), "tfdt times: {times:?}"); + } +} diff --git a/pixelflux/src/recording_sink.rs b/pixelflux/src/recording_sink.rs index f4ddf65..26579a0 100644 --- a/pixelflux/src/recording_sink.rs +++ b/pixelflux/src/recording_sink.rs @@ -211,6 +211,87 @@ impl Drop for RecordingSink { } } +#[cfg(test)] +mod cost_tests { + //! The sink's isolation contract, measured: feeding a frame must cost nothing when no + //! recorder is connected (empty-clients early return), microseconds when a healthy + //! recorder drains its queue, and stay bounded (a lock + `try_send`, never a blocking + //! write) when a recorder stalls completely — until the bounded queue overflows and the + //! client is dropped, returning the tap to idle cost. + + use super::*; + use std::io::Read; + use std::os::unix::net::UnixStream; + use std::time::Instant; + + fn frame(len: usize) -> EncodedStripe { + let mut data = vec![0u8; len]; + data[0] = 0x04; // wire-header tag so the 10-byte strip path runs + EncodedStripe { + data: Arc::new(data), + data_type: 2, + stripe_y_start: 0, + stripe_height: 720, + frame_id: 0, + } + } + + fn feed_timed(sink: &RecordingSink, n: usize, len: usize) -> (f64, f64) { + let f = frame(len); + let mut max_us = 0f64; + let mut total_us = 0f64; + for _ in 0..n { + let t = Instant::now(); + sink.write_encoded_frame(&f); + let us = t.elapsed().as_secs_f64() * 1e6; + total_us += us; + max_us = max_us.max(us); + thread::sleep(Duration::from_micros(200)); + } + (total_us / n as f64, max_us) + } + + #[test] + fn stalled_recorder_isolation_cost() { + let path = format!("/tmp/pf-sink-cost-{}.sock", std::process::id()); + let sink = RecordingSink::try_bind(&path).expect("bind"); + + // Idle: no client connected. + let (idle_mean, idle_max) = feed_timed(&sink, 500, 100_000); + + // Healthy: a client draining as fast as it can. + let mut healthy = UnixStream::connect(&path).expect("connect"); + thread::sleep(Duration::from_millis(200)); + let drain = thread::spawn(move || { + let mut buf = vec![0u8; 1 << 20]; + while healthy.read(&mut buf).map(|n| n > 0).unwrap_or(false) {} + }); + let (healthy_mean, healthy_max) = feed_timed(&sink, 500, 100_000); + + // Stalled: a connected client that never reads. The socket buffer fills, then the + // bounded queue fills, then the client is dropped (~256 frames later). + let stalled = UnixStream::connect(&path).expect("connect"); + thread::sleep(Duration::from_millis(200)); + let (stalled_mean, stalled_max) = feed_timed(&sink, 500, 100_000); + drop(stalled); + + println!( + "[sink-cost] idle mean {idle_mean:.3}us max {idle_max:.3}us\n\ + [sink-cost] healthy mean {healthy_mean:.3}us max {healthy_max:.3}us\n\ + [sink-cost] stalled mean {stalled_mean:.3}us max {stalled_max:.3}us" + ); + drop(sink); + let _ = drain.join(); + + assert!(idle_mean < 5.0, "idle feed should be sub-5us, was {idle_mean:.3}us"); + assert!(healthy_mean < 100.0, "healthy feed should be tens of us, was {healthy_mean:.3}us"); + assert!( + stalled_max < 10_000.0, + "a stalled recorder must never block the tap >10ms, was {stalled_max:.3}us" + ); + } +} + /// Write one whole frame to a recorder's socket, resuming across the soft timeouts a slow reader /// induces so a partial Annex-B NAL is never left behind. Aborts if `stop` is set (the client was /// dropped by [`RecordingSink::write_encoded_frame`]) or a hard error occurs. diff --git a/pixelflux/src/wayland/cursor.rs b/pixelflux/src/wayland/cursor.rs index f5e8086..e442603 100644 --- a/pixelflux/src/wayland/cursor.rs +++ b/pixelflux/src/wayland/cursor.rs @@ -9,6 +9,9 @@ //! and render the appropriate frame at the current scale. use image::{ImageBuffer, Rgba}; +use pyo3::prelude::*; +use pyo3::types::PyBytes; +use std::collections::HashMap; use std::io::Cursor as IoCursor; use std::io::Read; use std::time::Duration; @@ -17,6 +20,213 @@ use xcursor::{ CursorTheme, }; +/// One unit of cursor-callback work handed from the calloop thread to the `wl-cursor` +/// worker. The calloop resolves everything renderer/surface-affine (SHM copies, dmabuf +/// readbacks, hotspots); the worker does the PNG encode, the cache, and the GIL-bound Python +/// call so none of that latency lands on the input/render thread. One channel keeps cursor +/// updates ordered with callback (re)registration. +pub enum CursorJob { + SetCallback(Py), + /// Reload the worker's theme handle at a new pixel size; later `Named` jobs render at it. + SetSize(i32), + Named { name: &'static str }, + Hide, + /// wl_shm cursor sprite: raw pool bytes plus the sub-image descriptor. + Shm { + hash: u64, + width: i32, + height: i32, + stride: i32, + offset: i32, + opaque: bool, + bytes: Vec, + hot_x: i32, + hot_y: i32, + }, + /// dmabuf cursor sprite already read back to tightly-mapped RGBA on the calloop. + Gles { + hash: u64, + width: i32, + height: i32, + bytes: Vec, + hot_x: i32, + hot_y: i32, + }, +} + +/// Spawn the cursor delivery worker; returns its job channel. The worker owns its own +/// theme handle, the PNG cache, and the Python callback for the life of the process (like the +/// compositor thread itself); `PY_SHUTDOWN` gates every Python call. +pub fn spawn_cursor_worker(cursor_size: i32) -> std::sync::mpsc::Sender { + let (tx, rx) = std::sync::mpsc::channel::(); + let _ = std::thread::Builder::new().name("wl-cursor".into()).spawn(move || { + let mut helper = Cursor::load(cursor_size); + let mut cache: HashMap> = HashMap::new(); + let mut callback: Option> = None; + while let Ok(job) = rx.recv() { + let (msg_type, data, hot_x, hot_y): (&str, Vec, i32, i32) = match job { + CursorJob::SetCallback(cb) => { + callback = Some(cb); + continue; + } + CursorJob::SetSize(size) => { + helper = Cursor::load(size); + continue; + } + CursorJob::Named { name } => match helper.get_png_data(name) { + Some((png, x, y)) => ("png", png, x as i32, y as i32), + None => ("error", Vec::new(), 0, 0), + }, + CursorJob::Hide => ("hide", Vec::new(), 0, 0), + CursorJob::Shm { + hash, + width, + height, + stride, + offset, + opaque, + bytes, + hot_x, + hot_y, + } => { + let png = cached_png(&mut cache, hash, || { + encode_shm_cursor(width, height, stride, offset, opaque, &bytes) + }); + ("png", png, hot_x, hot_y) + } + CursorJob::Gles { hash, width, height, bytes, hot_x, hot_y } => { + let png = cached_png(&mut cache, hash, || { + encode_gles_cursor(width, height, &bytes) + }); + ("png", png, hot_x, hot_y) + } + }; + // A sprite whose pixels could not be read yields empty data; suppressing it + // preserves the consumer's last cursor instead of blanking it (only an + // intentional hide passes with no payload). + if data.is_empty() && msg_type != "hide" { + continue; + } + if crate::PY_SHUTDOWN.load(std::sync::atomic::Ordering::Relaxed) { + continue; + } + if let Some(ref cb) = callback { + Python::attach(|py| { + let py_bytes = PyBytes::new(py, &data); + let _ = cb.call1(py, (msg_type, py_bytes, hot_x, hot_y)); + }); + } + } + }); + tx +} + +/// Content-hash PNG cache lookup with bounded arbitrary eviction; an evicted sprite simply +/// re-encodes on its next appearance. +fn cached_png( + cache: &mut HashMap>, + hash: u64, + encode: impl FnOnce() -> Vec, +) -> Vec { + if let Some(png) = cache.get(&hash) { + return png.clone(); + } + let png = encode(); + if !png.is_empty() { + cache.insert(hash, png.clone()); + if cache.len() > 100 { + if let Some(&evict) = cache.keys().next() { + cache.remove(&evict); + } + } + } + png +} + +/// Convert a wl_shm BGRA/XRGB sprite sub-image to a straight-alpha PNG. Stride/offset are +/// clamped non-negative with checked arithmetic so a garbage descriptor skips pixels instead of +/// panicking; sprites larger than 128x128 are ignored (never a hardware cursor). +fn encode_shm_cursor( + width: i32, + height: i32, + stride: i32, + offset: i32, + opaque: bool, + raw_bytes: &[u8], +) -> Vec { + if width <= 0 || height <= 0 || width > 128 || height > 128 || raw_bytes.is_empty() { + return Vec::new(); + } + let mut img_buf = ImageBuffer::, Vec>::new(width as u32, height as u32); + let stride_usize = stride.max(0) as usize; + let base_offset = offset.max(0) as usize; + for y in 0..(height as u32) { + for x in 0..(width as u32) { + let offset = (y as usize) + .checked_mul(stride_usize) + .and_then(|row| base_offset.checked_add(row)) + .and_then(|o| o.checked_add((x as usize) * 4)); + let offset = match offset { + Some(o) => o, + None => continue, + }; + if offset.checked_add(4).is_some_and(|end| end <= raw_bytes.len()) { + let alpha = if opaque { 255 } else { raw_bytes[offset + 3] }; + img_buf.put_pixel( + x, + y, + Rgba([raw_bytes[offset + 2], raw_bytes[offset + 1], raw_bytes[offset], alpha]), + ); + } + } + } + crate::unpremultiply_rgba(&mut img_buf); + let mut bytes = Vec::new(); + if img_buf.write_to(&mut IoCursor::new(&mut bytes), image::ImageFormat::Png).is_ok() { + bytes + } else { + Vec::new() + } +} + +/// Convert a dmabuf sprite's RGBA readback (stride recovered from the mapping length) to a +/// straight-alpha PNG. +fn encode_gles_cursor(width: i32, height: i32, raw_bytes: &[u8]) -> Vec { + if width <= 0 || height <= 0 || width > 128 || height > 128 || raw_bytes.is_empty() { + return Vec::new(); + } + let stride = super::frontend::rgba_readback_stride( + raw_bytes.len(), + height as usize, + width as usize, + ); + let mut img_buf = ImageBuffer::, Vec>::new(width as u32, height as u32); + for y in 0..(height as u32) { + for x in 0..(width as u32) { + let offset = (y as usize * stride) + (x as usize * 4); + if offset + 4 <= raw_bytes.len() { + img_buf.put_pixel( + x, + y, + Rgba([ + raw_bytes[offset], + raw_bytes[offset + 1], + raw_bytes[offset + 2], + raw_bytes[offset + 3], + ]), + ); + } + } + } + crate::unpremultiply_rgba(&mut img_buf); + let mut bytes = Vec::new(); + if img_buf.write_to(&mut IoCursor::new(&mut bytes), image::ImageFormat::Png).is_ok() { + bytes + } else { + Vec::new() + } +} + /// The loaded XCursor theme, held for the whole capture so cursor lookups stay cheap. /// /// The default cursor's frames are parsed once and kept here because they are consulted on nearly diff --git a/pixelflux/src/wayland/frontend.rs b/pixelflux/src/wayland/frontend.rs index fd93224..743a673 100644 --- a/pixelflux/src/wayland/frontend.rs +++ b/pixelflux/src/wayland/frontend.rs @@ -20,11 +20,9 @@ use std::borrow::Cow; use std::fs::File; -use std::io::Cursor as IoCursor; use std::time::Instant; use gbm::{BufferObject, Device as RawGbmDevice}; -use image::{ImageBuffer, ImageFormat, Rgba}; use pyo3::prelude::*; use pyo3::types::PyBytes; use std::sync::Mutex; @@ -33,12 +31,19 @@ use std::hash::{Hash, Hasher}; use smithay::backend::renderer::utils::RendererSurfaceState; use smithay::backend::allocator::dmabuf::Dmabuf; use smithay::backend::allocator::{ Buffer, Fourcc}; +use smithay::backend::renderer::damage::OutputDamageTracker; use smithay::backend::renderer::{ Bind, ExportMem, gles::GlesRenderer, pixman::PixmanRenderer, ImportDma, }; use smithay::input::dnd::{DndFocus, Source}; use std::sync::Arc; -use crate::wayland::cursor::Cursor; +use crate::wayland::cursor::{Cursor, CursorJob}; +use crate::wayland::keymap::KeymapPolicy; +use smithay::reexports::wayland_protocols_misc::zwp_virtual_keyboard_v1::server::{ + zwp_virtual_keyboard_manager_v1::{self, ZwpVirtualKeyboardManagerV1}, + zwp_virtual_keyboard_v1::{self, ZwpVirtualKeyboardV1}, +}; +use smithay::reexports::wayland_server::{DataInit, Dispatch, GlobalDispatch, New}; use smithay::wayland::viewporter::ViewporterState; use smithay::delegate_viewporter; use smithay::wayland::pointer_warp::{PointerWarpHandler, PointerWarpManager}; @@ -78,8 +83,8 @@ use smithay::delegate_primary_selection; use smithay::{ delegate_compositor, delegate_data_device, delegate_dmabuf, delegate_fractional_scale, - delegate_output, delegate_seat, delegate_shm, delegate_virtual_keyboard_manager, - delegate_xdg_shell, delegate_relative_pointer, delegate_pointer_warp, + delegate_output, delegate_seat, delegate_shm, + delegate_xdg_shell, delegate_relative_pointer, delegate_pointer_warp, delegate_pointer_constraints, desktop::{Space, Window}, input::{ @@ -97,7 +102,7 @@ use smithay::{ reexports::{ wayland_protocols::xdg::shell::server::xdg_toplevel::State as XdgState, wayland_server::{ - backend::{ClientData, ClientId, DisconnectReason, ObjectId}, + backend::{ClientData, ClientId, DisconnectReason, GlobalId, ObjectId}, protocol::{wl_buffer::WlBuffer, wl_surface::WlSurface}, Client, DisplayHandle, Resource, }, @@ -115,8 +120,8 @@ use smithay::{ seat::WaylandFocus, selection::{ data_device::{ - request_data_device_client_selection, DataDeviceHandler, DataDeviceState, - WaylandDndGrabHandler, + request_data_device_client_selection, set_data_device_focus, DataDeviceHandler, + DataDeviceState, WaylandDndGrabHandler, }, SelectionHandler, SelectionSource, SelectionTarget, }, @@ -125,7 +130,6 @@ use smithay::{ XdgToplevelSurfaceData, }, shm::{with_buffer_contents, ShmHandler, ShmState, BufferAccessError}, - virtual_keyboard::VirtualKeyboardManagerState, }, }; @@ -193,14 +197,110 @@ pub enum GpuEncoder { Nvenc(NvencEncoder), } +/// One capture pipeline bound to one output (display id): its settings, encoder set, +/// frame pools, delivery thread, and per-stream bookkeeping. Exactly one capture may run per +/// output; all fields mirror the pipeline strategy documented on [`AppState`], instantiated +/// per display. +pub struct WlCapture { + pub settings: RustCaptureSettings, + /// Zero-copy GPU session (GLES render + same-GPU dmabuf encode), calloop-affine; + /// `None` whenever a readback path is active for this display. + pub video_encoder: Option, + pub vaapi_state: StripeState, + pub recording_sink: Option>, + pub deliver_tx: Option>>, + pub deliver_join: Option>, + pub pending_hw_delivery: Option>, + pub pending_hw_damage: bool, + pub encode_pool: Option>, + pub encode_join: Option>>, + pub encode_controls: Arc, + pub encode_stats: Arc, + pub pool_last_render: Vec, + pub render_seq: u64, + pub pool_content_gen: Vec, + pub content_gen: u64, + pub frame_counter: u16, + pub pending_force_idr: bool, + pub needs_full_render: bool, + /// Last tick this capture actually rendered; paces per-display fps under the one + /// shared render timer (which fires at the fastest active capture's rate). + pub last_tick: Option, +} + +/// One virtual output and everything sized to it: the Smithay `Output` + its advertised +/// global, its layout position, the damage tracker, render targets, and the (at most one) +/// capture bound to it. `id` is the Python-facing display key; id 0 is the primary +/// HEADLESS-1 output, which is never destroyed. +pub struct OutputNode { + pub id: u32, + pub output: Output, + pub global: GlobalId, + /// Layout offset: the output's logical position in the Space AND its physical offset + /// for absolute input injection. The two coincide at scale 1; with mixed scales the + /// caller must place outputs so neither the logical nor the physical rectangles + /// overlap. + pub pos: (i32, i32), + pub damage_tracker: OutputDamageTracker, + /// Host-side scratch target: pixman throttle path renders here, GLES screenshots read + /// back here. + pub frame_buffer: Vec, + /// GPU render target for this output (GLES mode): the GBM BO and its dmabuf export. + pub offscreen_buffer: Option<(BufferObject<()>, Dmabuf)>, + /// Watermark overlay for THIS output: loaded at the output's scale, positioned (and, + /// for the bouncing anchor, animated) against the output's own frame dimensions. + pub overlay_state: OverlayState, + pub capture: Option, +} + +impl OutputNode { + /// The output's logical geometry in layout coordinates: origin plus mode/scale-derived + /// logical size. + pub fn logical_geometry(&self) -> Option> { + let mode = self.output.current_mode()?; + let scale = self.output.current_scale().fractional_scale(); + Some(Rectangle::new( + Point::from(self.pos), + ( + (mode.size.w as f64 / scale).round() as i32, + (mode.size.h as f64 / scale).round() as i32, + ) + .into(), + )) + } +} + +static NEXT_WINDOW_ID: AtomicU32 = AtomicU32::new(1); + +/// Per-window bookkeeping carried in the window's user-data map: a stable numeric id the +/// Python side addresses the window by, and the display id of the output it is placed on. +pub struct WindowMeta { + pub id: u32, + pub output: AtomicU32, +} + +/// The window's meta, inserted at `new_toplevel`; windows created before that (none in +/// practice) read as id 0 / primary output. +pub fn window_meta(window: &Window) -> Option<&WindowMeta> { + window.user_data().get::() +} + +/// The display id of the output this window is placed on (primary when untagged). +pub fn window_output_id(window: &Window) -> u32 { + window_meta(window).map(|m| m.output.load(Ordering::Relaxed)).unwrap_or(0) +} + /// Central context threaded through every Smithay handler; owns the Wayland globals, the /// GBM/EGL (or pixman) renderer state, and the capture/encode pipeline state. /// /// A single `AppState` lives for the whole compositor thread and is mutated in place by the -/// calloop event sources (client dispatch, input injection, the capture timer). The non-obvious -/// fields encode the pipeline's threading and buffer strategy: +/// calloop event sources (client dispatch, input injection, the capture timer). The frame +/// pipeline is instantiated PER DISPLAY: each `OutputNode` in `output_nodes` owns its output, +/// damage tracker, and render targets, and its optional `WlCapture` owns that display's +/// encoder set and delivery. The non-obvious `WlCapture`/`OutputNode` fields encode the +/// pipeline's threading and buffer strategy: /// -/// 1. **Render / encode targets**: +/// 1. **Render / encode targets** (per display): /// - **`video_encoder`**: the zero-copy GPU session only (GLES render + same-GPU dmabuf /// encode). Its EGL/dmabuf handles are calloop-affine, so this encoder runs inline on the /// calloop thread; it is `None` whenever a readback path is active. @@ -255,7 +355,9 @@ pub struct AppState { pub dh: DisplayHandle, #[allow(dead_code)] pub seat: Seat, - pub outputs: Vec, + /// Every live output with its per-display render/capture state; index 0 is the + /// primary (display id 0), which is never destroyed. + pub output_nodes: Vec, pub pending_windows: Vec, pub foreign_toplevel_list: ForeignToplevelListState, @@ -263,69 +365,54 @@ pub struct AppState { pub xdg_activation_state: XdgActivationState, pub primary_selection_state: PrimarySelectionState, pub popups: PopupManager, - pub frame_buffer: Vec, pub gles_renderer: Option, pub pixman_renderer: Option, pub gbm_device: Option>, - pub offscreen_buffer: Option<(BufferObject<()>, Dmabuf)>, - pub is_capturing: bool, + /// Mirror of the PRIMARY display's capture settings (geometry fallbacks, computer-use + /// info); per-display settings live on each capture. pub settings: RustCaptureSettings, - pub callback: Option>, - pub cursor_callback: Option>, + /// A cursor callback is registered on the `wl-cursor` worker (which owns the actual + /// `Py` object); tracked here so sprite resolution is skipped while nobody listens. + pub cursor_callback_set: bool, + /// Cursor delivery jobs to the `wl-cursor` worker (PNG encode + Python call off-thread). + pub cursor_tx: std::sync::mpsc::Sender, pub clipboard_callback: Option>, pub pending_clipboard_read: Option, + /// Preferred mime of the current CLIENT-owned clipboard selection, recorded even while + /// no callback is registered so `SetClipboardCallback` can re-stage a read of a copy made + /// in the gap; `None` when the selection is cleared or compositor-owned. + pub current_selection_mime: Option, pub last_log_time: Instant, pub start_time: Instant, pub clock: Clock, - pub frame_counter: u16, - pub pending_force_idr: bool, - pub needs_full_render: bool, pub use_gpu: bool, - pub video_encoder: Option, - pub vaapi_state: StripeState, pub cursor_helper: Cursor, - pub overlay_state: OverlayState, - - pub virtual_keyboard_state: VirtualKeyboardManagerState, + /// Seat keymap owner: base layout plus batched overlay binds. Every seat keymap swap + /// flows through this policy, so keymap identity has exactly one writer. + pub keymap_policy: KeymapPolicy, pub current_cursor_icon: Option, pub cursor_buffer: Option, - pub cursor_cache: std::collections::HashMap>, pub render_cursor_on_framebuffer: bool, pub pointer_warp_state: PointerWarpManager, pub relative_pointer_state: RelativePointerManagerState, pub pointer_constraints_state: PointerConstraintsState, pub render_node_path: String, pub auto_gpu_selected: bool, - pub recording_sink: Option>, - pub deliver_tx: Option>>, - pub deliver_join: Option>, - /// Zero-copy frame parked after a full delivery channel. The hardware encode and - /// its delivery both run on the calloop thread, and a blocking send there would - /// freeze input/command/Wayland dispatch for as long as the Python consumer - /// stalls. An encoded frame is part of the H.264 reference chain and can never - /// be dropped, so it parks here instead, and new encodes pause until it leaves. - pub pending_hw_delivery: Option>, - /// Damage seen on ticks skipped while `pending_hw_delivery` was parked, folded - /// into the next encode's change detection so a change that happened during the - /// pause is never lost (each tick's damage list is otherwise discarded). - pub pending_hw_damage: bool, - pub encode_pool: Option>, - pub encode_join: Option>>, - pub encode_controls: Arc, - pub encode_stats: Arc, - pub pool_last_render: Vec, - pub render_seq: u64, - pub pool_content_gen: Vec, - pub content_gen: u64, - pub pending_screenshot: Option>>, + /// Computer-use screenshot request `(display id, reply)`; served from that output's + /// next render (the id was validated live when the request was queued). + pub pending_screenshot: Option<(u32, std::sync::mpsc::Sender, String>>)>, + /// The command channel, drained in place (wakeups arrive on a separate ping channel) so + /// the render tick can apply every queued command BEFORE starting a long render/encode — + /// queued input is never starved behind the tick it arrived during. + pub command_rx: Option>, } /// Pointer-constraints protocol wiring. The headless capture path never enforces a lock or @@ -440,9 +527,9 @@ impl WlrLayerShellHandler for AppState { namespace: String, ) { let smithay_output = if let Some(wlo) = output.as_ref() { - self.outputs.iter().find(|o| o.owns(wlo)) + self.output_nodes.iter().map(|n| &n.output).find(|o| o.owns(wlo)) } else { - self.outputs.first() + self.primary_output() }; if let Some(output) = smithay_output { @@ -505,17 +592,12 @@ impl CompositorHandler for AppState { fn commit(&mut self, surface: &WlSurface) { smithay::backend::renderer::utils::on_commit_buffer_handler::(surface); - if let Some(output) = self.outputs.first() { - let mut layer_map = layer_map_for_output(output); - let mut found = false; - for layer in layer_map.layers() { - if layer.wl_surface() == surface { - found = true; - break; - } - } + for node in &self.output_nodes { + let mut layer_map = layer_map_for_output(&node.output); + let found = layer_map.layers().any(|layer| layer.wl_surface() == surface); if found { layer_map.arrange(); + break; } } @@ -541,12 +623,26 @@ impl CompositorHandler for AppState { } } - if let Some(window) = self + let mapped = self .space .elements() .find(|w| w.toplevel().map(|tl| tl.wl_surface() == surface).unwrap_or(false)) - { + .cloned(); + if let Some(window) = mapped { window.on_commit(); + // A null-buffer commit unmaps the toplevel (xdg-shell): purge it from the + // space so it no longer lists or renders, and re-queue it so a client that + // maps again goes back through the configure handshake. + let has_buffer = smithay::backend::renderer::utils::with_renderer_surface_state( + surface, + |s| s.buffer().is_some(), + ) + .unwrap_or(false); + if !has_buffer { + self.space.unmap_elem(&window); + self.pending_windows.push(window); + return; + } } if let Some(idx) = self.pending_windows.iter().position(|w| { @@ -566,20 +662,23 @@ impl CompositorHandler for AppState { }); if !initial_configure_sent { - let (logical_width, logical_height) = if let Some(output) = self.outputs.first() { - let mode = output.current_mode().unwrap(); - let scale = output.current_scale().fractional_scale(); - ( - (mode.size.w as f64 / scale).round() as i32, - (mode.size.h as f64 / scale).round() as i32, - ) - } else { - let scale = self.settings.scale.max(0.1); - ( - (self.settings.width as f64 / scale).round() as i32, - (self.settings.height as f64 / scale).round() as i32, - ) - }; + // A new toplevel opens fullscreened on the output the pointer is on + // (primary when indeterminate); the choice is pinned on the window's + // meta so the acked commit maps to the same output. + let target_id = self.pointer_display(); + if let Some(meta) = window_meta(&window) { + meta.output.store(target_id, Ordering::Relaxed); + } + let (logical_width, logical_height) = + if let Some(size) = self.logical_size_of(target_id) { + size + } else { + let scale = self.settings.scale.max(0.1); + ( + (self.settings.width as f64 / scale).round() as i32, + (self.settings.height as f64 / scale).round() as i32, + ) + }; toplevel.with_pending_state(|state| { state.states.set(XdgState::Activated); @@ -589,28 +688,52 @@ impl CompositorHandler for AppState { toplevel.send_configure(); self.pending_windows.push(window); + } else if smithay::backend::renderer::utils::with_renderer_surface_state( + surface, + |s| s.buffer().is_none(), + ) + .unwrap_or(true) + { + // Configured but still buffer-less (a decoration-triggered configure, an + // ack-only commit, or a remap after a null-buffer unmap — xdg + // initial_configure_sent stays true there): keep waiting; mapping now would + // list and hit-test a phantom window. Answer with the forced-fullscreen + // configure so the client draws its first buffer at the right geometry. + let tl = toplevel.clone(); + self.pending_windows.push(window); + self.send_forced_fullscreen_configure(&tl); } else { - self.space.map_element(window.clone(), (0, 0), true); + let target_id = window_output_id(&window); + let node_idx = self.node_idx_for_id(target_id).unwrap_or(0); + let (target_output, pos) = { + let node = &self.output_nodes[node_idx]; + (node.output.clone(), node.pos) + }; + self.space.map_element(window.clone(), pos, true); window.on_commit(); - if let Some(output) = self.outputs.first() { - output.enter(surface); - - let mode = output.current_mode().unwrap(); - let scale = output.current_scale().fractional_scale(); - let (expected_w, expected_h) = ( - (mode.size.w as f64 / scale).round() as i32, - (mode.size.h as f64 / scale).round() as i32, - ); - - let geo = window.geometry(); - if (geo.size.w - expected_w).abs() > 1 || (geo.size.h - expected_h).abs() > 1 { - toplevel.with_pending_state(|state| { - state.states.set(XdgState::Activated); - state.states.set(XdgState::Fullscreen); - state.size = Some((expected_w, expected_h).into()); + { + target_output.enter(surface); + let scale = target_output.current_scale().fractional_scale(); + with_states(surface, |states| { + smithay::wayland::fractional_scale::with_fractional_scale(states, |fs| { + fs.set_preferred_scale(scale); }); - toplevel.send_configure(); + }); + + if let Some(geo_out) = self.output_nodes[node_idx].logical_geometry() { + let (expected_w, expected_h) = (geo_out.size.w, geo_out.size.h); + let geo = window.geometry(); + if (geo.size.w - expected_w).abs() > 1 + || (geo.size.h - expected_h).abs() > 1 + { + toplevel.with_pending_state(|state| { + state.states.set(XdgState::Activated); + state.states.set(XdgState::Fullscreen); + state.size = Some((expected_w, expected_h).into()); + }); + toplevel.send_configure(); + } } } @@ -626,18 +749,145 @@ impl CompositorHandler for AppState { impl AppState { + /// The primary (display id 0) output. + pub(crate) fn primary_output(&self) -> Option<&Output> { + self.output_nodes.first().map(|n| &n.output) + } + + pub(crate) fn node_idx_for_id(&self, id: u32) -> Option { + self.output_nodes.iter().position(|n| n.id == id) + } + + /// Index of the node whose LOGICAL rect contains `p`. + pub(crate) fn node_idx_under(&self, p: Point) -> Option { + self.output_nodes.iter().position(|n| { + n.logical_geometry() + .map(|g| g.to_f64().contains(p)) + .unwrap_or(false) + }) + } + + /// Map absolute PHYSICAL union-layout coordinates to a logical layout point: each + /// output occupies the physical rectangle at its layout offset sized by its mode; the + /// point is clamped into the nearest output when it falls outside all of them, so the + /// pointer can never leave the layout. + pub(crate) fn layout_physical_to_logical(&self, x: f64, y: f64) -> Point { + let mut best: Option<(f64, Point)> = None; + for node in &self.output_nodes { + let Some(mode) = node.output.current_mode() else { continue }; + let scale = node.output.current_scale().fractional_scale(); + let (px, py) = (node.pos.0 as f64, node.pos.1 as f64); + let cx = x.max(px).min(px + mode.size.w as f64 - 1.0); + let cy = y.max(py).min(py + mode.size.h as f64 - 1.0); + let d2 = (x - cx).powi(2) + (y - cy).powi(2); + let logical = Point::from(( + node.pos.0 as f64 + (cx - px) / scale, + node.pos.1 as f64 + (cy - py) / scale, + )); + if best.as_ref().map(|(bd, _)| d2 < *bd).unwrap_or(true) { + best = Some((d2, logical)); + } + } + best.map(|(_, p)| p).unwrap_or_else(|| (0.0, 0.0).into()) + } + + /// Clamp a logical layout point into the nearest output's logical rectangle. + pub(crate) fn clamp_logical(&self, p: Point) -> Point { + let mut best: Option<(f64, Point)> = None; + for node in &self.output_nodes { + let Some(geo) = node.logical_geometry() else { continue }; + let scale = node.output.current_scale().fractional_scale(); + let g = geo.to_f64(); + let margin = 1.0 / scale.max(0.1); + let cx = p.x.max(g.loc.x).min(g.loc.x + g.size.w - margin); + let cy = p.y.max(g.loc.y).min(g.loc.y + g.size.h - margin); + let d2 = (p.x - cx).powi(2) + (p.y - cy).powi(2); + if best.as_ref().map(|(bd, _)| d2 < *bd).unwrap_or(true) { + best = Some((d2, (cx, cy).into())); + } + } + best.map(|(_, p)| p).unwrap_or(p) + } + + /// Logical layout point -> physical union-layout coordinates (inverse of + /// `layout_physical_to_logical` for in-bounds points; primary-relative otherwise). + pub(crate) fn layout_logical_to_physical(&self, p: Point) -> (f64, f64) { + let idx = self.node_idx_under(p).unwrap_or(0); + let Some(node) = self.output_nodes.get(idx) else { return (p.x, p.y) }; + let scale = node.output.current_scale().fractional_scale(); + ( + node.pos.0 as f64 + (p.x - node.pos.0 as f64) * scale, + node.pos.1 as f64 + (p.y - node.pos.1 as f64) * scale, + ) + } + + /// Logical size of the given display's output. + pub(crate) fn logical_size_of(&self, id: u32) -> Option<(i32, i32)> { + let idx = self.node_idx_for_id(id)?; + let geo = self.output_nodes[idx].logical_geometry()?; + Some((geo.size.w, geo.size.h)) + } + + /// The display id under the pointer (primary when indeterminate). + pub(crate) fn pointer_display(&self) -> u32 { + self.seat + .get_pointer() + .map(|p| p.current_location()) + .and_then(|pos| self.node_idx_under(pos)) + .map(|idx| self.output_nodes[idx].id) + .unwrap_or(0) + } + + /// Place `window` on output `id`: retag its meta, remap it at the output's layout + /// origin, move output enter/leave, push the output's fractional scale, and send the + /// forced-fullscreen configure at that output's logical size. + pub(crate) fn place_window_on_output(&mut self, window: &Window, id: u32) -> bool { + let Some(idx) = self.node_idx_for_id(id) else { return false }; + let old_id = window_output_id(window); + let (new_output, pos) = { + let node = &self.output_nodes[idx]; + (node.output.clone(), node.pos) + }; + let old_output = self + .node_idx_for_id(old_id) + .map(|i| self.output_nodes[i].output.clone()); + if let Some(meta) = window_meta(window) { + meta.output.store(id, Ordering::Relaxed); + } + self.space.map_element(window.clone(), pos, true); + if let Some(surface) = window.wl_surface() { + if let Some(old) = old_output { + if old_id != id { + old.leave(&surface); + } + } + new_output.enter(&surface); + let scale = new_output.current_scale().fractional_scale(); + with_states(&surface, |states| { + smithay::wayland::fractional_scale::with_fractional_scale(states, |fs| { + fs.set_preferred_scale(scale); + }); + }); + } + if let Some(toplevel) = window.toplevel() { + let toplevel = toplevel.clone(); + self.send_forced_fullscreen_configure(&toplevel); + } + true + } + /// Drain a clipboard read staged by `new_selection` and hand `(mime, bytes)` to the /// Python callback off-thread. /// /// Runs from the loop *after* the dispatch that stored the new client source, so the request /// targets the current selection rather than the previous one. It clones the callback, opens a /// pipe, and asks the owning client source to write the chosen mime into the pipe's writer. A - /// spawned reader thread then reads the response — capped at 64 MiB so a hostile client cannot - /// balloon memory, and bounded by a 10 s poll deadline so a client that takes the selection but - /// never writes nor closes its fd cannot pin the thread forever (each clipboard change would - /// otherwise leak one zombie thread + pipe) — and, unless the interpreter is finalizing, - /// delivers the bytes to Python. The `PY_SHUTDOWN` checks keep this off a shutting-down - /// interpreter. + /// spawned reader thread then reads the response. The overall bound is by SIZE (64 MiB, then + /// delivered truncated) so a hostile client cannot balloon memory; time only bounds + /// INACTIVITY — a producer that keeps bytes flowing may take as long as it needs (a large + /// transfer from a slow source still delivers), while one that goes silent for 10 s without + /// closing its fd is dropped so each clipboard change cannot leak a pinned thread + pipe. + /// The `PY_SHUTDOWN` checks keep this off a shutting-down interpreter. pub(crate) fn process_pending_clipboard_read(&mut self) { let Some(mime) = self.pending_clipboard_read.take() else { return }; if crate::PY_SHUTDOWN.load(std::sync::atomic::Ordering::Relaxed) { @@ -660,12 +910,14 @@ impl AppState { use std::io::Read; use std::os::fd::AsRawFd; const CAP: usize = 64 * 1024 * 1024; - const DEADLINE: std::time::Duration = std::time::Duration::from_secs(10); - let start = Instant::now(); + const IDLE_DEADLINE: std::time::Duration = std::time::Duration::from_secs(10); + let mut last_data = Instant::now(); let mut buf = Vec::new(); let mut chunk = [0u8; 65536]; loop { - let Some(remaining) = DEADLINE.checked_sub(start.elapsed()) else { return }; + let Some(remaining) = IDLE_DEADLINE.checked_sub(last_data.elapsed()) else { + return; + }; let mut pfd = libc::pollfd { fd: reader.as_raw_fd(), events: libc::POLLIN, @@ -680,11 +932,12 @@ impl AppState { return; } if ready == 0 { - return; + continue; } match (&reader).read(&mut chunk) { Ok(0) => break, Ok(n) => { + last_data = Instant::now(); let room = CAP - buf.len(); let take_n = n.min(room); buf.extend_from_slice(&chunk[..take_n]); @@ -711,253 +964,230 @@ impl AppState { } - /// Resolve a `CursorImageStatus` to PNG bytes plus hotspot and invoke the Python cursor - /// callback. Also re-invoked from the calloop command handlers to replay the retained cursor - /// when a callback re-registers or a capture restarts. + /// Resolve a `CursorImageStatus` into a job for the `wl-cursor` worker, which does the + /// PNG encode, caching, and the GIL-bound Python call off the calloop thread. Also re-invoked + /// from the calloop command handlers to replay the retained cursor when a callback + /// (re)registers or a capture restarts. /// - /// Dispatches on the cursor status: + /// Only the renderer/surface-affine work happens here: /// - /// 1. **Named**: look the themed cursor up by CSS name and emit its cached PNG (`"png"`), or - /// `"error"` if the theme lacks it. - /// 2. **Hidden**: emit `"hide"` — the only message allowed through with empty data, since an - /// intentional pointer hide must blank the consumer's cursor. - /// 3. **Surface** (a client-supplied cursor sprite): ignore any surface without the + /// 1. **Named** / **Hidden**: forwarded as-is (the worker owns its own theme handle). + /// 2. **Surface** (a client-supplied cursor sprite): ignore any surface without the /// `cursor_image` role, read the hotspot, then read the backing buffer by one of two paths: /// - **SHM**: hash only the sprite's sub-region — width/height/stride/offset/format plus the /// pixel span — because many sprites share one pool and differ only by `offset`, so hashing - /// the whole pool would collide. On a cache miss (and only when ≤128×128) convert the - /// BGRA/XRGB pixels to RGBA, un-premultiply (wl_shm cursor content is premultiplied by - /// convention; PNG carries straight alpha), and PNG-encode; `Xrgb8888` has no alpha so - /// byte 3 is forced opaque, and stride/offset are clamped non-negative with checked - /// arithmetic so a garbage descriptor skips pixels instead of panicking. + /// the whole pool would collide; ship the raw pool bytes plus descriptor to the worker. /// - **dmabuf** (`NotManaged`): bind it to the GLES renderer, copy the framebuffer to - /// `Abgr8888`, map it back, un-premultiply, and PNG-encode using the derived readback - /// stride. - /// The PNG cache is bounded at 100 entries by arbitrary eviction; content-hashing means an - /// evicted sprite simply re-inserts on the next render. - /// - /// A surface whose buffer could not be read yields `("surface", empty)`, which the final gate - /// suppresses — Python is called only for non-empty data or `"hide"` — so a transient read miss - /// (common during a stop/start replay) preserves the consumer's last cursor instead of blanking - /// it. `PY_SHUTDOWN` gates the whole call off a finalizing interpreter. + /// `Abgr8888`, map it back (calloop-affine), and ship the raw RGBA readback. + /// A sprite whose buffer could not be read is dropped by the worker, preserving the + /// consumer's last cursor instead of blanking it (only "hide" carries empty data). pub(crate) fn send_cursor_image(&mut self, image: &CursorImageStatus) { - if crate::PY_SHUTDOWN.load(std::sync::atomic::Ordering::Relaxed) { + if !self.cursor_callback_set { return; } - if let Some(ref cb) = self.cursor_callback { - let (msg_type, data, hot_x, hot_y) = match image { - CursorImageStatus::Named(icon) => { - self.cursor_buffer = None; - let name = cursor_icon_to_str(icon); - if let Some((png_bytes, x, y)) = self.cursor_helper.get_png_data(name) { - ("png", png_bytes, x as i32, y as i32) - } else { - ("error", Vec::new(), 0, 0) + let job = match image { + CursorImageStatus::Named(icon) => { + self.cursor_buffer = None; + CursorJob::Named { name: cursor_icon_to_str(icon) } + } + CursorImageStatus::Hidden => { + self.cursor_buffer = None; + CursorJob::Hide + } + CursorImageStatus::Surface(ref surface) => { + let mut hot_x = 0; + let mut hot_y = 0; + let mut is_cursor_role = false; + + with_states(surface, |states| { + if states.role == Some("cursor_image") { + is_cursor_role = true; } + if let Some(attributes) = states.data_map.get::>() { + if let Ok(guard) = attributes.lock() { + hot_x = guard.hotspot.x; + hot_y = guard.hotspot.y; + } + } + }); + + if !is_cursor_role { + return; } - CursorImageStatus::Hidden => { - self.cursor_buffer = None; - ("hide", Vec::new(), 0, 0) - }, - CursorImageStatus::Surface(ref surface) => { - let mut final_png = Vec::new(); - let mut hot_x = 0; - let mut hot_y = 0; - let mut is_cursor_role = false; - with_states(surface, |states| { - if states.role == Some("cursor_image") { - is_cursor_role = true; - } - if let Some(attributes) = states.data_map.get::>() { - if let Ok(guard) = attributes.lock() { - hot_x = guard.hotspot.x; - hot_y = guard.hotspot.y; - } - } - }); + let buffer_found = with_states(surface, |states| { + let mut attrs = states.cached_state.get::(); - if !is_cursor_role { - return; + if let Some(BufferAssignment::NewBuffer(b)) = &attrs.current().buffer { + return Some(b.clone()); } - let buffer_found = with_states(surface, |states| { - let mut attrs = states.cached_state.get::(); - - if let Some(BufferAssignment::NewBuffer(b)) = &attrs.current().buffer { - return Some(b.clone()); - } - - if let Some(mutex) = states.data_map.get::>() { - if let Ok(renderer_state) = mutex.try_lock() { - if let Some(b) = renderer_state.buffer() { - let wl_buffer: &wayland_server::protocol::wl_buffer::WlBuffer = b; - return Some(wl_buffer.clone()); - } + if let Some(mutex) = states.data_map.get::>() { + if let Ok(renderer_state) = mutex.try_lock() { + if let Some(b) = renderer_state.buffer() { + let wl_buffer: &wayland_server::protocol::wl_buffer::WlBuffer = b; + return Some(wl_buffer.clone()); } } - None - }); - - if let Some(buffer) = buffer_found { - let shm_result = with_buffer_contents(&buffer, |ptr, len, spec| { - let slice = unsafe { std::slice::from_raw_parts(ptr, len) }; - let mut hasher = DefaultHasher::new(); - spec.width.hash(&mut hasher); - spec.height.hash(&mut hasher); - spec.stride.hash(&mut hasher); - spec.offset.hash(&mut hasher); - spec.format.hash(&mut hasher); - let start = (spec.offset.max(0) as usize).min(len); - let span = (spec.stride.max(0) as usize) - .saturating_mul(spec.height.max(0) as usize); - let end = start.saturating_add(span).min(len); - slice[start..end].hash(&mut hasher); - let hash = hasher.finish(); - (hash, spec.width, spec.height, spec.stride, spec.format, spec.offset, slice.to_vec()) - }); + } + None + }); - match shm_result { - Ok((hash, width, height, stride, format, buf_offset, raw_bytes)) => { - if let Some(cached_png) = self.cursor_cache.get(&hash) { - final_png = cached_png.clone(); - } else { - if width <= 128 && height <= 128 && !raw_bytes.is_empty() { - let mut img_buf = ImageBuffer::, Vec>::new(width as u32, height as u32); - let stride_usize = stride.max(0) as usize; - let base_offset = buf_offset.max(0) as usize; - - for y in 0..(height as u32) { - for x in 0..(width as u32) { - let offset = (y as usize) - .checked_mul(stride_usize) - .and_then(|row| base_offset.checked_add(row)) - .and_then(|o| o.checked_add((x as usize) * 4)); - let offset = match offset { - Some(o) => o, - None => continue, - }; - if offset.checked_add(4).is_some_and(|end| end <= raw_bytes.len()) { - let alpha = if format == wl_shm::Format::Xrgb8888 { - 255 - } else { - raw_bytes[offset + 3] - }; - img_buf.put_pixel(x, y, Rgba([ - raw_bytes[offset + 2], - raw_bytes[offset + 1], - raw_bytes[offset], - alpha - ])); - } - } - } + let Some(buffer) = buffer_found else { return }; + + let shm_result = with_buffer_contents(&buffer, |ptr, len, spec| { + let slice = unsafe { std::slice::from_raw_parts(ptr, len) }; + let mut hasher = DefaultHasher::new(); + spec.width.hash(&mut hasher); + spec.height.hash(&mut hasher); + spec.stride.hash(&mut hasher); + spec.offset.hash(&mut hasher); + spec.format.hash(&mut hasher); + let start = (spec.offset.max(0) as usize).min(len); + let span = (spec.stride.max(0) as usize) + .saturating_mul(spec.height.max(0) as usize); + let end = start.saturating_add(span).min(len); + slice[start..end].hash(&mut hasher); + let hash = hasher.finish(); + (hash, spec.width, spec.height, spec.stride, spec.format, spec.offset, slice.to_vec()) + }); - crate::unpremultiply_rgba(&mut img_buf); - let mut bytes = Vec::new(); - if img_buf.write_to(&mut IoCursor::new(&mut bytes), ImageFormat::Png).is_ok() { - self.cursor_cache.insert(hash, bytes.clone()); - final_png = bytes; - if self.cursor_cache.len() > 100 { - let evict = *self.cursor_cache.keys().next().unwrap(); - self.cursor_cache.remove(&evict); - } - } - } - } - }, - Err(BufferAccessError::NotManaged) => { - let mut gles_data: Option<(u64, i32, i32, Vec)> = None; - - let dmabuf_opt = get_dmabuf(&buffer).ok().cloned(); - - if let Some(mut dmabuf) = dmabuf_opt { - if let Some(renderer) = self.gles_renderer.as_mut() { - let width = dmabuf.width() as i32; - let height = dmabuf.height() as i32; - - match renderer.bind(&mut dmabuf) { - Ok(frame) => { - let rect = Rectangle::new((0, 0).into(), (width, height).into()); - - match renderer.copy_framebuffer(&frame, rect, Fourcc::Abgr8888) { - Ok(mapping) => { - match renderer.map_texture(&mapping) { - Ok(data) => { - let mut hasher = DefaultHasher::new(); - data.hash(&mut hasher); - let hash = hasher.finish(); - gles_data = Some((hash, width, height, data.to_vec())); - }, - Err(e) => eprintln!("Failed to map texture: {:?}", e) - } - }, - Err(e) => eprintln!("Failed to copy framebuffer: {:?}", e) + let job = match shm_result { + Ok((hash, width, height, stride, format, buf_offset, raw_bytes)) => { + Some(CursorJob::Shm { + hash, + width, + height, + stride, + offset: buf_offset, + opaque: format == wl_shm::Format::Xrgb8888, + bytes: raw_bytes, + hot_x, + hot_y, + }) + } + Err(BufferAccessError::NotManaged) => { + let mut gles_job = None; + let dmabuf_opt = get_dmabuf(&buffer).ok().cloned(); + if let Some(mut dmabuf) = dmabuf_opt { + if let Some(renderer) = self.gles_renderer.as_mut() { + let width = dmabuf.width() as i32; + let height = dmabuf.height() as i32; + + match renderer.bind(&mut dmabuf) { + Ok(frame) => { + let rect = Rectangle::new((0, 0).into(), (width, height).into()); + match renderer.copy_framebuffer(&frame, rect, Fourcc::Abgr8888) { + Ok(mapping) => match renderer.map_texture(&mapping) { + Ok(data) => { + let mut hasher = DefaultHasher::new(); + data.hash(&mut hasher); + gles_job = Some(CursorJob::Gles { + hash: hasher.finish(), + width, + height, + bytes: data.to_vec(), + hot_x, + hot_y, + }); } + Err(e) => eprintln!("Failed to map texture: {:?}", e), }, - Err(e) => eprintln!("Failed to bind dmabuf to renderer: {:?}", e) + Err(e) => eprintln!("Failed to copy framebuffer: {:?}", e), } } + Err(e) => eprintln!("Failed to bind dmabuf to renderer: {:?}", e), } - - if let Some((hash, width, height, raw_bytes)) = gles_data { - if let Some(cached_png) = self.cursor_cache.get(&hash) { - final_png = cached_png.clone(); - } else { - if width <= 128 && height <= 128 && !raw_bytes.is_empty() { - let mut img_buf = ImageBuffer::, Vec>::new(width as u32, height as u32); - let stride_usize = rgba_readback_stride(raw_bytes.len(), height as usize, width as usize); - - for y in 0..(height as u32) { - for x in 0..(width as u32) { - let offset = (y as usize * stride_usize) + (x as usize * 4); - if offset + 4 <= raw_bytes.len() { - img_buf.put_pixel(x, y, Rgba([ - raw_bytes[offset], - raw_bytes[offset + 1], - raw_bytes[offset + 2], - raw_bytes[offset + 3] - ])); - } - } - } - - crate::unpremultiply_rgba(&mut img_buf); - let mut bytes = Vec::new(); - if img_buf.write_to(&mut IoCursor::new(&mut bytes), ImageFormat::Png).is_ok() { - self.cursor_cache.insert(hash, bytes.clone()); - final_png = bytes; - if self.cursor_cache.len() > 100 { - let evict = *self.cursor_cache.keys().next().unwrap(); - self.cursor_cache.remove(&evict); - } - } - } - } - } - }, - Err(_) => {} + } } - - self.cursor_buffer = Some(buffer); + gles_job } + Err(_) => None, + }; - if !final_png.is_empty() { - ("png", final_png, hot_x, hot_y) - } else { - ("surface", Vec::new(), 0, 0) - } - } - }; + self.cursor_buffer = Some(buffer); + let Some(job) = job else { return }; + job + } + }; + let _ = self.cursor_tx.send(job); + } - if !data.is_empty() || msg_type == "hide" { - Python::attach(|py| { - let py_bytes = PyBytes::new(py, &data); - let _ = cb.call1(py, (msg_type, py_bytes, hot_x, hot_y)); - }); + /// Re-apply the policy's keymap (base + overlays) to the seat keyboard, broadcasting to + /// clients only when the content actually changed (smithay dedupes by content hash). + pub(crate) fn apply_keymap_policy(&mut self) { + let text = self.keymap_policy.keymap_text(); + if text.is_empty() { + return; + } + if let Some(keyboard) = self.seat.get_keyboard() { + if let Err(e) = keyboard.set_keymap_from_string(self, text) { + eprintln!("[Wayland] keymap swap failed: {e:?}"); } } } + + /// Resolve `keysyms` to `(keycode, level)` pairs, overlay-binding whatever the base + /// cannot produce — at most ONE keymap swap for the whole batch, and never rebinding a + /// keycode that is currently held down. + pub(crate) fn bind_keysyms(&mut self, keysyms: &[u32]) -> Vec<(u32, u32)> { + let pressed: std::collections::HashSet = self + .seat + .get_keyboard() + .map(|k| k.pressed_keys().iter().map(|c| c.raw()).collect()) + .unwrap_or_default(); + let (out, changed) = self.keymap_policy.bind_many(keysyms, &pressed); + if changed { + self.apply_keymap_policy(); + } + out + } + + /// `bind_keysyms` restricted to level-0 resolutions (see + /// [`KeymapPolicy::bind_many_plain`]); used by the virtual-keyboard translation path, which + /// cannot synthesize modifiers. + pub(crate) fn bind_keysyms_plain(&mut self, keysyms: &[u32]) -> Vec { + let pressed: std::collections::HashSet = self + .seat + .get_keyboard() + .map(|k| k.pressed_keys().iter().map(|c| c.raw()).collect()) + .unwrap_or_default(); + let (out, changed) = self.keymap_policy.bind_many_plain(keysyms, &pressed); + if changed { + self.apply_keymap_policy(); + } + out + } + + /// Answer any client fullscreen/maximize (un)set request with the compositor's forced + /// policy: every toplevel is Fullscreen+Activated at the CURRENT logical size of the + /// output the window is placed on. The configure is always sent, so an app toggling + /// fullscreen gets an explicit, current-geometry answer instead of silence or stale + /// pending state. + pub(crate) fn send_forced_fullscreen_configure(&mut self, toplevel: &ToplevelSurface) { + let output_id = self + .space + .elements() + .chain(self.pending_windows.iter()) + .find(|w| w.toplevel().map(|t| t == toplevel).unwrap_or(false)) + .map(window_output_id) + .unwrap_or(0); + let (logical_width, logical_height) = if let Some(size) = self.logical_size_of(output_id) { + size + } else { + let scale = self.settings.scale.max(0.1); + ( + (self.settings.width as f64 / scale).round() as i32, + (self.settings.height as f64 / scale).round() as i32, + ) + }; + toplevel.with_pending_state(|state| { + state.states.set(XdgState::Fullscreen); + state.states.set(XdgState::Activated); + state.size = Some((logical_width, logical_height).into()); + }); + toplevel.send_configure(); + } } /// Clipboard mime types the bridge can hand to Python, most specific first; `new_selection` @@ -997,21 +1227,25 @@ impl SelectionHandler for AppState { if ty != SelectionTarget::Clipboard { return; } + let Some(source) = source else { + self.current_selection_mime = None; + return; + }; + let mimes = source.mime_types(); + let mime = CLIPBOARD_MIME_PREFERENCE + .iter() + .find(|want| mimes.iter().any(|m| m == *want)) + .map(|s| s.to_string()); + // Recorded even with no callback armed, so SetClipboardCallback can re-stage a + // read of a copy made while nobody was listening. + self.current_selection_mime = mime.clone(); if crate::PY_SHUTDOWN.load(std::sync::atomic::Ordering::Relaxed) { return; } if self.clipboard_callback.is_none() { return; } - let Some(source) = source else { return }; - let mimes = source.mime_types(); - let Some(mime) = CLIPBOARD_MIME_PREFERENCE - .iter() - .find(|want| mimes.iter().any(|m| m == *want)) - .map(|s| s.to_string()) - else { - return; - }; + let Some(mime) = mime else { return }; self.pending_clipboard_read = Some(mime); let _ = seat; } @@ -1102,7 +1336,7 @@ impl FractionalScaleHandler for AppState { &mut self, surface: smithay::reexports::wayland_server::protocol::wl_surface::WlSurface, ) { - if let Some(output) = self.outputs.first() { + if let Some(output) = self.primary_output() { let scale = output.current_scale().fractional_scale(); with_states(&surface, |states| { smithay::wayland::fractional_scale::with_fractional_scale(states, |fs| { @@ -1613,17 +1847,17 @@ impl SeatHandler for AppState { self.send_cursor_image(&image); } - /// Keep the primary selection's focus following keyboard focus, so middle-click paste - /// targets the currently focused client (or clears it when nothing is focused). + /// Keep BOTH selections' focus following keyboard focus: without the data-device half, + /// the focused client never receives wl_data_offer events and Ctrl+V paste is a silent + /// no-op even while the compositor-side selection is correct (primary covers only + /// middle-click paste). fn focus_changed(&mut self, seat: &Seat, focus: Option<&Self::KeyboardFocus>) { - if let Some(focus_target) = focus { - let dh = &self.dh; - let client = focus_target.wl_surface().and_then(|s| dh.get_client(s.id()).ok()); - set_primary_focus(dh, seat, client); - } else { - let dh = &self.dh; - set_primary_focus(dh, seat, None); - } + let dh = &self.dh; + let client = focus + .and_then(|t| t.wl_surface()) + .and_then(|s| dh.get_client(s.id()).ok()); + set_data_device_focus(dh, seat, client.clone()); + set_primary_focus(dh, seat, client); } } @@ -1679,8 +1913,16 @@ impl XdgShellHandler for AppState { } /// A new toplevel appears: wrap it in a `Window`, queue it for mapping, and register a /// foreign-toplevel handle (seeded with title / app-id) stored on the surface for later updates. + /// The window is pinned to the pointer's output HERE — the first commit can't be relied on + /// for that, because a decoration-negotiating client (foot) has its initial configure sent + /// by `new_decoration` before it ever commits, skipping the pre-configure commit branch. fn new_toplevel(&mut self, surface: ToplevelSurface) { + let target_id = self.pointer_display(); let window = Window::new_wayland_window(surface.clone()); + window.user_data().insert_if_missing_threadsafe(|| WindowMeta { + id: NEXT_WINDOW_ID.fetch_add(1, Ordering::Relaxed), + output: AtomicU32::new(target_id), + }); self.pending_windows.push(window); let (title, app_id) = with_states(surface.wl_surface(), |states| { let attributes = states.data_map.get::().unwrap().lock().unwrap(); @@ -1729,19 +1971,233 @@ impl XdgShellHandler for AppState { } let _ = surface.send_repositioned(token); } + /// Client fullscreen request: always granted at the compositor's forced-fullscreen + /// geometry (the CURRENT logical size), so the toggle gets a definite answer. + fn fullscreen_request( + &mut self, + surface: ToplevelSurface, + _output: Option, + ) { + self.send_forced_fullscreen_configure(&surface); + } + /// Client unfullscreen request: the forced-fullscreen policy stands, but the client + /// still receives an explicit configure at the current geometry (the Smithay default sends + /// NOTHING here, leaving the app waiting on a toggle that never answers). + fn unfullscreen_request(&mut self, surface: ToplevelSurface) { + self.send_forced_fullscreen_configure(&surface); + } + /// Maximize request: answered with the forced-fullscreen configure (same geometry). + fn maximize_request(&mut self, surface: ToplevelSurface) { + self.send_forced_fullscreen_configure(&surface); + } + /// Unmaximize request: explicit current-geometry configure, policy unchanged. + fn unmaximize_request(&mut self, surface: ToplevelSurface) { + self.send_forced_fullscreen_configure(&surface); + } /// A toplevel closed: drop it from the pending-window queue so a window that never - /// finished mapping can't linger there, and remove its foreign-toplevel handle so taskbar-style - /// clients stop listing a window that is gone. + /// finished mapping can't linger there, unmap it from the space at once so `list_windows` + /// and hit-testing never see a husk (the periodic `space.refresh` would only reap it + /// later), and remove its foreign-toplevel handle so taskbar-style clients stop listing a + /// window that is gone. fn toplevel_destroyed(&mut self, surface: ToplevelSurface) { if let Some(idx) = self.pending_windows.iter().position(|w| w.toplevel().map(|t| *t == surface).unwrap_or(false)) { self.pending_windows.remove(idx); } + let mapped = self + .space + .elements() + .find(|w| w.toplevel().map(|t| *t == surface).unwrap_or(false)) + .cloned(); + if let Some(window) = mapped { + self.space.unmap_elem(&window); + } if let Some(handle) = with_states(surface.wl_surface(), |states| states.data_map.get::().cloned()) { self.foreign_toplevel_list.remove_toplevel(&handle); } } } +/// In-house `zwp_virtual_keyboard_v1` implementation. Smithay's manager swaps the +/// client-visible seat keymap to the virtual keyboard's keymap on every VK event and never +/// restores it, leaving every client holding a foreign keymap (and killing the compositor's +/// overlay keycodes) after any VK use. Here VK key events are TRANSLATED instead: each keycode +/// resolves to its level-0 keysym under the VK client's own uploaded keymap, maps onto the seat +/// keymap (overlay-binding on demand, batched at keymap upload), and injects through the seat's +/// regular input path — the seat keymap identity never changes and modifier/pressed-key state +/// stays coherent with server-side injection. VK `modifiers` requests are ignored: applying a +/// foreign modifier mask would corrupt the seat's own tracked state, and the supported VK +/// client (selkies' wayland_typer) binds every keysym at level 0 and never sends them. +pub struct PfVirtualKeyboard { + inner: Mutex, +} + +#[derive(Default)] +struct PfVkState { + /// Level-0 keysym per xkb keycode of the client's uploaded keymap. + syms: Option>, + /// VK xkb keycode -> injected seat keycode, so a release always matches its press even + /// across policy rebinds. + pressed: std::collections::HashMap, +} + +impl GlobalDispatch for AppState { + fn bind( + _state: &mut Self, + _dh: &DisplayHandle, + _client: &Client, + resource: New, + _global_data: &(), + data_init: &mut DataInit<'_, Self>, + ) { + data_init.init(resource, ()); + } +} + +impl Dispatch for AppState { + fn request( + _state: &mut Self, + _client: &Client, + _resource: &ZwpVirtualKeyboardManagerV1, + request: zwp_virtual_keyboard_manager_v1::Request, + _data: &(), + _dh: &DisplayHandle, + data_init: &mut DataInit<'_, Self>, + ) { + if let zwp_virtual_keyboard_manager_v1::Request::CreateVirtualKeyboard { seat: _, id } = + request + { + data_init.init(id, PfVirtualKeyboard { inner: Mutex::new(PfVkState::default()) }); + } + } +} + +impl Dispatch for AppState { + fn request( + state: &mut Self, + _client: &Client, + resource: &ZwpVirtualKeyboardV1, + request: zwp_virtual_keyboard_v1::Request, + data: &PfVirtualKeyboard, + _dh: &DisplayHandle, + _data_init: &mut DataInit<'_, Self>, + ) { + use smithay::input::keyboard::xkb; + match request { + zwp_virtual_keyboard_v1::Request::Keymap { format, fd, size } => { + if format != 1 { + return; + } + let ctx = xkb::Context::new(xkb::CONTEXT_NO_FLAGS); + let keymap = unsafe { + xkb::Keymap::new_from_fd( + &ctx, + fd, + size as usize, + xkb::KEYMAP_FORMAT_TEXT_V1, + xkb::KEYMAP_COMPILE_NO_FLAGS, + ) + }; + let Ok(Some(keymap)) = keymap else { + eprintln!("[Wayland] virtual-keyboard keymap failed to compile; ignoring."); + return; + }; + let syms = crate::wayland::keymap::level0_syms(&keymap); + // Pre-bind everything this keymap can type that the seat cannot, in ONE + // seat keymap swap, so the following key events bind nothing. + let missing: Vec = syms + .values() + .copied() + .filter(|&s| !state.keymap_policy.resolves_plain(s)) + .collect(); + if !missing.is_empty() { + let _ = state.bind_keysyms_plain(&missing); + } + data.inner.lock().unwrap().syms = Some(syms); + } + zwp_virtual_keyboard_v1::Request::Key { time: _, key, state: key_state } => { + let mut vk = data.inner.lock().unwrap(); + if vk.syms.is_none() { + drop(vk); + resource.post_error( + zwp_virtual_keyboard_v1::Error::NoKeymap, + "`key` sent before keymap.", + ); + return; + } + let vk_kc = key.wrapping_add(8); + let seat_kc = if key_state == 1 { + let sym = vk.syms.as_ref().and_then(|m| m.get(&vk_kc)).copied(); + // Translate through the seat keymap; an untranslatable keycode + // passes through raw (base sections of both keymaps agree for + // ordinary pc keycodes). + let kc = match sym { + Some(sym) => { + let bound = state.bind_keysyms_plain(&[sym])[0]; + if bound != 0 { bound } else { vk_kc } + } + None => vk_kc, + }; + vk.pressed.insert(vk_kc, kc); + kc + } else { + vk.pressed.remove(&vk_kc).unwrap_or(vk_kc) + }; + drop(vk); + let pressed = key_state == 1; + if let Some(keyboard) = state.seat.get_keyboard() { + let keyboard = keyboard.clone(); + let serial = next_serial(); + let time = wayland_time(); + keyboard.input( + state, + smithay::backend::input::Keycode::new(seat_kc), + if pressed { + smithay::backend::input::KeyState::Pressed + } else { + smithay::backend::input::KeyState::Released + }, + serial, + time, + |_, _, _| smithay::input::keyboard::FilterResult::<()>::Forward, + ); + } + } + zwp_virtual_keyboard_v1::Request::Modifiers { .. } => {} + zwp_virtual_keyboard_v1::Request::Destroy => {} + _ => {} + } + } + + /// Release every seat key this virtual keyboard still holds, so a VK client that + /// disconnects mid-press cannot leave keys logically stuck. + fn destroyed( + state: &mut Self, + _client: ClientId, + _resource: &ZwpVirtualKeyboardV1, + data: &PfVirtualKeyboard, + ) { + let held: Vec = data.inner.lock().unwrap().pressed.drain().map(|(_, kc)| kc).collect(); + if held.is_empty() { + return; + } + if let Some(keyboard) = state.seat.get_keyboard() { + let keyboard = keyboard.clone(); + for kc in held { + let serial = next_serial(); + let time = wayland_time(); + keyboard.input( + state, + smithay::backend::input::Keycode::new(kc), + smithay::backend::input::KeyState::Released, + serial, + time, + |_, _, _| smithay::input::keyboard::FilterResult::<()>::Forward, + ); + } + } + } +} + /// Per-client data attached to every Wayland client connection; holds the compositor's /// per-client surface state. #[derive(Default)] @@ -1761,7 +2217,6 @@ delegate_seat!(AppState); delegate_xdg_shell!(AppState); delegate_dmabuf!(AppState); delegate_fractional_scale!(AppState); -delegate_virtual_keyboard_manager!(AppState); delegate_data_device!(AppState); delegate_data_control!(AppState); delegate_pointer_warp!(AppState); @@ -1782,7 +2237,7 @@ delegate_primary_selection!(AppState); /// Dividing the buffer length by the height recovers a padded stride, so a padded readback cannot /// skew the cursor image; the result never drops below one full `width*4` row, and a zero height /// short-circuits to one row to avoid dividing by zero. -fn rgba_readback_stride(buf_len: usize, height: usize, width: usize) -> usize { +pub(crate) fn rgba_readback_stride(buf_len: usize, height: usize, width: usize) -> usize { let row = width.saturating_mul(4); if height == 0 { return row; diff --git a/pixelflux/src/wayland/keymap.rs b/pixelflux/src/wayland/keymap.rs new file mode 100644 index 0000000..5034484 --- /dev/null +++ b/pixelflux/src/wayland/keymap.rs @@ -0,0 +1,384 @@ +/* + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at https://mozilla.org/MPL/2.0/. + */ + +//! Compositor-side keymap policy: one owner for the seat keymap. +//! +//! The seat keymap is BASE text (US by default, replaceable at runtime with a full +//! XKB_KEYMAP_FORMAT_TEXT_V1 string or RMLVO names) plus an OVERLAY of spare keycodes bound to +//! keysyms the base cannot produce (Unicode / IME output). All rebinds are batched: resolving N +//! new keysyms produces ONE keymap swap, and a keycode that is currently held down is never +//! recycled, so its release event always means the symbol its press meant. + +use std::collections::{HashMap, HashSet, VecDeque}; + +use smithay::input::keyboard::xkb; + +/// First overlay keycode. Sits above both the evdev/pc105 range and the legacy selkies +/// overlay range (257-272) so a base keymap carrying those legacy binds cannot collide. +pub const OVERLAY_FIRST_KEYCODE: u32 = 0x120; +/// Last overlay keycode (inclusive). Keycodes past the X11 255 ceiling are fine for +/// pure-Wayland clients: they look keycodes up in the delivered keymap via xkbcommon. +pub const OVERLAY_LAST_KEYCODE: u32 = 0x2ff; +/// Overlay slot count (keycodes `OVERLAY_FIRST..=OVERLAY_LAST`). +pub const OVERLAY_CAPACITY: usize = (OVERLAY_LAST_KEYCODE - OVERLAY_FIRST_KEYCODE + 1) as usize; + +/// Highest shift level consulted when reverse-mapping the base keymap (plain, Shift, +/// AltGr, Shift+AltGr). +const MAX_LEVELS: u32 = 4; + +/// Seat keymap state: the base text, its reverse keysym map, and the overlay slots. +pub struct KeymapPolicy { + base_text: String, + /// keysym -> (xkb keycode, level) in the base keymap; lowest level wins. + base_map: HashMap, + /// slot index -> bound keysym. + slots: Vec>, + /// keysym -> slot index. + by_sym: HashMap, + /// Slot recycle order, oldest bind first. + lru: VecDeque, +} + +/// Compile an XKB_KEYMAP_FORMAT_TEXT_V1 string, or `None` when it does not compile. +pub fn compile_keymap(text: &str) -> Option { + let ctx = xkb::Context::new(xkb::CONTEXT_NO_FLAGS); + xkb::Keymap::new_from_string( + &ctx, + text.to_string(), + xkb::KEYMAP_FORMAT_TEXT_V1, + xkb::KEYMAP_COMPILE_NO_FLAGS, + ) +} + +/// Compile RMLVO names to keymap text, or `None` when compilation fails. Empty strings +/// select the xkbcommon defaults for that component. +pub fn compile_rmlvo( + rules: &str, + model: &str, + layout: &str, + variant: &str, + options: &str, +) -> Option { + let ctx = xkb::Context::new(xkb::CONTEXT_NO_FLAGS); + let options = (!options.is_empty()).then(|| options.to_string()); + let keymap = xkb::Keymap::new_from_names( + &ctx, + rules, + model, + layout, + variant, + options, + xkb::KEYMAP_COMPILE_NO_FLAGS, + )?; + Some(keymap.get_as_string(xkb::KEYMAP_FORMAT_TEXT_V1)) +} + +/// Level-0 keysym for every key of a compiled keymap (used to pre-bind a virtual-keyboard +/// client's keymap in one batch). Keys with zero or multiple level-0 syms are skipped. +pub fn level0_syms(keymap: &xkb::Keymap) -> HashMap { + let mut out = HashMap::new(); + let lo = keymap.min_keycode().raw(); + let hi = keymap.max_keycode().raw(); + for kc in lo..=hi { + let syms = keymap.key_get_syms_by_level(xkb::Keycode::new(kc), 0, 0); + if syms.len() == 1 { + let sym = syms[0].raw(); + if sym != 0 { + out.insert(kc, sym); + } + } + } + out +} + +impl KeymapPolicy { + /// Placeholder policy before the seat keymap is known; `rebuild_base` fills it in. + pub fn empty() -> Self { + Self { + base_text: String::new(), + base_map: HashMap::new(), + slots: Vec::new(), + by_sym: HashMap::new(), + lru: VecDeque::new(), + } + } + + /// Replace the base keymap text and rebuild the reverse map. Overlay assignments are + /// retained (same keycodes), so keycodes already handed out stay valid across the swap. + pub fn rebuild_base(&mut self, base_text: String) { + self.base_map.clear(); + if let Some(keymap) = compile_keymap(&base_text) { + let lo = keymap.min_keycode().raw(); + let hi = keymap.max_keycode().raw(); + // Lower levels win across ALL keys, so a keysym reachable unshifted never + // resolves to a shifted position. + for level in 0..MAX_LEVELS { + for kc in lo..=hi { + let code = xkb::Keycode::new(kc); + if keymap.num_levels_for_key(code, 0) <= level { + continue; + } + for sym in keymap.key_get_syms_by_level(code, 0, level) { + let raw = sym.raw(); + if raw != 0 { + self.base_map.entry(raw).or_insert((kc, level)); + } + } + } + } + } + self.base_text = base_text; + } + + /// True once a base keymap has been installed. + pub fn has_base(&self) -> bool { + !self.base_text.is_empty() + } + + /// Resolve `keysym` without binding: base first, then an existing overlay slot. + pub fn resolve(&self, keysym: u32) -> Option<(u32, u32)> { + if let Some(&hit) = self.base_map.get(&keysym) { + return Some(hit); + } + self.by_sym + .get(&keysym) + .map(|&slot| (OVERLAY_FIRST_KEYCODE + slot as u32, 0)) + } + + /// True when `keysym` resolves at level 0 (base or overlay) — i.e. typable without + /// synthetic modifiers. + pub fn resolves_plain(&self, keysym: u32) -> bool { + matches!(self.resolve(keysym), Some((_, 0))) + } + + /// Resolve every keysym, overlay-binding the unresolvable ones. Returns one + /// `(keycode, level)` per input keysym (`(0, 0)` when it cannot be bound) plus whether the + /// keymap changed and must be re-applied — at most ONE swap per call, however many new + /// keysyms were bound. Slots whose keycode is in `pressed` are never recycled. + pub fn bind_many( + &mut self, + keysyms: &[u32], + pressed: &HashSet, + ) -> (Vec<(u32, u32)>, bool) { + let mut out = Vec::with_capacity(keysyms.len()); + let mut changed = false; + for &sym in keysyms { + out.push(self.bind_one(sym, pressed, false, &mut changed)); + } + (out, changed) + } + + /// Like `bind_many` but only accepts level-0 resolutions: a keysym reachable in the + /// base solely behind a modifier (e.g. `A` behind Shift) is overlay-bound instead, so the + /// caller can inject it without synthesizing modifiers. Returns keycodes (0 = unbindable). + pub fn bind_many_plain(&mut self, keysyms: &[u32], pressed: &HashSet) -> (Vec, bool) { + let mut out = Vec::with_capacity(keysyms.len()); + let mut changed = false; + for &sym in keysyms { + out.push(self.bind_one(sym, pressed, true, &mut changed).0); + } + (out, changed) + } + + fn bind_one( + &mut self, + sym: u32, + pressed: &HashSet, + plain_only: bool, + changed: &mut bool, + ) -> (u32, u32) { + if sym == 0 { + return (0, 0); + } + if let Some(&(kc, level)) = self.base_map.get(&sym) { + if !plain_only || level == 0 { + return (kc, level); + } + } + if let Some(&slot) = self.by_sym.get(&sym) { + if let Some(at) = self.lru.iter().position(|&s| s == slot) { + self.lru.remove(at); + } + self.lru.push_back(slot); + return (OVERLAY_FIRST_KEYCODE + slot as u32, 0); + } + let slot = if self.slots.len() < OVERLAY_CAPACITY { + self.slots.push(None); + self.slots.len() - 1 + } else { + match self.recycle_slot(pressed) { + Some(s) => s, + None => return (0, 0), + } + }; + if let Some(old) = self.slots[slot].replace(sym) { + self.by_sym.remove(&old); + } + self.by_sym.insert(sym, slot); + self.lru.push_back(slot); + *changed = true; + (OVERLAY_FIRST_KEYCODE + slot as u32, 0) + } + + /// Oldest slot whose keycode is not currently held down; a held keycode must keep its + /// meaning until its release has been delivered. + fn recycle_slot(&mut self, pressed: &HashSet) -> Option { + let at = self + .lru + .iter() + .position(|&slot| !pressed.contains(&(OVERLAY_FIRST_KEYCODE + slot as u32)))?; + self.lru.remove(at) + } + + /// The full seat keymap: the base text with every occupied overlay slot spliced into the + /// `xkb_keycodes` and `xkb_symbols` sections (and `maximum` raised to cover them). With no + /// overlays, the base text verbatim. + pub fn keymap_text(&self) -> String { + let occupied: Vec<(usize, u32)> = self + .slots + .iter() + .enumerate() + .filter_map(|(i, s)| s.map(|sym| (i, sym))) + .collect(); + if occupied.is_empty() { + return self.base_text.clone(); + } + let base = &self.base_text; + let Some(max_at) = base.find("maximum = ") else { + return self.base_text.clone(); + }; + let num_at = max_at + "maximum = ".len(); + let Some(num_len) = base[num_at..].find(';') else { + return self.base_text.clone(); + }; + let old_max: u32 = base[num_at..num_at + num_len].trim().parse().unwrap_or(255); + let need_max = OVERLAY_FIRST_KEYCODE + occupied.last().map(|&(i, _)| i as u32).unwrap_or(0); + let mut text = String::with_capacity(base.len() + occupied.len() * 48); + text.push_str(&base[..num_at]); + text.push_str(&old_max.max(need_max).to_string()); + let rest = &base[num_at + num_len..]; + // First "};" after the maximum line closes xkb_keycodes. + let Some(kc_end) = rest.find("};") else { + return self.base_text.clone(); + }; + text.push_str(&rest[..kc_end]); + for &(i, _) in &occupied { + text.push_str(&format!("\t = {};\n", i, OVERLAY_FIRST_KEYCODE + i as u32)); + } + let rest = &rest[kc_end..]; + let Some(close_at) = rest + .find("xkb_symbols") + .and_then(|sym_at| Self::section_close(rest, sym_at)) + else { + return self.base_text.clone(); + }; + text.push_str(&rest[..close_at]); + for &(i, sym) in &occupied { + text.push_str(&format!("\tkey {{ [ {:#x} ] }};\n", i, sym)); + } + text.push_str(&rest[close_at..]); + text + } + + /// Byte offset of the `}` closing the brace-block that starts at/after `from`. + fn section_close(text: &str, from: usize) -> Option { + let open = from + text[from..].find('{')?; + let mut depth = 0usize; + for (i, ch) in text[open..].char_indices() { + match ch { + '{' => depth += 1, + '}' => { + depth -= 1; + if depth == 0 { + return Some(open + i); + } + } + _ => {} + } + } + None + } +} + +#[cfg(test)] +mod tests { + //! Invariants: one `bind_many` call binds any number of new keysyms with a single + //! keymap change; base keysyms resolve without consuming overlay slots; a pressed + //! overlay keycode survives LRU pressure; the spliced keymap text compiles and + //! resolves the overlay keysyms at their assigned keycodes. + use super::*; + + fn us_base() -> String { + compile_rmlvo("", "", "us", "", "").expect("us keymap") + } + + fn policy() -> KeymapPolicy { + let mut p = KeymapPolicy::empty(); + p.rebuild_base(us_base()); + p + } + + #[test] + fn base_keysyms_resolve_without_overlay() { + let mut p = policy(); + // 'a' plain, 'A' shifted. + let (out, changed) = p.bind_many(&[0x61, 0x41], &HashSet::new()); + assert!(!changed); + assert_eq!(out[0].1, 0); + assert_eq!(out[1].0, out[0].0); + assert_eq!(out[1].1, 1); + } + + #[test] + fn batch_bind_is_one_swap_and_compiles() { + let mut p = policy(); + let syms: Vec = (0..30).map(|i| 0x1004E00 + i).collect(); + let (out, changed) = p.bind_many(&syms, &HashSet::new()); + assert!(changed); + let (_, changed_again) = p.bind_many(&syms, &HashSet::new()); + assert!(!changed_again, "re-binding bound keysyms must not swap"); + let text = p.keymap_text(); + let km = compile_keymap(&text).expect("overlay keymap compiles"); + for (i, &(kc, level)) in out.iter().enumerate() { + assert_eq!(level, 0); + let got = km.key_get_syms_by_level(xkb::Keycode::new(kc), 0, 0); + assert_eq!(got.len(), 1, "keycode {kc} has one sym"); + assert_eq!(got[0].raw(), syms[i]); + } + } + + #[test] + fn pressed_keycode_is_never_recycled() { + let mut p = policy(); + let syms: Vec = (0..OVERLAY_CAPACITY as u32).map(|i| 0x1005000 + i).collect(); + let (out, _) = p.bind_many(&syms, &HashSet::new()); + let held_kc = out[0].0; + let held_sym = syms[0]; + let pressed: HashSet = [held_kc].into_iter().collect(); + // Force full recycling pressure past capacity. + let extra: Vec = (0..8).map(|i| 0x1006000 + i).collect(); + let (extra_out, changed) = p.bind_many(&extra, &pressed); + assert!(changed); + for &(kc, _) in &extra_out { + assert_ne!(kc, held_kc, "held keycode must not be rebound"); + } + assert_eq!(p.resolve(held_sym), Some((held_kc, 0))); + } + + #[test] + fn rebuild_base_keeps_overlay_assignments() { + let mut p = policy(); + let (out, _) = p.bind_many(&[0x1004E2D], &HashSet::new()); + let de = compile_rmlvo("", "", "de", "", "").expect("de keymap"); + p.rebuild_base(de); + assert_eq!(p.resolve(0x1004E2D), Some((out[0].0, 0))); + // udiaeresis resolves in the German base without an overlay. + let (u_out, changed) = p.bind_many(&[0xFC], &HashSet::new()); + assert!(!changed); + assert_eq!(u_out[0].1, 0); + assert!(compile_keymap(&p.keymap_text()).is_some()); + } +} diff --git a/pixelflux/src/wayland/mod.rs b/pixelflux/src/wayland/mod.rs index f2d0679..08b1720 100644 --- a/pixelflux/src/wayland/mod.rs +++ b/pixelflux/src/wayland/mod.rs @@ -11,5 +11,7 @@ /// Headless Smithay compositor, protocol handlers, and input routing. pub mod frontend; -/// Wayland cursor shape to PNG resolution. +/// Wayland cursor shape to PNG resolution and the cursor delivery worker. pub mod cursor; +/// Seat keymap ownership: base layout plus batched overlay keysym binding. +pub mod keymap; diff --git a/pixelflux/src/x11/computer_use.rs b/pixelflux/src/x11/computer_use.rs new file mode 100644 index 0000000..52f4d11 --- /dev/null +++ b/pixelflux/src/x11/computer_use.rs @@ -0,0 +1,503 @@ +/* + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at https://mozilla.org/MPL/2.0/. + */ + +//! X11 backend for the Computer Use HTTP API: XTEST injection and one-shot root screenshots +//! over a private x11rb connection, so the agent can drive an X session with no active capture +//! and no shared state with any streaming capture thread (the X server itself serializes). +//! +//! Keysyms the active layout cannot produce (after base+AltGr resolution, which stays the +//! preferred path) are typed through a transient remap: under `XGrabServer` the keymap is +//! re-fetched, the needed keysyms are bound onto all-NoSymbol spare keycodes with ONE +//! `XChangeKeyboardMapping`, and the key sequence is injected; after a settle window (which +//! lets the focused client re-fetch the bound map — see `type_with_transient_binds`) one +//! conditional `XChangeKeyboardMapping` under a second grab puts the spares back — two +//! MappingNotify broadcasts per action. This coexists with a selkies session's own +//! spare-keycode overlay allocator on the same server: spares here are chosen DESCENDING +//! from the top of the keycode range while selkies allocates ASCENDING, each grab re-fetches +//! the keymap so every binding selkies already made is respected, the restore clears only +//! keycodes still carrying our content, and selkies' foreign-change invalidation (fired by +//! each MappingNotify) is self-healing because its held keys live on BOUND (non-NoSymbol) +//! keycodes an all-NoSymbol spare can never steal. + +use std::cell::RefCell; +use std::collections::HashMap; +use std::thread; +use std::time::Duration; + +use x11rb::connection::Connection; +use x11rb::protocol::xfixes::ConnectionExt as XfixesExt; +use x11rb::protocol::xproto::{ + ConnectionExt as XprotoExt, ImageFormat, BUTTON_PRESS_EVENT, BUTTON_RELEASE_EVENT, + KEY_PRESS_EVENT, KEY_RELEASE_EVENT, MOTION_NOTIFY_EVENT, +}; +use x11rb::protocol::xtest::ConnectionExt as XtestExt; +use x11rb::rust_connection::RustConnection; + +use crate::computer_use::{encode_png_rgba, CuBackend, CuButton}; + +/// One wheel "click" of scroll per unit of CU `scroll_amount`, capped so a hostile amount +/// cannot flood the server with press/release pairs. +const MAX_SCROLL_CLICKS: i32 = 100; + +const KEYSYM_ISO_LEVEL3_SHIFT: u32 = 0xfe03; +const KEYSYM_MODE_SWITCH: u32 = 0xff7e; + +/// Reverse view of the server's current keymap. +struct ServerKeymap { + /// keysym -> (keycode, shift level); level bit 0 = Shift, bit 1 = AltGr, lowest + /// level wins across ALL keys so a keysym reachable unshifted never resolves to a + /// modified position. + by_sym: HashMap, + /// Keycode carrying `ISO_Level3_Shift` (or, failing that, `Mode_switch`) in the core + /// map — the key synthesized around AltGr-level hits. 0 when the layout has neither, + /// in which case no AltGr levels are offered at all. + altgr_keycode: u32, +} + +pub struct CuX11Backend { + conn: RustConnection, + root: u32, + has_xfixes: bool, + /// Reverse map of the SERVER keymap, built lazily from `GetKeyboardMapping` and kept + /// for this backend's lifetime — one HTTP request (the backend is per-request), so a + /// `setxkbmap` switch is seen by the next request. + reverse_keymap: RefCell>, +} + +impl CuX11Backend { + /// Connect to the X server named by `DISPLAY` and negotiate XTEST (required for any + /// injection). XFixes is optional and only gates cursor compositing in screenshots. + pub fn connect() -> Result { + let (conn, screen_num) = + x11rb::connect(None).map_err(|e| format!("X11 connect failed: {e}"))?; + let root = conn.setup().roots[screen_num].root; + conn.xtest_get_version(2, 2) + .map_err(|e| format!("xtest_get_version: {e}"))? + .reply() + .map_err(|e| format!("XTEST unavailable: {e}"))?; + let has_xfixes = conn + .xfixes_query_version(5, 0) + .ok() + .and_then(|c| c.reply().ok()) + .is_some(); + Ok(Self { conn, root, has_xfixes, reverse_keymap: RefCell::new(None) }) + } + + /// Build the reverse view of the server's current keymap from `GetKeyboardMapping`. + /// + /// Column -> (group, level) convention, as observed on XKB-compat servers (Xvfb/Xorg + /// under `setxkbmap de`: keycode 24 = `q Q q Q at Greek_OMEGA at`): columns 0/1 are + /// group-1 plain/Shift, columns 2/3 mirror them as core group 2, and columns 4/5 + /// carry group-1 levels 3/4 (AltGr, Shift+AltGr). Columns 4/5 are consulted only + /// when the map carries a level-3 modifier key to synthesize; columns 2/3 (group 2) + /// never are. + fn build_reverse_keymap(&self) -> ServerKeymap { + let mut km = ServerKeymap { by_sym: HashMap::new(), altgr_keycode: 0 }; + let setup = self.conn.setup(); + let (lo, hi) = (setup.min_keycode, setup.max_keycode); + let Some(reply) = self + .conn + .get_keyboard_mapping(lo, hi - lo + 1) + .ok() + .and_then(|c| c.reply().ok()) + else { + return km; + }; + let per = reply.keysyms_per_keycode as usize; + if per == 0 { + return km; + } + let keycode_of = |wanted: u32| { + reply + .keysyms + .chunks_exact(per) + .position(|syms| syms.contains(&wanted)) + .map(|i| lo as u32 + i as u32) + }; + km.altgr_keycode = keycode_of(KEYSYM_ISO_LEVEL3_SHIFT) + .or_else(|| keycode_of(KEYSYM_MODE_SWITCH)) + .unwrap_or(0); + // Ascending level order so lower levels win; level bit 0 = Shift, bit 1 = AltGr. + let columns: [(usize, u32); 4] = [(0, 0), (1, 1), (4, 2), (5, 3)]; + for (col, level) in columns { + if col >= per || (level & 2 != 0 && km.altgr_keycode == 0) { + continue; + } + for (i, syms) in reply.keysyms.chunks_exact(per).enumerate() { + let sym = syms[col]; + if sym != 0 { + km.by_sym.entry(sym).or_insert((lo as u32 + i as u32, level)); + } + } + } + km + } + + /// Fire one XTEST fake event and flush so it reaches the server before the action + /// layer's inter-event pacing sleep, matching the immediacy of real input. + fn fake_input(&self, kind: u8, detail: u8, root: u32, x: i16, y: i16) { + let _ = self + .conn + .xtest_fake_input(kind, detail, x11rb::CURRENT_TIME, root, x, y, 0) + .map(|c| c.ignore_error()); + let _ = self.conn.flush(); + } + + fn root_geometry(&self) -> Result<(u16, u16), String> { + let geo = self + .conn + .get_geometry(self.root) + .map_err(|e| format!("get_geometry: {e}"))? + .reply() + .map_err(|e| format!("get_geometry reply: {e}"))?; + Ok((geo.width, geo.height)) + } + + /// One server round trip, forcing everything already sent on this connection to be + /// processed before the next request is issued (the XSync idiom). + fn sync(&self) -> Result<(), String> { + self.conn + .get_input_focus() + .map_err(|e| format!("sync: {e}"))? + .reply() + .map_err(|e| format!("sync reply: {e}"))?; + Ok(()) + } + + /// Bind `keysyms` onto spare keycodes and run `seq` with the keysym -> keycode map, + /// choose/bind/inject atomically under a server grab; then, after a settle window, + /// restore the spares under a second grab. The settle exists because clients + /// translate a keycode by RE-FETCHING the keymap when they process the bind's + /// MappingNotify — a fetch the grab itself blocks — so a restore issued inside the + /// first grab would be what they read back and every transient key would translate + /// to nothing (observed with xterm). Once `seq` has run, every failure is logged and + /// swallowed so the caller never re-runs the sequence. + fn type_with_transient_binds( + &self, + keysyms: &[u32], + seq: &mut dyn FnMut(&HashMap), + ) -> Result<(), String> { + let setup = self.conn.setup(); + let (lo, hi) = (setup.min_keycode, setup.max_keycode); + let mut chosen: Vec = Vec::with_capacity(keysyms.len()); + let (span_lo, count, per, bound_syms) = { + self.conn + .grab_server() + .map_err(|e| format!("grab_server: {e}"))?; + let _guard = ServerGrabGuard { conn: &self.conn }; + // Re-fetched UNDER the grab: a spare keycode any other client bound before + // the grab is visible as bound here and never chosen. + let reply = self + .conn + .get_keyboard_mapping(lo, hi - lo + 1) + .map_err(|e| format!("get_keyboard_mapping: {e}"))? + .reply() + .map_err(|e| format!("get_keyboard_mapping reply: {e}"))?; + let per = reply.keysyms_per_keycode as usize; + if per == 0 { + return Err("empty keymap".to_string()); + } + // Spares are all-NoSymbol keycodes taken DESCENDING from the top of the + // range. selkies' overlay allocator scans ASCENDING, so the two only meet + // when nearly every spare on the server is taken; and any held key + // (selkies' overlay binds included) lives on a BOUND, non-NoSymbol keycode, + // so an all-NoSymbol spare can never steal a key that is currently down. + for (i, syms) in reply.keysyms.chunks_exact(per).enumerate().rev() { + if syms.iter().all(|&s| s == 0) { + chosen.push(lo as u32 + i as u32); + if chosen.len() == keysyms.len() { + break; + } + } + } + if chosen.len() < keysyms.len() { + return Err(format!( + "only {} spare keycodes for {} unresolved keysyms", + chosen.len(), + keysyms.len() + )); + } + // ONE ChangeKeyboardMapping over the span from the lowest chosen spare + // upward. Every all-NoSymbol keycode above the lowest chosen one was itself + // chosen, so the span's other keycodes are bound ones, rewritten with their + // existing content (a content no-op). Each transient key carries its keysym + // at the plain and Shift levels so a stray held Shift cannot change what it + // types. + let span_lo = *chosen.last().unwrap(); + let span_hi = chosen[0]; + let base = ((span_lo - lo as u32) as usize) * per; + let end = ((span_hi - lo as u32) as usize + 1) * per; + let mut bound_syms = reply.keysyms[base..end].to_vec(); + let mut map = HashMap::new(); + for (&sym, &kc) in keysyms.iter().zip(chosen.iter()) { + let off = ((kc - span_lo) as usize) * per; + bound_syms[off] = sym; + if per > 1 { + bound_syms[off + 1] = sym; + } + map.insert(sym, kc); + } + let count = (span_hi - span_lo + 1) as u8; + self.conn + .change_keyboard_mapping(count, span_lo as u8, per as u8, &bound_syms) + .map_err(|e| format!("change_keyboard_mapping: {e}"))? + .check() + .map_err(|e| format!("change_keyboard_mapping check: {e}"))?; + // The binding must be live server-side before the first fake press resolves + // against it. + self.sync()?; + seq(&map); + (span_lo, count, per, bound_syms) + // Guard drops: ungrab + flush, releasing clients to process the injected + // events against the still-live bindings. + }; + // Settle: clients consume the queued MappingNotify + key events and re-fetch the + // BOUND map before the spares disappear again. + thread::sleep(TRANSIENT_BIND_SETTLE); + if let Err(e) = self.restore_transient_binds(span_lo, count, per, &bound_syms, &chosen) { + eprintln!("[ComputerUse] transient keysym restore failed: {e}"); + } + Ok(()) + } + + /// Return the transiently bound spares to all-NoSymbol with ONE conditional + /// `ChangeKeyboardMapping` under its own grab. The span is re-fetched under that + /// grab and only spares still carrying OUR content are cleared; a keycode another + /// client (selkies' allocator) claimed during the settle window keeps that client's + /// content — this restore can never clobber a foreign binding. + fn restore_transient_binds( + &self, + span_lo: u32, + count: u8, + per: usize, + bound_syms: &[u32], + chosen: &[u32], + ) -> Result<(), String> { + self.conn + .grab_server() + .map_err(|e| format!("grab_server: {e}"))?; + let _guard = ServerGrabGuard { conn: &self.conn }; + let reply = self + .conn + .get_keyboard_mapping(span_lo as u8, count) + .map_err(|e| format!("get_keyboard_mapping: {e}"))? + .reply() + .map_err(|e| format!("get_keyboard_mapping reply: {e}"))?; + let cur_per = reply.keysyms_per_keycode as usize; + if cur_per == 0 || per == 0 { + return Err("empty keymap".to_string()); + } + let mut restore = reply.keysyms.clone(); + let mut changed = false; + for &kc in chosen { + let sym = bound_syms[((kc - span_lo) as usize) * per]; + let cur = &mut restore[((kc - span_lo) as usize) * cur_per..][..cur_per]; + // Still ours when every populated level carries OUR keysym: the server's XKB + // integration mirrors a core single-group binding into the group-2 columns, + // so the refetch shows `sym` at more levels than the bind wrote. + let still_ours = + cur.iter().any(|&s| s == sym) && cur.iter().all(|&s| s == 0 || s == sym); + if still_ours { + cur.fill(0); + changed = true; + } + } + if changed { + self.conn + .change_keyboard_mapping(count, span_lo as u8, cur_per as u8, &restore) + .map_err(|e| format!("change_keyboard_mapping: {e}"))? + .check() + .map_err(|e| format!("change_keyboard_mapping check: {e}"))?; + } + Ok(()) + } +} + +/// How long transient binds outlive the injected key events before being restored: the +/// focused client has to wake up, see the bind's MappingNotify, and re-fetch the keymap +/// while the bindings are still live, or the presses translate against the restored map +/// and type nothing. +const TRANSIENT_BIND_SETTLE: Duration = Duration::from_millis(50); + +/// RAII server grab release: the grab is dropped (and the request flushed) on every exit +/// path, early error returns and panics included — a leaked server grab freezes every +/// client on the display. +struct ServerGrabGuard<'a> { + conn: &'a RustConnection, +} + +impl Drop for ServerGrabGuard<'_> { + fn drop(&mut self) { + let _ = self.conn.ungrab_server().map(|c| c.ignore_error()); + let _ = self.conn.flush(); + } +} + +impl Drop for CuX11Backend { + /// Closing the connection can race the server's client teardown against still-buffered + /// fake-input requests (observed as lost button releases); one round trip forces the + /// server to consume everything sent on this connection before it goes away. + fn drop(&mut self) { + if let Ok(cookie) = self.conn.get_input_focus() { + let _ = cookie.reply(); + } + } +} + +impl CuBackend for CuX11Backend { + fn name(&self) -> &'static str { + "x11" + } + + fn fb_size(&self) -> Result<(i32, i32), String> { + let (w, h) = self.root_geometry()?; + Ok((w as i32, h as i32)) + } + + fn key(&self, scancode: u32, pressed: bool) { + if scancode > u8::MAX as u32 { + return; + } + let kind = if pressed { KEY_PRESS_EVENT } else { KEY_RELEASE_EVENT }; + self.fake_input(kind, scancode as u8, x11rb::NONE, 0, 0); + } + + fn mouse_move(&self, x: f64, y: f64) { + // detail = 0 makes the motion absolute in root coordinates. + self.fake_input( + MOTION_NOTIFY_EVENT, + 0, + self.root, + x.round() as i16, + y.round() as i16, + ); + } + + fn button(&self, btn: CuButton, pressed: bool) { + let detail = match btn { + CuButton::Left => 1, + CuButton::Middle => 2, + CuButton::Right => 3, + }; + let kind = if pressed { BUTTON_PRESS_EVENT } else { BUTTON_RELEASE_EVENT }; + self.fake_input(kind, detail, x11rb::NONE, 0, 0); + } + + fn scroll(&self, dx: f64, dy: f64) { + // X has no smooth axis over XTEST: a scroll is N discrete clicks of the wheel + // buttons (4 = up, 5 = down, 6 = left, 7 = right), one press/release pair each. + let emit = |button: u8, clicks: i32| { + for _ in 0..clicks.min(MAX_SCROLL_CLICKS) { + self.fake_input(BUTTON_PRESS_EVENT, button, x11rb::NONE, 0, 0); + self.fake_input(BUTTON_RELEASE_EVENT, button, x11rb::NONE, 0, 0); + } + }; + let vy = dy.round() as i32; + let vx = dx.round() as i32; + if vy != 0 { + emit(if vy < 0 { 4 } else { 5 }, vy.abs()); + } + if vx != 0 { + emit(if vx < 0 { 6 } else { 7 }, vx.abs()); + } + } + + fn screenshot_png(&self, display: u32) -> Result, String> { + // One X server, one root: only display 0 exists on this backend. + if display != 0 { + return Err(format!("Unknown display: {display}")); + } + let (w, h) = self.root_geometry()?; + let img = self + .conn + .get_image(ImageFormat::Z_PIXMAP, self.root, 0, 0, w, h, !0u32) + .map_err(|e| format!("get_image: {e}"))? + .reply() + .map_err(|e| format!("get_image reply: {e}"))?; + let mut data = img.data; + let expected = w as usize * h as usize * 4; + if data.len() != expected { + return Err(format!( + "unexpected image size {} for {}x{} (only 32-bpp roots are supported)", + data.len(), w, h + )); + } + // The agent needs to see the pointer; the stream's cursor settings do not apply here. + if self.has_xfixes { + if let Some(c) = self + .conn + .xfixes_get_cursor_image() + .ok() + .and_then(|c| c.reply().ok()) + { + if c.width > 0 && c.height > 0 { + let (img_x, img_y) = + super::cursor_image_origin(c.x, c.y, c.xhot, c.yhot, 0, 0); + super::overlay_cursor( + &mut data, + w as usize * 4, + w as i32, + h as i32, + c.width as i32, + c.height as i32, + &c.cursor_image, + img_x, + img_y, + ); + } + } + } + // The grab is BGRX; the padding byte is undefined for depth-24 roots, so alpha is + // forced opaque or the PNG would come out transparent. + for px in data.chunks_exact_mut(4) { + px.swap(0, 2); + px[3] = 0xFF; + } + encode_png_rgba(&data, w as u32, h as u32) + } + + fn cursor_pos(&self) -> Result<(f64, f64), String> { + let ptr = self + .conn + .query_pointer(self.root) + .map_err(|e| format!("query_pointer: {e}"))? + .reply() + .map_err(|e| format!("query_pointer reply: {e}"))?; + Ok((ptr.root_x as f64, ptr.root_y as f64)) + } + + fn resolve_keysyms(&self, keysyms: &[u32]) -> Vec<(u32, u32)> { + let mut cached = self.reverse_keymap.borrow_mut(); + let km = cached.get_or_insert_with(|| self.build_reverse_keymap()); + keysyms + .iter() + .map(|sym| km.by_sym.get(sym).copied().unwrap_or((0, 0))) + .collect() + } + + fn altgr_keycode(&self) -> u32 { + let mut cached = self.reverse_keymap.borrow_mut(); + cached.get_or_insert_with(|| self.build_reverse_keymap()).altgr_keycode + } + + fn with_transient_keysyms(&self, keysyms: &[u32], seq: &mut dyn FnMut(&HashMap)) { + // Dedup defensively; a duplicate would burn a spare keycode for nothing. + let mut unique: Vec = Vec::with_capacity(keysyms.len()); + for &s in keysyms { + if s != 0 && !unique.contains(&s) { + unique.push(s); + } + } + if unique.is_empty() { + seq(&HashMap::new()); + return; + } + if let Err(e) = self.type_with_transient_binds(&unique, seq) { + eprintln!("[ComputerUse] transient keysym bind failed ({e}); typing without it"); + seq(&HashMap::new()); + } + } +} diff --git a/pixelflux/src/x11/mod.rs b/pixelflux/src/x11/mod.rs index 2a73268..386419c 100644 --- a/pixelflux/src/x11/mod.rs +++ b/pixelflux/src/x11/mod.rs @@ -39,6 +39,7 @@ use crate::pipeline::X11Pipeline; use crate::recording_sink::RecordingSink; use crate::RustCaptureSettings; +pub mod computer_use; pub mod cursor; /// Cross-thread controls for a running capture: a bag of atomics (plus two mutex-guarded @@ -290,7 +291,7 @@ fn blend_pixel(dst: &mut [u8], r: u8, g: u8, b: u8, a: u8) { /// The result may go negative near the frame edges, which `overlay_cursor` clips per pixel to match /// the server's own edge clipping. #[inline] -fn cursor_image_origin(x: i16, y: i16, xhot: u16, yhot: u16, cap_x: i32, cap_y: i32) -> (i32, i32) { +pub(crate) fn cursor_image_origin(x: i16, y: i16, xhot: u16, yhot: u16, cap_x: i32, cap_y: i32) -> (i32, i32) { (x as i32 - xhot as i32 - cap_x, y as i32 - yhot as i32 - cap_y) } @@ -298,7 +299,7 @@ fn cursor_image_origin(x: i16, y: i16, xhot: u16, yhot: u16, cap_x: i32, cap_y: /// at `(img_x, img_y)`, blending each pixel through `blend_pixel` with per-pixel bounds clipping so /// an image straddling a frame edge writes only its in-frame portion. #[allow(clippy::too_many_arguments)] -fn overlay_cursor( +pub(crate) fn overlay_cursor( frame: &mut [u8], stride: usize, frame_w: i32, @@ -719,7 +720,7 @@ where .is_some_and(|pl| pl.reshape(&psettings, size_changed)); if !reshaped { drop(pipeline.take()); - pipeline = Some(X11Pipeline::new(psettings.clone(), recording_sink.clone())); + pipeline = Some(X11Pipeline::new(psettings.clone())); if let Some(pl) = &pipeline { let enc_name = pl.encoder_name(); let mut log_msg = format!( @@ -749,7 +750,13 @@ where } let pl = pipeline.as_mut().unwrap(); - if controls.force_idr.swap(false, Ordering::Relaxed) { + // A recorder connecting to the socket sink needs a fresh decode entry point; it is + // folded into the same standard request-IDR path as a client-driven force_idr. + let sink_idr = recording_sink + .as_ref() + .map(|s| s.should_force_idr()) + .unwrap_or(false); + if controls.force_idr.swap(false, Ordering::Relaxed) || sink_idr { pl.request_idr(); } if controls.rate_dirty.swap(false, Ordering::Acquire) { From 98f204954eb68d62f3bc1108dc29dff7a02b3b58 Mon Sep 17 00:00:00 2001 From: Seungmin Kim <8457324+ehfd@users.noreply.github.com> Date: Mon, 20 Jul 2026 00:09:36 +0900 Subject: [PATCH 04/21] feat(wayland): Host capture mechanisms --- pixelflux/Cargo.toml | 6 +- pixelflux/src/computer_use.rs | 131 +++-- pixelflux/src/lib.rs | 324 ++++++++++- pixelflux/src/wayland/dcclient.rs | 372 +++++++++++++ pixelflux/src/wayland/frontend.rs | 39 +- pixelflux/src/wayland/host.rs | 866 ++++++++++++++++++++++++++++++ pixelflux/src/wayland/keymap.rs | 57 +- pixelflux/src/wayland/mod.rs | 8 + pixelflux/src/wayland/vkclient.rs | 157 ++++++ pixelflux/src/wayland/wlclient.rs | 201 +++++++ 10 files changed, 2107 insertions(+), 54 deletions(-) create mode 100644 pixelflux/src/wayland/dcclient.rs create mode 100644 pixelflux/src/wayland/host.rs create mode 100644 pixelflux/src/wayland/vkclient.rs create mode 100644 pixelflux/src/wayland/wlclient.rs diff --git a/pixelflux/Cargo.toml b/pixelflux/Cargo.toml index 4baaf5e..76fac70 100644 --- a/pixelflux/Cargo.toml +++ b/pixelflux/Cargo.toml @@ -10,7 +10,11 @@ crate-type = ["cdylib"] [dependencies] pyo3 = { version = "0.29.0", features = ["extension-module"] } wayland-server = { version = "0.31.10", features = ["dlopen"] } -wayland-protocols = { version = "0.32", features = ["server"] } +wayland-protocols = { version = "0.32", features = ["server", "client", "staging"] } +wayland-client = "0.31" +wayland-protocols-misc = { version = "0.3", features = ["client"] } +wayland-protocols-wlr = { version = "0.3", features = ["client"] } +memmap2 = "0.9" crossbeam-channel = "0.5" calloop = "0.14" turbojpeg = "1.3" diff --git a/pixelflux/src/computer_use.rs b/pixelflux/src/computer_use.rs index 18b8798..65cb8ec 100644 --- a/pixelflux/src/computer_use.rs +++ b/pixelflux/src/computer_use.rs @@ -30,6 +30,7 @@ use image::{ImageBuffer, Rgba, ImageFormat}; use serde::Deserialize; use tiny_http; +use crate::wayland::keymap::keysym_for_char; use crate::ThreadCommand; fn clamp(v: T, lo: T, hi: T) -> T { @@ -50,14 +51,6 @@ pub fn encode_png_rgba(data: &[u8], width: u32, height: u32) -> Result, Ok(png) } -/// Keysym for one literal character: Latin-1 printables map 1:1, control characters map -/// onto their `0xffXX` function keysyms, everything else onto the `0x01000000 | codepoint` -/// Unicode form (0 = unmappable). The keysym then resolves against the backend's ACTIVE -/// keymap, never a hardcoded layout table. -fn keysym_for_char(c: char) -> u32 { - xkb::utf32_to_keysym(c as u32).raw() -} - fn scancode_for_keyname(name: &str) -> Option { Some(match name.to_lowercase().as_str() { "return" | "enter" => 36, @@ -498,33 +491,49 @@ fn handle_action_inner(req: CuActionRequest, b: &dyn CuBackend) -> Result { let text = req.text.as_deref().ok_or("Missing text")?; + // A nested app compositor owns the apps on its own socket; type there as + // a virtual-keyboard client, since keys on pixelflux's own seat carry an + // overlay keymap the inner compositor never sees. Falls through to the + // local seat if the app socket is unreachable. + if b.name() == "wayland" { + if let Some(sock) = app_wayland_socket_path() { + // Failures log once per socket value; every request still + // retries, so a compositor that comes back is used again + // immediately (and re-arms the logging). + static FAILED_SOCK: Mutex> = Mutex::new(None); + match crate::wayland::vkclient::type_text_to(&sock, text) { + Ok(()) => { + *FAILED_SOCK.lock().unwrap() = None; + return Ok(ok_json()); + } + Err(e) => { + let mut last = FAILED_SOCK.lock().unwrap(); + if last.as_deref() != Some(sock.as_str()) { + eprintln!( + "[ComputerUse] app-compositor type via {sock} failed ({e}); using local seat until it is reachable" + ); + *last = Some(sock); + } + } + } + } + } let mut resolver = KeyResolver::new(b); let syms: Vec = text.chars().map(keysym_for_char).collect(); resolver.prefetch(&syms); // Base+AltGr resolution stays the preferred path; only what the active keymap // cannot reach at all goes through the backend's transient-bind fallback. let mut unresolved: Vec = Vec::new(); - for (ch, &sym) in text.chars().zip(&syms) { - if ch != '\n' - && sym != 0 - && resolver.resolve(sym).is_none() - && !unresolved.contains(&sym) - { + for &sym in &syms { + if sym != 0 && resolver.resolve(sym).is_none() && !unresolved.contains(&sym) { unresolved.push(sym); } } b.with_transient_keysyms(&unresolved, &mut |bound| { - for (i, (ch, &sym)) in text.chars().zip(&syms).enumerate() { + for (i, &sym) in syms.iter().enumerate() { if i > 0 && i % 50 == 0 { sleep_ms(20); } - if ch == '\n' { - b.key(36, true); - sleep_ms(10); - b.key(36, false); - sleep_ms(10); - continue; - } if let Some((sc, level)) = resolver.resolve(sym) { let level_mods = level_modifiers(level, 0, b.altgr_keycode()); for &m in &level_mods { @@ -798,6 +807,35 @@ fn handle_record_endpoint(url: &str, body: &str) -> Option { Some(reply) } +/// Socket of the compositor apps run under when it is nested under pixelflux +/// (labwc/kwin): keys injected into pixelflux's own seat carry an overlay keymap +/// the inner compositor never sees, so CU text is typed as a client of this +/// socket instead. Set by selkies over the ScreenCapture ABI. +static CU_APP_WAYLAND_DISPLAY: Mutex> = Mutex::new(None); + +/// Set (or clear, with None/empty) the app compositor socket for CU text injection. +pub fn set_app_wayland_display(display: Option) { + *CU_APP_WAYLAND_DISPLAY.lock().unwrap() = display.filter(|s| !s.is_empty()); +} + +/// Resolve the app compositor socket PATH for CU typing, or None to type on the +/// local seat. The ABI value selkies set wins; a standalone CU (no selkies) falls +/// back to PIXELFLUX_APP_WAYLAND_DISPLAY, then SELKIES_APP_WAYLAND_DISPLAY. A value +/// naming pixelflux's own compositor means nothing is nested. +fn app_wayland_socket_path() -> Option { + let name = CU_APP_WAYLAND_DISPLAY + .lock() + .unwrap() + .clone() + .or_else(|| std::env::var("PIXELFLUX_APP_WAYLAND_DISPLAY").ok()) + .or_else(|| std::env::var("SELKIES_APP_WAYLAND_DISPLAY").ok()) + .filter(|s| !s.is_empty())?; + if crate::wait_socket_name(Duration::from_millis(0)).as_deref() == Some(name.as_str()) { + return None; + } + crate::wayland::wlclient::socket_path(&name) +} + /// Make the Wayland compositor the preferred CU backend: once a live calloop sender is /// registered, every subsequent request routes to it instead of an X11 connection. pub fn register_wayland_backend(tx: smithay::reexports::calloop::channel::Sender) { @@ -811,21 +849,39 @@ pub(crate) fn wayland_command_sender( WAYLAND_TX.lock().unwrap().clone() } -/// Start the CU server if `PIXELFLUX_CU` names a port. Guarded so that exactly one server -/// binds per process no matter how many call sites (module import, Wayland compositor init) -/// race to spawn it; backend selection stays per-request, so a server bound at import serves -/// a compositor that only starts later. +/// Start the CU server if `PIXELFLUX_CU` names a bind (the standalone fallback; +/// a selkies-managed session passes the setting through [`start_cu_server`]). pub fn spawn_cu_from_env() { - let Ok(port_str) = std::env::var("PIXELFLUX_CU") else { return }; - let Ok(port) = port_str.parse::() else { - println!("[ComputerUse] Invalid PIXELFLUX_CU value: '{}' - expected a port number", port_str); - return; + if let Ok(bind) = std::env::var("PIXELFLUX_CU") { + start_cu_server(&bind); + } +} + +/// Start the CU server on `bind`: a bare port listens on all interfaces (the +/// container-deployment default), `host:port` scopes it. Guarded so that exactly +/// one server binds per process no matter how many call sites (module import, +/// Wayland compositor init, the selkies setting) race to spawn it; backend +/// selection stays per-request, so a server bound at import serves a compositor +/// that only starts later. +pub fn start_cu_server(bind: &str) { + let addr = if bind.contains(':') { + bind.to_string() + } else { + match bind.parse::() { + Ok(port) => format!("0.0.0.0:{port}"), + Err(_) => { + println!("[ComputerUse] Invalid bind '{bind}' - expected a port or host:port"); + return; + } + } }; static SPAWNED: OnceLock<()> = OnceLock::new(); let mut first = false; - SPAWNED.get_or_init(|| { first = true; }); + SPAWNED.get_or_init(|| { + first = true; + }); if first { - thread::spawn(move || run_cu_server(port)); + thread::spawn(move || run_cu_server(addr)); } } @@ -843,19 +899,16 @@ fn resolve_backend() -> Result, String> { /// Expose the captured desktop to an AI agent over HTTP, so a Computer Use client can drive /// the session much as a human viewer would. /// -/// It runs on its own thread listening on `0.0.0.0:` for POST `/computer-use` JSON actions +/// It runs on its own thread listening on `addr` for POST `/computer-use` JSON actions /// (screenshot, mouse_move, click, key, scroll, …). The backend and the framebuffer dimensions /// are re-resolved on every request rather than cached, because the compositor can start, the /// stream can resize, or the X server can restart underneath the agent — a stale size would /// misplace every coordinate. On Wayland a screenshot forces a one-frame GPU readback when the /// pipeline is in zero-copy mode; on X11 it is a one-shot `GetImage` of the root window. -pub fn run_cu_server(port: u16) { - println!( - "[ComputerUse] Server listening on port {}", - port, - ); +pub fn run_cu_server(addr: String) { + println!("[ComputerUse] Server listening on {}", addr); - let server = match tiny_http::Server::http(format!("0.0.0.0:{}", port)) { + let server = match tiny_http::Server::http(addr.as_str()) { Ok(s) => s, Err(e) => { eprintln!("[ComputerUse] Failed to start server: {}", e); diff --git a/pixelflux/src/lib.rs b/pixelflux/src/lib.rs index 0dce26a..90abe4d 100644 --- a/pixelflux/src/lib.rs +++ b/pixelflux/src/lib.rs @@ -248,7 +248,7 @@ smithay::backend::renderer::element::render_elements! { /// the buffer is described precisely enough (fd, stride, DRM modifier) for the importer to interpret /// it. One ARGB8888 plane is all that is carried because the compositor's offscreen target is exactly /// that single-plane format. -fn create_dmabuf_from_bo(bo: &BufferObject<()>) -> Dmabuf { +pub(crate) fn create_dmabuf_from_bo(bo: &BufferObject<()>) -> Dmabuf { let fd = bo.fd().expect("Failed to get FD from GBM BO"); let modifier = bo.modifier(); let stride = bo.stride(); @@ -308,6 +308,9 @@ pub struct RustCaptureSettings { pub debug_logging: bool, pub auto_adjust_screen_capture_size: bool, pub recording_socket: String, + /// Wayland display of an EXTERNAL compositor to capture (host-capture mode); + /// empty composites own clients as usual. + pub wayland_host_display: String, /// When true, encoders emit the raw payload without the per-stripe header byte block; /// stripe metadata is then carried only on the frame attributes. pub omit_stripe_headers: bool, @@ -411,6 +414,7 @@ impl Default for RustCaptureSettings { debug_logging: false, auto_adjust_screen_capture_size: false, recording_socket: String::new(), + wayland_host_display: String::new(), omit_stripe_headers: false, video_cbr_mode: false, video_bitrate_kbps: 4000, @@ -501,6 +505,11 @@ pub(crate) fn extract_settings(settings: &Bound<'_, PyAny>) -> PyResult().ok()) .unwrap_or_default(), + wayland_host_display: settings + .getattr("wayland_host_display") + .ok() + .and_then(|v| v.extract::().ok()) + .unwrap_or_default(), omit_stripe_headers: settings .getattr("omit_stripe_headers") .ok() @@ -1535,6 +1544,45 @@ fn start_capture_on_display( ) { use smithay::wayland::fractional_scale::with_fractional_scale; + // Host-capture mode (primary display): connect on first use and point the + // capture thread at the requested size. The compositor keeps running (CU, + // clipboard callbacks, input fallbacks) but its renderer is bypassed. + if display_id == 0 && !settings.wayland_host_display.is_empty() { + if state.host.is_none() { + // The compositor's own allocator device: capture buffers come from the + // same render node the encoder imports from (reopened via its live fd, + // since the resolved path string is not retained in auto mode). + let gbm = if state.use_gpu { + state.gbm_device.as_ref().and_then(|dev| { + use std::os::fd::{AsFd as _, AsRawFd as _}; + let fd = dev.as_fd().as_raw_fd(); + std::fs::read_link(format!("/proc/self/fd/{fd}")) + .ok() + .and_then(|p| std::fs::File::options().read(true).write(true).open(p).ok()) + .and_then(|f| GbmDevice::new(f).ok()) + }) + } else { + None + }; + match crate::wayland::host::HostSession::connect(&settings.wayland_host_display, gbm) { + Ok(h) => { + println!( + "[HostCapture] capturing host compositor '{}'.", + settings.wayland_host_display + ); + state.host = Some(h); + } + Err(e) => eprintln!( + "[HostCapture] connect '{}' failed: {e}", + settings.wayland_host_display + ), + } + } + if let Some(host) = &state.host { + host.start_capture(settings.width, settings.height); + } + } + let Some(node_idx) = state.node_idx_for_id(display_id) else { eprintln!("[Wayland] StartCapture: no output with display id {display_id}."); return; @@ -1967,6 +2015,15 @@ fn render_node_tick( } } let requested_idr = node.capture.as_ref().map(|c| c.pending_force_idr).unwrap_or(false); + // A client keyframe request lands on the hardware path's atomic (RequestIdr sets + // it whenever an encode pool exists); host mode consults it — without consuming — + // to decide whether a static screen must re-encode its retained frame. + let hw_idr_pending = node + .capture + .as_ref() + .map(|c| c.encode_controls.force_idr.load(Ordering::Relaxed)) + .unwrap_or(false); + let want_idr_for_host = requested_idr || hw_idr_pending; let mut pool_slot: Option<(usize, Vec)> = None; if let Some(cap) = node.capture.as_ref() { @@ -2014,7 +2071,84 @@ fn render_node_tick( let mut damage_rects: Vec> = Vec::new(); let needs_full = node.capture.as_ref().map(|c| c.needs_full_render).unwrap_or(true); - if state.use_gpu { + // Host-capture mode: the host compositor already blitted this display's frame + // into one of our buffers (screencopy); adopt it in place of compositing. + let host_mode = state.host.is_some() && node.id == 0; + // Dmabuf handed to the GPU encoder in host mode (from the new or retained frame). + let mut host_enc_dmabuf: Option = None; + if host_mode { + let gpu_encoder = node + .capture + .as_ref() + .map(|c| c.video_encoder.is_some()) + .unwrap_or(false); + let new_frame = state.host.as_ref().unwrap().try_take_frame(); + let have_new = new_frame.is_some(); + // Streaming mode wants a constant-rate stream (the client's decoder pipeline + // is built for it), so re-encode the retained frame every tick like the + // compositor path does. Outside streaming mode, stay damage-driven and only + // re-encode when an IDR is pending (a viewer opening its keyframe gate). + let streaming = node + .capture + .as_ref() + .map(|c| c.settings.video_streaming_mode) + .unwrap_or(false); + if !have_new && !want_idr_for_host && !streaming { + if let Some((id, buf)) = pool_slot.take() { + if let Some(cap) = node.capture.as_ref() { + if let Some(ref pool) = cap.encode_pool { + pool.cancel(id, buf); + } + } + } + return false; + } + if let Some(f) = new_frame { + state.host.as_ref().unwrap().retain_frame(f); + } + // Consume the (new or prior) retained frame's content into the pool slot / + // GPU dmabuf. The frame stays retained for the next IDR. + let bad_combo = state.host.as_ref().unwrap().with_retained(|r| { + let Some(f) = r else { return true }; + damage_rects = f.damage.clone(); + if let Some(cpu) = f.cpu.as_ref() { + if gpu_encoder { + return true; // software frames, GPU encoder — mismatch + } + if let Some((_, ref mut buf)) = pool_slot { + cpu.write_bgra(f.width, f.height, buf); + } + cpu.write_bgra(f.width, f.height, &mut node.frame_buffer); + } else if let Some(dmabuf) = f.dmabuf.as_ref() { + if !gpu_encoder { + return true; // GPU frames, CPU encoder — mismatch + } + host_enc_dmabuf = Some(dmabuf.clone()); + } + false + }); + if bad_combo { + static WARNED: std::sync::atomic::AtomicBool = + std::sync::atomic::AtomicBool::new(false); + if !WARNED.swap(true, Ordering::Relaxed) { + eprintln!( + "[HostCapture] host frame type and encoder mismatch \ + (GPU host needs a GPU encoder; software host needs a CPU encoder)." + ); + } + if let Some((id, buf)) = pool_slot.take() { + if let Some(cap) = node.capture.as_ref() { + if let Some(ref pool) = cap.encode_pool { + pool.cancel(id, buf); + } + } + } + return false; + } + render_success = true; + } + + if !host_mode && state.use_gpu { if let Some(renderer) = state.gles_renderer.as_mut() { let mut cap = node.capture.as_mut(); if let Some((_bo, dmabuf)) = node.offscreen_buffer.as_mut() { @@ -2193,7 +2327,7 @@ fn render_node_tick( } } } - } else { + } else if !host_mode { if let Some(renderer) = state.pixman_renderer.as_mut() { let mut cap = node.capture.as_mut(); let (ptr, buf_age) = match pool_slot { @@ -2436,16 +2570,21 @@ fn render_node_tick( if let Some(sync) = render_sync.take() { let _ = sync.wait(); } + // Host-capture frames encode from the buffer the host blitted + // into; otherwise from this display's own composited buffer. + let enc_dmabuf: Option = host_enc_dmabuf + .clone() + .or_else(|| node.offscreen_buffer.as_ref().map(|(_, d)| d.clone())); let result = match encoder { GpuEncoder::Nvenc(enc) => { - if let Some((_, ref dmabuf)) = node.offscreen_buffer { + if let Some(ref dmabuf) = enc_dmabuf { enc.encode(dmabuf, cap.frame_counter as u64, target_qp, force_idr) } else { Err("NVENC ZeroCopy requires offscreen buffer (GPU context)".to_string()) } }, GpuEncoder::Vaapi(enc) => { - if let Some((_, ref dmabuf)) = node.offscreen_buffer { + if let Some(ref dmabuf) = enc_dmabuf { enc.encode_dmabuf(dmabuf, cap.frame_counter as u64, target_qp, force_idr) } else { Err("Vaapi ZeroCopy requires offscreen buffer (GPU context)".to_string()) @@ -2658,6 +2797,33 @@ fn create_output_on( overlay_state: OverlayState::default(), capture: None, }); + // A nested session opens one host toplevel per screen and stacks the extras + // on an existing output until a display exists for them: hand the newest + // stacked window to the new output. + let mut counts: Vec<(u32, usize)> = Vec::new(); + for w in state.space.elements() { + let oid = wayland::frontend::window_output_id(w); + match counts.iter_mut().find(|(o, _)| *o == oid) { + Some((_, c)) => *c += 1, + None => counts.push((oid, 1)), + } + } + let adopt = state + .space + .elements() + .filter(|w| { + let oid = wayland::frontend::window_output_id(w); + counts.iter().any(|(o, c)| *o == oid && *c >= 2) + }) + .max_by_key(|w| wayland::frontend::window_meta(w).map(|m| m.id).unwrap_or(0)) + .cloned(); + if let Some(window) = adopt { + state.place_window_on_output(&window, id); + println!( + "[Wayland] Output {id}: adopted stacked window {}.", + wayland::frontend::window_meta(&window).map(|m| m.id).unwrap_or(0) + ); + } true } @@ -2979,6 +3145,7 @@ fn run_wayland_thread( use_gpu, cursor_helper: Cursor::load(cursor_size), keymap_policy: wayland::keymap::KeymapPolicy::empty(), + host: None, current_cursor_icon: None, cursor_buffer: None, render_cursor_on_framebuffer: false, @@ -3174,6 +3341,10 @@ fn run_wayland_thread( } } ThreadCommand::KeyboardKey { scancode, state: key_state_val } => { + if let Some(host) = state.host.as_ref() { + host.key(scancode, key_state_val > 0); + return; + } let key_state = if key_state_val > 0 { KeyState::Pressed } else { KeyState::Released }; let serial = next_serial(); let time = wayland_time(); @@ -3251,6 +3422,10 @@ fn run_wayland_thread( let _ = reply.send(keymap_str); } ThreadCommand::PointerMotion { x, y } => { + if let Some(host) = state.host.as_ref() { + host.pointer_motion_abs(x, y); + return; + } let serial = next_serial(); let time = wayland_time(); // (x, y) are physical union-layout coordinates: each output occupies @@ -3342,6 +3517,10 @@ fn run_wayland_thread( } } ThreadCommand::PointerButton { btn, state: btn_state_val } => { + if let Some(host) = state.host.as_ref() { + host.pointer_button(btn, btn_state_val > 0); + return; + } let serial = next_serial(); let time = wayland_time(); let button_state = if btn_state_val > 0 { smithay::backend::input::ButtonState::Pressed } else { smithay::backend::input::ButtonState::Released }; @@ -3364,6 +3543,10 @@ fn run_wayland_thread( } } ThreadCommand::PointerAxis { x, y } => { + if let Some(host) = state.host.as_ref() { + host.pointer_axis(x, y); + return; + } let time = wayland_time(); if let Some(pointer) = state.seat.get_pointer() { @@ -3587,6 +3770,24 @@ fn run_wayland_thread( let any_capturing = state.output_nodes.iter().any(|n| n.capture.is_some()); if !any_capturing && state.pending_screenshot.is_none() { + // No render/encode work, but committed clients still need their + // frame callbacks: a vsynced client (FIFO Vulkan present, games, a + // nested compositor's own clients) otherwise blocks in its swap + // until a viewer attaches — apps appear frozen whenever nobody is + // watching. + let time = state.clock.now(); + for node in &state.output_nodes { + for window in state + .space + .elements_for_output(&node.output) + .cloned() + .collect::>() + { + window.send_frame(&node.output, time, Some(Duration::ZERO), |_, _| { + Some(node.output.clone()) + }); + } + } return TimeoutAction::ToDuration(Duration::from_millis(16)); } @@ -4128,6 +4329,8 @@ struct CaptureSettings { #[pyo3(get, set)] use_wayland: Py, /// H.264 recording tap: a Unix socket path to bind, or empty for none. #[pyo3(get, set)] recording_socket: Py, + /// Wayland display of an EXTERNAL compositor to capture (host-capture mode). + #[pyo3(get, set)] wayland_host_display: Py, /// Compositor cursor-theme size in pixels; <=0 keeps the theme default (24). #[pyo3(get, set)] cursor_size: i32, /// Longest cursor edge the X11 out-of-band cursor callback delivers; larger @@ -4155,7 +4358,8 @@ impl CaptureSettings { auto_adjust_screen_capture_size: false, omit_stripe_headers: false, encode_node_path: py.None(), render_node_path: py.None(), auto_gpu: py.None(), use_wayland: py.None(), - recording_socket: py.None(), cursor_size: -1, cursor_size_cap: 32, + recording_socket: py.None(), wayland_host_display: py.None(), + cursor_size: -1, cursor_size_cap: 32, } } } @@ -4860,6 +5064,88 @@ impl ScreenCapture { fn set_keymap_string(&self, py: Python<'_>, text: String) -> PyResult<()> { wayland_backend_running(py).map_or(Ok(()), |be| be.bind(py).borrow().set_keymap_string(text)) } + /// Compositor apps run under (a nested labwc/kwin session pixelflux captures), + /// the target for Computer-Use text injection; selkies resolves it and hands it + /// over. Empty clears it. Stored process-wide since the CU server is per-process. + fn set_app_wayland_display(&self, display: String) { + crate::computer_use::set_app_wayland_display( + if display.is_empty() { None } else { Some(display) }, + ); + } + /// Type `text` through `display`'s zwp_virtual_keyboard_manager_v1 as a one-shot + /// client: selkies' text-injection path, targeting whichever compositor the apps + /// live under (the nested session's, or pixelflux's own in a direct session). + /// Blocking; releases the GIL for the duration. Raises + /// [`VirtualKeyboardUnavailable`] when the compositor lacks the protocol. + fn type_text_wayland(&self, py: Python<'_>, display: String, text: String) -> PyResult<()> { + py.detach(move || { + let path = crate::wayland::wlclient::socket_path(&display) + .ok_or_else(|| "XDG_RUNTIME_DIR is unset".to_string())?; + crate::wayland::vkclient::type_text_to(&path, &text) + }) + .map_err(|e: String| { + if e.contains("zwp_virtual_keyboard_manager_v1") { + VirtualKeyboardUnavailable::new_err(e) + } else { + pyo3::exceptions::PyRuntimeError::new_err(e) + } + }) + } + /// Mimes the app compositor's current selection offers (empty = nothing copied). + fn clipboard_types_app(&self, py: Python<'_>, display: String) -> PyResult> { + py.detach(move || { + crate::wayland::dcclient::list_types(&app_socket_path(&display)?) + }) + .map_err(pyo3::exceptions::PyRuntimeError::new_err) + } + /// The app compositor selection's payload for `mime`, or None when nothing is + /// copied or the selection does not offer that mime. + fn clipboard_read_app( + &self, + py: Python<'_>, + display: String, + mime: String, + ) -> PyResult>> { + let data = py + .detach(move || crate::wayland::dcclient::read(&app_socket_path(&display)?, &mime)) + .map_err(pyo3::exceptions::PyRuntimeError::new_err)?; + Ok(data.map(|d| pyo3::types::PyBytes::new(py, &d).unbind())) + } + /// Take the app compositor's selection, serving `entries` (mime, bytes) to + /// every paster from a background thread until another client copies. + fn clipboard_write_app( + &self, + py: Python<'_>, + display: String, + entries: Vec<(String, Vec)>, + ) -> PyResult<()> { + py.detach(move || crate::wayland::dcclient::write(&app_socket_path(&display)?, entries)) + .map_err(pyo3::exceptions::PyRuntimeError::new_err) + } + /// Drop the app compositor's selection. + fn clipboard_clear_app(&self, py: Python<'_>, display: String) -> PyResult<()> { + py.detach(move || crate::wayland::dcclient::clear(&app_socket_path(&display)?)) + .map_err(pyo3::exceptions::PyRuntimeError::new_err) + } + /// Invoke `callback(mimes: list[str])` from a background thread on every + /// selection change in the app compositor (including the one current at call + /// time). A second watch for the same display replaces the first. + fn clipboard_watch_app( + &self, + py: Python<'_>, + display: String, + callback: Py, + ) -> PyResult<()> { + py.detach(move || crate::wayland::dcclient::watch(&app_socket_path(&display)?, callback)) + .map_err(pyo3::exceptions::PyRuntimeError::new_err) + } + /// Stop the selection watch for `display` (no-op without one). + fn clipboard_unwatch_app(&self, py: Python<'_>, display: String) { + let _ = py.detach(move || { + crate::wayland::dcclient::unwatch(&app_socket_path(&display)?); + Ok::<(), String>(()) + }); + } fn inject_mouse_move(&self, py: Python<'_>, x: f64, y: f64) -> PyResult<()> { wayland_backend_running(py).map_or(Ok(()), |be| be.bind(py).borrow().inject_mouse_move(x, y)) } @@ -5194,6 +5480,27 @@ fn _stop_all_captures(py: Python<'_>) { /// The `pixelflux` Python module: registers the exported classes and functions, and hooks /// `_stop_all_captures` into `atexit` so every live capture is stopped before interpreter shutdown. +/// Socket path for a Wayland display name, with the ABI methods' error shape. +fn app_socket_path(display: &str) -> Result { + crate::wayland::wlclient::socket_path(display) + .ok_or_else(|| "XDG_RUNTIME_DIR is unset".to_string()) +} + +pyo3::create_exception!( + pixelflux, + VirtualKeyboardUnavailable, + pyo3::exceptions::PyRuntimeError, + "The target compositor does not advertise zwp_virtual_keyboard_manager_v1." +); + +/// Start the Computer-Use HTTP server: a bare port listens on all interfaces, +/// `host:port` scopes it. Idempotent; the PIXELFLUX_CU env var remains the +/// standalone fallback. +#[pyfunction] +fn start_computer_use(bind: String) { + crate::computer_use::start_cu_server(&bind); +} + #[pymodule] fn pixelflux(m: &Bound<'_, PyModule>) -> PyResult<()> { m.add_class::()?; @@ -5208,6 +5515,11 @@ fn pixelflux(m: &Bound<'_, PyModule>) -> PyResult<()> { m.add_function(wrap_pyfunction!(start_recording, m)?)?; m.add_function(wrap_pyfunction!(stop_recording, m)?)?; m.add_function(wrap_pyfunction!(recording_status, m)?)?; + m.add_function(wrap_pyfunction!(start_computer_use, m)?)?; + m.add( + "VirtualKeyboardUnavailable", + m.py().get_type::(), + )?; m.add_function(wrap_pyfunction!(_stop_all_captures, m)?)?; if let Ok(atexit) = m.py().import("atexit") { let _ = atexit.call_method1("register", (m.getattr("_stop_all_captures")?,)); diff --git a/pixelflux/src/wayland/dcclient.rs b/pixelflux/src/wayland/dcclient.rs new file mode 100644 index 0000000..0756662 --- /dev/null +++ b/pixelflux/src/wayland/dcclient.rs @@ -0,0 +1,372 @@ +//! `zwlr_data_control_v1` CLIENT: the native clipboard bridge to a NESTED app +//! compositor (labwc/kwin running under pixelflux). +//! +//! Apps in a nested session use the inner compositor's selection, which +//! pixelflux's own clipboard machinery never sees. selkies bridges it over the +//! ScreenCapture ABI backed by this module instead of forking wl-copy/wl-paste +//! per operation: one-shot [`list_types`]/[`read`], a persistent-source [`write`] +//! (a detached thread serves paste requests until another client takes the +//! selection), and [`watch`] (a thread reporting selection changes to a Python +//! callback). Every compositor round-trip is deadline-bounded via +//! [`wlclient::bounded_roundtrip`]. + +use std::collections::HashMap; +use std::os::fd::AsFd; +use std::os::unix::net::UnixStream; +use std::sync::atomic::{AtomicBool, Ordering}; +use std::sync::{Arc, Mutex}; +use std::time::Duration; + +use pyo3::{Py, PyAny, Python}; +use wayland_client::backend::ObjectId; +use wayland_client::protocol::{wl_registry, wl_seat}; +use wayland_client::{delegate_noop, Connection, Dispatch, Proxy, QueueHandle}; +use wayland_protocols_wlr::data_control::v1::client::{ + zwlr_data_control_device_v1::{self, ZwlrDataControlDeviceV1}, + zwlr_data_control_manager_v1::ZwlrDataControlManagerV1, + zwlr_data_control_offer_v1::{self, ZwlrDataControlOfferV1}, + zwlr_data_control_source_v1::{self, ZwlrDataControlSourceV1}, +}; + +use crate::wayland::wlclient::{ + bounded_roundtrip, impl_sync_callback, pipe_cloexec, read_fd_to_end, wait_readable, + write_fd_all, SyncState, IO_TIMEOUT, +}; + +/// How often a background thread wakes from its socket poll to check its stop +/// flag, bounding unwatch/shutdown latency. +const STOP_POLL: Duration = Duration::from_millis(500); + +#[derive(Default)] +struct DcState { + seat: Option, + manager: Option, + /// Advertised mimes per live offer. + offer_mimes: HashMap>, + selection: Option, + /// Set on every `selection` event (the watch loop's change edge). + selection_changed: bool, + /// Compositor told this device it is done (seat gone). + finished: bool, + /// The write path's source lost the selection to another client. + cancelled: bool, + /// Mime -> bytes served by the write path's source. + serve: Vec<(String, Vec)>, + sync_done: bool, +} + +impl SyncState for DcState { + fn sync_done_mut(&mut self) -> &mut bool { + &mut self.sync_done + } +} +impl_sync_callback!(DcState); + +impl Dispatch for DcState { + fn event( + state: &mut Self, + registry: &wl_registry::WlRegistry, + event: wl_registry::Event, + _: &(), + _: &Connection, + qh: &QueueHandle, + ) { + if let wl_registry::Event::Global { name, interface, .. } = event { + // Version 1 of each suffices: the seat is only an argument, and v1 + // data-control carries the regular selection this bridge needs. + match interface.as_str() { + "wl_seat" if state.seat.is_none() => { + state.seat = Some(registry.bind(name, 1, qh, ())); + } + "zwlr_data_control_manager_v1" if state.manager.is_none() => { + state.manager = Some(registry.bind(name, 1, qh, ())); + } + _ => {} + } + } + } +} + +delegate_noop!(DcState: ignore wl_seat::WlSeat); +delegate_noop!(DcState: ZwlrDataControlManagerV1); + +impl Dispatch for DcState { + fn event( + state: &mut Self, + offer: &ZwlrDataControlOfferV1, + event: zwlr_data_control_offer_v1::Event, + _: &(), + _: &Connection, + _: &QueueHandle, + ) { + if let zwlr_data_control_offer_v1::Event::Offer { mime_type } = event { + state.offer_mimes.entry(offer.id()).or_default().push(mime_type); + } + } +} + +impl Dispatch for DcState { + fn event( + state: &mut Self, + _: &ZwlrDataControlDeviceV1, + event: zwlr_data_control_device_v1::Event, + _: &(), + _: &Connection, + _: &QueueHandle, + ) { + match event { + zwlr_data_control_device_v1::Event::DataOffer { id } => { + state.offer_mimes.entry(id.id()).or_default(); + } + zwlr_data_control_device_v1::Event::Selection { id } => { + // Replaced offers are dead objects; drop their proxy and mimes so + // a long-lived watch connection doesn't accumulate them. + if let Some(old) = state.selection.take() { + if id.as_ref().map(|o| o.id()) != Some(old.id()) { + state.offer_mimes.remove(&old.id()); + old.destroy(); + } + } + state.selection = id; + state.selection_changed = true; + } + zwlr_data_control_device_v1::Event::Finished => { + state.finished = true; + } + _ => {} + } + } + + wayland_client::event_created_child!(DcState, ZwlrDataControlDeviceV1, [ + zwlr_data_control_device_v1::EVT_DATA_OFFER_OPCODE => (ZwlrDataControlOfferV1, ()), + ]); +} + +impl Dispatch for DcState { + fn event( + state: &mut Self, + _: &ZwlrDataControlSourceV1, + event: zwlr_data_control_source_v1::Event, + _: &(), + _: &Connection, + _: &QueueHandle, + ) { + match event { + zwlr_data_control_source_v1::Event::Send { mime_type, fd } => { + if let Some((_, data)) = state.serve.iter().find(|(m, _)| *m == mime_type) { + let _ = write_fd_all(&fd, data, IO_TIMEOUT); + } + // fd drops here, closing the pipe so the paster sees EOF. + } + zwlr_data_control_source_v1::Event::Cancelled => { + state.cancelled = true; + } + _ => {} + } + } +} + +/// Connect to `socket_path` and return (connection, queue, state, device) with +/// the current selection already delivered. +fn open_device( + socket_path: &str, +) -> Result< + (Connection, wayland_client::EventQueue, DcState, ZwlrDataControlDeviceV1), + String, +> { + let stream = + UnixStream::connect(socket_path).map_err(|e| format!("connect {socket_path}: {e}"))?; + let conn = Connection::from_socket(stream).map_err(|e| format!("wayland setup: {e}"))?; + let mut queue = conn.new_event_queue(); + let qh = queue.handle(); + let _registry = conn.display().get_registry(&qh, ()); + let mut state = DcState::default(); + bounded_roundtrip(&conn, &mut queue, &mut state)?; + let seat = state.seat.clone().ok_or("app compositor advertises no wl_seat")?; + let manager = state + .manager + .clone() + .ok_or("app compositor does not advertise zwlr_data_control_manager_v1")?; + let device = manager.get_data_device(&seat, &qh, ()); + bounded_roundtrip(&conn, &mut queue, &mut state)?; + Ok((conn, queue, state, device)) +} + +/// Mimes offered by the current selection (empty when nothing is copied). +pub(crate) fn list_types(socket_path: &str) -> Result, String> { + let (_conn, _queue, state, device) = open_device(socket_path)?; + let out = state + .selection + .as_ref() + .and_then(|o| state.offer_mimes.get(&o.id()).cloned()) + .unwrap_or_default(); + device.destroy(); + Ok(out) +} + +/// The current selection's payload for `mime`, or None when there is no +/// selection or it does not offer that mime. +pub(crate) fn read(socket_path: &str, mime: &str) -> Result>, String> { + let (conn, mut queue, mut state, device) = open_device(socket_path)?; + let Some(offer) = state.selection.clone() else { + device.destroy(); + return Ok(None); + }; + let offered = state.offer_mimes.get(&offer.id()).map_or(false, |m| m.iter().any(|x| x == mime)); + if !offered { + device.destroy(); + return Ok(None); + } + let (rd, wr) = pipe_cloexec()?; + offer.receive(mime.to_string(), wr.as_fd()); + queue.flush().map_err(|e| format!("flush: {e}"))?; + drop(wr); + // The source app writes into the pipe as it pleases; dispatch is not needed + // for the bytes, only the fd read. + let data = read_fd_to_end(&rd, IO_TIMEOUT)?; + let _ = bounded_roundtrip(&conn, &mut queue, &mut state); + device.destroy(); + Ok(Some(data)) +} + +/// Take the selection, serving `entries` (mime, bytes) to every paster from a +/// detached thread until another client takes the selection (or the compositor +/// goes away). Replacing a previous write is implicit: the compositor cancels +/// the old source when the new one takes the selection. +pub(crate) fn write(socket_path: &str, entries: Vec<(String, Vec)>) -> Result<(), String> { + let path = socket_path.to_string(); + std::thread::Builder::new() + .name("pf-dc-selection".into()) + .spawn(move || { + if let Err(e) = serve_selection(&path, entries) { + eprintln!("[Clipboard] app-compositor selection serve ended: {e}"); + } + }) + .map_err(|e| format!("spawn: {e}"))?; + Ok(()) +} + +fn serve_selection(socket_path: &str, entries: Vec<(String, Vec)>) -> Result<(), String> { + let stream = + UnixStream::connect(socket_path).map_err(|e| format!("connect {socket_path}: {e}"))?; + let conn = Connection::from_socket(stream).map_err(|e| format!("wayland setup: {e}"))?; + let mut queue = conn.new_event_queue(); + let qh = queue.handle(); + let _registry = conn.display().get_registry(&qh, ()); + let mut state = DcState::default(); + bounded_roundtrip(&conn, &mut queue, &mut state)?; + let seat = state.seat.clone().ok_or("app compositor advertises no wl_seat")?; + let manager = state + .manager + .clone() + .ok_or("app compositor does not advertise zwlr_data_control_manager_v1")?; + let device = manager.get_data_device(&seat, &qh, ()); + let source = manager.create_data_source(&qh, ()); + for (mime, _) in &entries { + source.offer(mime.clone()); + } + state.serve = entries; + device.set_selection(Some(&source)); + bounded_roundtrip(&conn, &mut queue, &mut state)?; + while !state.cancelled && !state.finished { + // Sends are served inside dispatch; block until the compositor has + // something (with a poll so a dead compositor can't pin the thread). + queue.flush().map_err(|e| format!("flush: {e}"))?; + match conn.prepare_read() { + Some(guard) => { + use std::os::fd::AsRawFd; + if wait_readable(guard.connection_fd().as_raw_fd(), STOP_POLL)? { + guard.read().map_err(|e| format!("read: {e}"))?; + } + } + None => {} + } + queue.dispatch_pending(&mut state).map_err(|e| format!("dispatch: {e}"))?; + } + source.destroy(); + device.destroy(); + let _ = queue.flush(); + Ok(()) +} + +/// Drop the selection (the compositor also cancels whatever source held it). +pub(crate) fn clear(socket_path: &str) -> Result<(), String> { + let (conn, mut queue, mut state, device) = open_device(socket_path)?; + device.set_selection(None); + bounded_roundtrip(&conn, &mut queue, &mut state)?; + device.destroy(); + Ok(()) +} + +struct WatchHandle { + stop: Arc, +} + +static WATCHERS: Mutex>> = Mutex::new(None); + +/// Report every selection change on `socket_path` (including the one current at +/// start) to `callback(mimes: list[str])` from a background thread. A second +/// watch on the same socket replaces the first. +pub(crate) fn watch(socket_path: &str, callback: Py) -> Result<(), String> { + let stop = Arc::new(AtomicBool::new(false)); + { + let mut reg = WATCHERS.lock().unwrap(); + let map = reg.get_or_insert_with(HashMap::new); + if let Some(old) = map.insert(socket_path.to_string(), WatchHandle { stop: stop.clone() }) + { + old.stop.store(true, Ordering::Relaxed); + } + } + let path = socket_path.to_string(); + std::thread::Builder::new() + .name("pf-dc-watch".into()) + .spawn(move || { + if let Err(e) = watch_loop(&path, callback, &stop) { + eprintln!("[Clipboard] app-compositor watch ended: {e}"); + } + }) + .map_err(|e| format!("spawn: {e}"))?; + Ok(()) +} + +/// Stop the watch on `socket_path` (no-op when none is running). +pub(crate) fn unwatch(socket_path: &str) { + let mut reg = WATCHERS.lock().unwrap(); + if let Some(map) = reg.as_mut() { + if let Some(handle) = map.remove(socket_path) { + handle.stop.store(true, Ordering::Relaxed); + } + } +} + +fn watch_loop(socket_path: &str, callback: Py, stop: &AtomicBool) -> Result<(), String> { + let (conn, mut queue, mut state, device) = open_device(socket_path)?; + while !stop.load(Ordering::Relaxed) && !state.finished { + if state.selection_changed { + state.selection_changed = false; + let mimes = state + .selection + .as_ref() + .and_then(|o| state.offer_mimes.get(&o.id()).cloned()) + .unwrap_or_default(); + if !mimes.is_empty() { + Python::attach(|py| { + if let Err(e) = callback.call1(py, (mimes,)) { + e.print(py); + } + }); + } + } + queue.flush().map_err(|e| format!("flush: {e}"))?; + if let Some(guard) = conn.prepare_read() { + use std::os::fd::AsRawFd; + if wait_readable(guard.connection_fd().as_raw_fd(), STOP_POLL)? { + guard.read().map_err(|e| format!("read: {e}"))?; + } + } + queue.dispatch_pending(&mut state).map_err(|e| format!("dispatch: {e}"))?; + } + device.destroy(); + let _ = queue.flush(); + Ok(()) +} diff --git a/pixelflux/src/wayland/frontend.rs b/pixelflux/src/wayland/frontend.rs index 743a673..b56fff8 100644 --- a/pixelflux/src/wayland/frontend.rs +++ b/pixelflux/src/wayland/frontend.rs @@ -397,6 +397,9 @@ pub struct AppState { /// Seat keymap owner: base layout plus batched overlay binds. Every seat keymap swap /// flows through this policy, so keymap identity has exactly one writer. pub keymap_policy: KeymapPolicy, + /// Host-capture session when pixelflux captures an EXTERNAL compositor: + /// frames arrive by screencopy and input routes to its virtual devices. + pub host: Option, pub current_cursor_icon: Option, pub cursor_buffer: Option, @@ -665,7 +668,35 @@ impl CompositorHandler for AppState { // A new toplevel opens fullscreened on the output the pointer is on // (primary when indeterminate); the choice is pinned on the window's // meta so the acked commit maps to the same output. - let target_id = self.pointer_display(); + let mut target_id = self.pointer_display(); + // A second fullscreen surface from a client that already owns one on + // the target output is screen-like (a nested compositor opens one + // host toplevel per screen): place it on an empty output when one + // exists instead of stacking it. + if let Some(client) = toplevel.wl_surface().client() { + let same_client = |w: &Window| { + w.wl_surface() + .and_then(|s| s.client()) + .map_or(false, |c| c.id() == client.id()) + }; + let crowded = self + .space + .elements() + .chain(self.pending_windows.iter()) + .any(|w| window_output_id(w) == target_id && same_client(w)); + if crowded { + let empty = self.output_nodes.iter().map(|n| n.id).find(|oid| { + !self + .space + .elements() + .chain(self.pending_windows.iter()) + .any(|w| window_output_id(w) == *oid) + }); + if let Some(oid) = empty { + target_id = oid; + } + } + } if let Some(meta) = window_meta(&window) { meta.output.store(target_id, Ordering::Relaxed); } @@ -1120,6 +1151,12 @@ impl AppState { if text.is_empty() { return; } + // Host-capture mode: the same managed keymap rides on the virtual + // keyboard, so the host compositor translates injected keycodes with + // selkies' keymap (overlay binds included) instead of its own. + if let Some(host) = &self.host { + host.set_keymap(&text); + } if let Some(keyboard) = self.seat.get_keyboard() { if let Err(e) = keyboard.set_keymap_from_string(self, text) { eprintln!("[Wayland] keymap swap failed: {e:?}"); diff --git a/pixelflux/src/wayland/host.rs b/pixelflux/src/wayland/host.rs new file mode 100644 index 0000000..d0015da --- /dev/null +++ b/pixelflux/src/wayland/host.rs @@ -0,0 +1,866 @@ +//! Host-capture mode: pixelflux as a CLIENT of an external Wayland compositor +//! (e.g. labwc running `WLR_BACKENDS=headless`), inverting the nested topology. +//! +//! The host compositor owns the session — one seat, one selection, one screen +//! model — and pixelflux captures and injects as a privileged-protocol client: +//! frames via `zwlr_screencopy_v1` (v3 `copy_with_damage`, so idle screens cost +//! nothing), keyboard via a persistent `zwp_virtual_keyboard_v1` device carrying +//! selkies' own keymap text, pointer via `zwlr_virtual_pointer_v1`. Zero-copy is +//! preserved by allocating the capture buffers from pixelflux's OWN GBM device +//! (render node — no privileges): the compositor blits straight into the dmabufs +//! the encoder imports, so no CPU ever touches a frame. Without a GPU the same +//! loop degrades to wl_shm buffers feeding the CPU encode pool. +//! +//! One [`HostSession`] per backend: the capture loop runs on its own thread and +//! hands ready buffers to the calloop tick over a channel (slots are returned +//! after encode, bounding frames in flight); the input proxies are called from +//! the calloop thread directly (wayland-client proxies are thread-safe). + +use std::fs::File; +use std::os::fd::AsFd; +use std::os::unix::net::UnixStream; +use std::sync::atomic::{AtomicBool, Ordering}; +use std::sync::mpsc::{Receiver, Sender, TryRecvError}; +use std::sync::{Arc, Mutex}; + +use gbm::{BufferObjectFlags, Device as GbmDevice, Format as GbmFormat}; +use smithay::backend::allocator::dmabuf::Dmabuf; +use smithay::backend::allocator::Buffer as _; +use smithay::utils::{Physical, Rectangle}; +use wayland_client::protocol::{wl_output, wl_pointer, wl_registry, wl_seat, wl_shm, wl_shm_pool}; +use wayland_client::{delegate_noop, Connection, Dispatch, Proxy, QueueHandle, WEnum}; +use wayland_protocols::wp::linux_dmabuf::zv1::client::{ + zwp_linux_buffer_params_v1::{self, ZwpLinuxBufferParamsV1}, + zwp_linux_dmabuf_v1::ZwpLinuxDmabufV1, +}; +use wayland_protocols_misc::zwp_virtual_keyboard_v1::client::{ + zwp_virtual_keyboard_manager_v1::ZwpVirtualKeyboardManagerV1, + zwp_virtual_keyboard_v1::ZwpVirtualKeyboardV1, +}; +use wayland_protocols_wlr::screencopy::v1::client::{ + zwlr_screencopy_frame_v1::{self, ZwlrScreencopyFrameV1}, + zwlr_screencopy_manager_v1::ZwlrScreencopyManagerV1, +}; +use wayland_protocols_wlr::output_management::v1::client::{ + zwlr_output_configuration_head_v1::ZwlrOutputConfigurationHeadV1, + zwlr_output_configuration_v1::{self, ZwlrOutputConfigurationV1}, + zwlr_output_head_v1::{self, ZwlrOutputHeadV1}, + zwlr_output_manager_v1::{self, ZwlrOutputManagerV1}, + zwlr_output_mode_v1::ZwlrOutputModeV1, +}; +use wayland_protocols_wlr::virtual_pointer::v1::client::{ + zwlr_virtual_pointer_manager_v1::ZwlrVirtualPointerManagerV1, + zwlr_virtual_pointer_v1::ZwlrVirtualPointerV1, +}; + +use crate::wayland::wlclient::{ + bounded_roundtrip, impl_sync_callback, memfd_with, socket_path, SyncState, +}; + +const KEYMAP_FORMAT_XKB_V1: u32 = 1; +/// Frames in flight between the capture thread and the encode tick. Two slots: +/// one being blitted by the compositor while the previous one is encoded. +const SLOTS: usize = 2; + +/// A frame the compositor finished blitting, ready for the encoder. +pub struct HostFrame { + slot: usize, + /// GPU path: the filled dmabuf (Arc-backed; the buffer itself stays owned by + /// the capture thread and is reused once the slot is released). + pub dmabuf: Option, + /// Software path: the shm mapping itself — the consumer converts straight + /// out of it (one copy total); the slot is not reused until release. + pub cpu: Option, + pub width: i32, + pub height: i32, + pub damage: Vec>, +} + +/// Borrow-by-Arc view of a software frame still sitting in its shm mapping. +pub struct HostCpuFrame { + map: Arc, + stride: usize, + format: u32, +} + +impl HostCpuFrame { + /// Convert the frame into tight BGRA rows in `dst` (sized `w*4*h`); the + /// announced shm format decides the per-pixel conversion (compositors pick + /// their renderer's preferred read format, e.g. 24-bit BGR on NVIDIA GLES). + pub fn write_bgra(&self, w: i32, h: i32, dst: &mut [u8]) { + let row = (w * 4) as usize; + let (src_bpp, swap_rb) = match self.format { + f if f == wl_shm::Format::Xbgr8888 as u32 + || f == wl_shm::Format::Abgr8888 as u32 => (4usize, true), + f if f == wl_shm::Format::Bgr888 as u32 => (3, false), + _ => (4, false), // xrgb/argb: already BGRA byte order + }; + for y in 0..h as usize { + let src_start = y * self.stride; + let src_end = (src_start + src_bpp * w as usize).min(self.map.len()); + if src_start >= src_end || (y + 1) * row > dst.len() { + break; + } + let src_row = &self.map[src_start..src_end]; + let dst_row = &mut dst[y * row..(y + 1) * row]; + match (src_bpp, swap_rb) { + (4, false) => { + let n = row.min(src_row.len()); + dst_row[..n].copy_from_slice(&src_row[..n]); + } + (4, true) => { + for (d, s) in dst_row.chunks_exact_mut(4).zip(src_row.chunks_exact(4)) { + d[0] = s[2]; + d[1] = s[1]; + d[2] = s[0]; + d[3] = s[3]; + } + } + _ => { + for (d, s) in dst_row.chunks_exact_mut(4).zip(src_row.chunks_exact(3)) { + d[0] = s[0]; + d[1] = s[1]; + d[2] = s[2]; + d[3] = 0xff; + } + } + } + } + } +} + +enum ToHost { + Start { width: i32, height: i32 }, + Release(usize), + Stop, +} + +struct HostState { + shm: Option, + dmabuf: Option, + screencopy: Option, + seat: Option, + vk_mgr: Option, + vptr_mgr: Option, + outputs: Vec, + output_mgr: Option, + heads: Vec, + om_serial: Option, + /// Outcome of the pending output configuration: Some(true) applied, + /// Some(false) failed, retried internally on `cancelled`. + cfg_result: Option, + cfg_cancelled: bool, + // Per-frame capture negotiation/results. + announce_dmabuf: Option<(u32, i32, i32)>, // fourcc, w, h + announce_shm: Option<(u32, i32, i32, i32)>, // format, w, h, stride + buffer_done: bool, + damage: Vec>, + ready: bool, + failed: bool, + sync_done: bool, +} + +impl Default for HostState { + fn default() -> Self { + Self { + shm: None, + dmabuf: None, + screencopy: None, + seat: None, + vk_mgr: None, + vptr_mgr: None, + outputs: Vec::new(), + output_mgr: None, + heads: Vec::new(), + om_serial: None, + cfg_result: None, + cfg_cancelled: false, + announce_dmabuf: None, + announce_shm: None, + buffer_done: false, + damage: Vec::new(), + ready: false, + failed: false, + sync_done: false, + } + } +} + +impl SyncState for HostState { + fn sync_done_mut(&mut self) -> &mut bool { + &mut self.sync_done + } +} +impl_sync_callback!(HostState); + +impl Dispatch for HostState { + fn event( + state: &mut Self, + registry: &wl_registry::WlRegistry, + event: wl_registry::Event, + _: &(), + _: &Connection, + qh: &QueueHandle, + ) { + if let wl_registry::Event::Global { name, interface, version } = event { + match interface.as_str() { + "wl_shm" => state.shm = Some(registry.bind(name, 1, qh, ())), + "zwp_linux_dmabuf_v1" if version >= 3 => { + state.dmabuf = Some(registry.bind(name, 3, qh, ())) + } + "zwlr_screencopy_manager_v1" if version >= 3 => { + state.screencopy = Some(registry.bind(name, 3, qh, ())) + } + "wl_seat" if state.seat.is_none() => { + state.seat = Some(registry.bind(name, 1, qh, ())) + } + "zwp_virtual_keyboard_manager_v1" => { + state.vk_mgr = Some(registry.bind(name, 1, qh, ())) + } + "zwlr_virtual_pointer_manager_v1" => { + state.vptr_mgr = Some(registry.bind(name, version.min(2), qh, ())) + } + "wl_output" => state.outputs.push(registry.bind(name, 1, qh, ())), + "zwlr_output_manager_v1" => { + state.output_mgr = Some(registry.bind(name, 1, qh, ())) + } + _ => {} + } + } + } +} + +delegate_noop!(HostState: ignore wl_seat::WlSeat); +delegate_noop!(HostState: ignore wl_output::WlOutput); +delegate_noop!(HostState: ignore wl_shm::WlShm); +delegate_noop!(HostState: ignore wl_shm_pool::WlShmPool); +delegate_noop!(HostState: ignore wayland_client::protocol::wl_buffer::WlBuffer); +delegate_noop!(HostState: ZwpVirtualKeyboardManagerV1); +delegate_noop!(HostState: ZwpVirtualKeyboardV1); +delegate_noop!(HostState: ZwlrVirtualPointerManagerV1); +delegate_noop!(HostState: ZwlrVirtualPointerV1); +delegate_noop!(HostState: ZwlrScreencopyManagerV1); +delegate_noop!(HostState: ignore ZwlrOutputModeV1); +delegate_noop!(HostState: ZwlrOutputConfigurationHeadV1); + +impl Dispatch for HostState { + fn event( + state: &mut Self, + _: &ZwlrOutputManagerV1, + event: zwlr_output_manager_v1::Event, + _: &(), + _: &Connection, + _: &QueueHandle, + ) { + match event { + zwlr_output_manager_v1::Event::Head { head } => state.heads.push(head), + zwlr_output_manager_v1::Event::Done { serial } => state.om_serial = Some(serial), + _ => {} + } + } + + wayland_client::event_created_child!(HostState, ZwlrOutputManagerV1, [ + zwlr_output_manager_v1::EVT_HEAD_OPCODE => (ZwlrOutputHeadV1, ()), + ]); +} + +impl Dispatch for HostState { + fn event( + state: &mut Self, + head: &ZwlrOutputHeadV1, + event: zwlr_output_head_v1::Event, + _: &(), + _: &Connection, + _: &QueueHandle, + ) { + if let zwlr_output_head_v1::Event::Finished = event { + state.heads.retain(|h| h.id() != head.id()); + } + } + + wayland_client::event_created_child!(HostState, ZwlrOutputHeadV1, [ + zwlr_output_head_v1::EVT_MODE_OPCODE => (ZwlrOutputModeV1, ()), + ]); +} + +impl Dispatch for HostState { + fn event( + state: &mut Self, + _: &ZwlrOutputConfigurationV1, + event: zwlr_output_configuration_v1::Event, + _: &(), + _: &Connection, + _: &QueueHandle, + ) { + match event { + zwlr_output_configuration_v1::Event::Succeeded => state.cfg_result = Some(true), + zwlr_output_configuration_v1::Event::Failed => state.cfg_result = Some(false), + zwlr_output_configuration_v1::Event::Cancelled => state.cfg_cancelled = true, + _ => {} + } + } +} + +impl Dispatch for HostState { + fn event( + _: &mut Self, + _: &ZwpLinuxDmabufV1, + _: ::Event, + _: &(), + _: &Connection, + _: &QueueHandle, + ) { + // format/modifier advertisements: the allocation follows the screencopy + // frame's announcement instead. + } +} + +impl Dispatch for HostState { + fn event( + _: &mut Self, + _: &ZwpLinuxBufferParamsV1, + _: zwp_linux_buffer_params_v1::Event, + _: &(), + _: &Connection, + _: &QueueHandle, + ) { + // created/failed are only sent for non-immed creation. + } +} + +impl Dispatch for HostState { + fn event( + state: &mut Self, + _: &ZwlrScreencopyFrameV1, + event: zwlr_screencopy_frame_v1::Event, + _: &(), + _: &Connection, + _: &QueueHandle, + ) { + match event { + zwlr_screencopy_frame_v1::Event::Buffer { format, width, height, stride } => { + if let WEnum::Value(f) = format { + state.announce_shm = Some((f as u32, width as i32, height as i32, stride as i32)); + } + } + zwlr_screencopy_frame_v1::Event::LinuxDmabuf { format, width, height } => { + state.announce_dmabuf = Some((format, width as i32, height as i32)); + } + zwlr_screencopy_frame_v1::Event::BufferDone => state.buffer_done = true, + zwlr_screencopy_frame_v1::Event::Damage { x, y, width, height } => { + state.damage.push(Rectangle::new( + (x as i32, y as i32).into(), + (width as i32, height as i32).into(), + )); + } + zwlr_screencopy_frame_v1::Event::Ready { .. } => state.ready = true, + zwlr_screencopy_frame_v1::Event::Failed => state.failed = true, + _ => {} + } + } +} + +enum SlotBuffer { + Gpu { + _bo: gbm::BufferObject<()>, + dmabuf: Dmabuf, + wl: wayland_client::protocol::wl_buffer::WlBuffer, + }, + Cpu { + _pool: wl_shm_pool::WlShmPool, + map: Arc, + stride: i32, + format: u32, + wl: wayland_client::protocol::wl_buffer::WlBuffer, + }, +} + +/// The calloop-side handle: input proxies plus the frame channel. +pub struct HostSession { + conn: Connection, + vk: Option, + vptr: Option, + to_thread: Sender, + frames: Receiver, + /// Newest frame, kept (slot and all) until replaced so an IDR request on a + /// static screen can re-encode current content like compositor mode does. + retained: Mutex>, + pub display_size: Arc>, + alive: Arc, +} + +impl HostSession { + /// Connect to `display` and bring up input devices + the capture thread + /// (idle until [`start_capture`]). `gbm` enables the zero-copy path. + pub fn connect(display: &str, gbm: Option>) -> Result { + let path = socket_path(display).ok_or("XDG_RUNTIME_DIR is unset")?; + let stream = UnixStream::connect(&path).map_err(|e| format!("connect {path}: {e}"))?; + let conn = Connection::from_socket(stream).map_err(|e| format!("wayland setup: {e}"))?; + let mut queue = conn.new_event_queue(); + let qh = queue.handle(); + let _registry = conn.display().get_registry(&qh, ()); + let mut state = HostState::default(); + bounded_roundtrip(&conn, &mut queue, &mut state)?; + + let seat = state.seat.clone().ok_or("host compositor advertises no wl_seat")?; + state.screencopy.as_ref().ok_or("host compositor lacks zwlr_screencopy_manager_v1")?; + + let vk = match &state.vk_mgr { + Some(mgr) => { + let vk = mgr.create_virtual_keyboard(&seat, &qh, ()); + // A keymap must precede any key event; selkies replaces this with + // its managed keymap through the ABI as soon as it starts. + if let Some(text) = crate::wayland::vkclient::us_base_text() { + let mut data = text.as_bytes().to_vec(); + data.push(0); + let fd = memfd_with(&data)?; + vk.keymap(KEYMAP_FORMAT_XKB_V1, fd.as_fd(), data.len() as u32); + } + Some(vk) + } + None => { + eprintln!("[HostCapture] no zwp_virtual_keyboard_manager_v1: keyboard injection disabled."); + None + } + }; + let vptr = match &state.vptr_mgr { + Some(mgr) => Some(mgr.create_virtual_pointer(Some(&seat), &qh, ())), + None => { + eprintln!("[HostCapture] no zwlr_virtual_pointer_manager_v1: pointer injection disabled."); + None + } + }; + bounded_roundtrip(&conn, &mut queue, &mut state)?; + + let (to_thread, from_main) = std::sync::mpsc::channel::(); + let (frame_tx, frames) = std::sync::mpsc::channel::(); + let alive = Arc::new(AtomicBool::new(true)); + let display_size = Arc::new(Mutex::new((0, 0))); + { + let conn = conn.clone(); + let alive = alive.clone(); + let display_size = display_size.clone(); + std::thread::Builder::new() + .name("pf-host-capture".into()) + .spawn(move || { + if let Err(e) = + capture_loop(conn, queue, state, gbm, from_main, frame_tx, display_size) + { + eprintln!("[HostCapture] capture loop ended: {e}"); + } + alive.store(false, Ordering::Relaxed); + }) + .map_err(|e| format!("spawn: {e}"))?; + } + Ok(Self { + conn, + vk, + vptr, + to_thread, + frames, + retained: Mutex::new(None), + display_size, + alive, + }) + } + + pub fn start_capture(&self, width: i32, height: i32) { + let _ = self.to_thread.send(ToHost::Start { width, height }); + } + + pub fn stop(&self) { + let _ = self.to_thread.send(ToHost::Stop); + } + + pub fn alive(&self) -> bool { + self.alive.load(Ordering::Relaxed) + } + + /// Newest ready frame, releasing any staler ones straight back to the pool. + pub fn try_take_frame(&self) -> Option { + let mut newest: Option = None; + loop { + match self.frames.try_recv() { + Ok(frame) => { + if let Some(stale) = newest.replace(frame) { + let _ = self.to_thread.send(ToHost::Release(stale.slot)); + } + } + Err(TryRecvError::Empty) | Err(TryRecvError::Disconnected) => break, + } + } + newest + } + + /// Return an encoded frame's slot for the host's next blit. + pub fn release_frame(&self, frame: HostFrame) { + let _ = self.to_thread.send(ToHost::Release(frame.slot)); + } + + /// Keep `frame` as the current content (releasing the one it replaces). + pub fn retain_frame(&self, frame: HostFrame) { + let old = self.retained.lock().unwrap().replace(frame); + if let Some(old) = old { + self.release_frame(old); + } + } + + /// Run `f` with the retained (current-content) frame, if any. + pub fn with_retained(&self, f: impl FnOnce(Option<&HostFrame>) -> R) -> R { + f(self.retained.lock().unwrap().as_ref()) + } + + /// Upload selkies' managed keymap to the virtual keyboard verbatim. + pub fn set_keymap(&self, text: &str) { + let Some(vk) = &self.vk else { return }; + let mut data = text.as_bytes().to_vec(); + data.push(0); + match memfd_with(&data) { + Ok(fd) => { + vk.keymap(KEYMAP_FORMAT_XKB_V1, fd.as_fd(), data.len() as u32); + let _ = self.conn.flush(); + } + Err(e) => eprintln!("[HostCapture] keymap upload failed: {e}"), + } + } + + /// Key event in xkb numbering (evdev + 8), matching the seat injectors. + pub fn key(&self, xkb_keycode: u32, pressed: bool) { + let Some(vk) = &self.vk else { return }; + if xkb_keycode < 8 { + return; + } + vk.key(0, xkb_keycode - 8, if pressed { 1 } else { 0 }); + let _ = self.conn.flush(); + } + + pub fn pointer_motion_abs(&self, x: f64, y: f64) { + let Some(vp) = &self.vptr else { return }; + let (w, h) = *self.display_size.lock().unwrap(); + if w <= 0 || h <= 0 { + return; + } + let cx = x.clamp(0.0, (w - 1) as f64) as u32; + let cy = y.clamp(0.0, (h - 1) as f64) as u32; + vp.motion_absolute(0, cx, cy, w as u32, h as u32); + vp.frame(); + let _ = self.conn.flush(); + } + + pub fn pointer_button(&self, btn: u32, pressed: bool) { + let Some(vp) = &self.vptr else { return }; + vp.button( + 0, + btn, + if pressed { wl_pointer::ButtonState::Pressed } else { wl_pointer::ButtonState::Released }, + ); + vp.frame(); + let _ = self.conn.flush(); + } + + pub fn pointer_axis(&self, dx: f64, dy: f64) { + let Some(vp) = &self.vptr else { return }; + if dy != 0.0 { + vp.axis(0, wl_pointer::Axis::VerticalScroll, dy); + } + if dx != 0.0 { + vp.axis(0, wl_pointer::Axis::HorizontalScroll, dx); + } + vp.frame(); + let _ = self.conn.flush(); + } +} + +impl Drop for HostSession { + fn drop(&mut self) { + let _ = self.to_thread.send(ToHost::Stop); + } +} + +/// Ask the host (wlr-output-management) to set a custom mode on its first head. +/// Retries once across a `cancelled` (stale serial). False when the host lacks +/// the protocol or refused the mode. +fn resize_host_output( + queue: &mut wayland_client::EventQueue, + state: &mut HostState, + qh: &QueueHandle, + width: i32, + height: i32, +) -> bool { + let Some(mgr) = state.output_mgr.clone() else { return false }; + for _ in 0..2 { + for _ in 0..50 { + if state.om_serial.is_some() && !state.heads.is_empty() { + break; + } + if queue.blocking_dispatch(state).is_err() { + return false; + } + } + let (Some(serial), Some(head)) = (state.om_serial, state.heads.first().cloned()) else { + return false; + }; + state.cfg_result = None; + state.cfg_cancelled = false; + let cfg = mgr.create_configuration(serial, qh, ()); + let cfg_head = cfg.enable_head(&head, qh, ()); + cfg_head.set_custom_mode(width, height, 0); + cfg.apply(); + let _ = queue.flush(); + while state.cfg_result.is_none() && !state.cfg_cancelled { + if queue.blocking_dispatch(state).is_err() { + return false; + } + } + cfg.destroy(); + if state.cfg_cancelled { + // Stale serial: the compositor re-announces its state with a fresh one. + state.om_serial = None; + continue; + } + return state.cfg_result == Some(true); + } + false +} + +fn fourcc_to_gbm(fourcc: u32) -> GbmFormat { + // XR24 / AR24; anything else falls back to ARGB (the encoder reads BGRA bytes + // and ignores alpha). + match fourcc { + 0x34325258 => GbmFormat::Xrgb8888, + _ => GbmFormat::Argb8888, + } +} + +fn capture_loop( + _conn: Connection, + mut queue: wayland_client::EventQueue, + mut state: HostState, + gbm: Option>, + from_main: Receiver, + frame_tx: Sender, + display_size: Arc>, +) -> Result<(), String> { + // Idle until capture starts. + let (mut want_w, mut want_h) = loop { + match from_main.recv() { + Ok(ToHost::Start { width, height }) => break (width, height), + Ok(ToHost::Release(..)) => continue, + Ok(ToHost::Stop) | Err(_) => return Ok(()), + } + }; + let qh = queue.handle(); + let output = state.outputs.first().cloned().ok_or("host compositor has no wl_output")?; + let screencopy = state.screencopy.clone().unwrap(); + if !resize_host_output(&mut queue, &mut state, &qh, want_w, want_h) { + eprintln!( + "[HostCapture] host output resize to {want_w}x{want_h} not applied; \ + capture follows the host's own size." + ); + } + + let mut slots: Vec> = (0..SLOTS).map(|_| None).collect(); + let mut free: Vec = (0..SLOTS).collect(); + let mut announced: Option<(i32, i32)> = None; + let mut warned_mismatch = false; + let mut consecutive_failures = 0u32; + + loop { + // Drain control messages; block only when out of buffers. + loop { + let msg = if free.is_empty() { + match from_main.recv() { + Ok(m) => m, + Err(_) => return Ok(()), + } + } else { + match from_main.try_recv() { + Ok(m) => m, + Err(TryRecvError::Empty) => break, + Err(TryRecvError::Disconnected) => return Ok(()), + } + }; + match msg { + ToHost::Release(slot) => free.push(slot), + ToHost::Stop => return Ok(()), + ToHost::Start { width, height } => { + // A capture reconfigure (client resolution change): resize the + // host output to follow, exactly like compositor mode does. + if (width, height) != (want_w, want_h) { + want_w = width; + want_h = height; + warned_mismatch = false; + if !resize_host_output(&mut queue, &mut state, &qh, want_w, want_h) { + eprintln!( + "[HostCapture] host output resize to {want_w}x{want_h} \ + not applied; capture follows the host's own size." + ); + } + } + } + } + } + + // One frame: negotiate (first time), attach our buffer, wait for damage. + state.announce_dmabuf = None; + state.announce_shm = None; + state.buffer_done = false; + state.damage.clear(); + state.ready = false; + state.failed = false; + let frame = screencopy.capture_output(1, &output, &qh, ()); + while !state.buffer_done && !state.failed { + queue.blocking_dispatch(&mut state).map_err(|e| format!("dispatch: {e}"))?; + } + if state.failed { + frame.destroy(); + consecutive_failures += 1; + if consecutive_failures == 3 { + eprintln!("[HostCapture] repeated screencopy failures; is the output alive?"); + } + std::thread::sleep(std::time::Duration::from_millis(100)); + continue; + } + + let (fw, fh) = state + .announce_dmabuf + .map(|(_, w, h)| (w, h)) + .or(state.announce_shm.map(|(_, w, h, _)| (w, h))) + .ok_or("screencopy announced no buffer type")?; + if announced != Some((fw, fh)) { + announced = Some((fw, fh)); + *display_size.lock().unwrap() = (fw, fh); + eprintln!( + "[HostCapture] negotiated: {fw}x{fh} gbm={} dmabuf_global={} dmabuf_announce={:?} shm={:?}", + gbm.is_some(), + state.dmabuf.is_some(), + state.announce_dmabuf, + state.announce_shm, + ); + // Size change: drop stale buffers. + for s in slots.iter_mut() { + *s = None; + } + free = (0..SLOTS).collect(); + } + if (fw, fh) != (want_w, want_h) { + // The encoder is configured for (want_w, want_h): hold frames until + // the host applies the resize rather than encode mismatched buffers. + frame.destroy(); + if !warned_mismatch { + warned_mismatch = true; + eprintln!( + "[HostCapture] waiting for host output {want_w}x{want_h} (currently {fw}x{fh})." + ); + } + std::thread::sleep(std::time::Duration::from_millis(150)); + continue; + } + warned_mismatch = false; + + let slot_idx = *free.last().unwrap(); + if slots[slot_idx].is_none() { + slots[slot_idx] = Some(match (&gbm, &state.dmabuf, state.announce_dmabuf) { + (Some(dev), Some(dmabuf_global), Some((fourcc, w, h))) => { + let bo = dev + .create_buffer_object::<()>( + w as u32, + h as u32, + fourcc_to_gbm(fourcc), + BufferObjectFlags::RENDERING, + ) + .map_err(|e| format!("GBM allocation {w}x{h}: {e:?}"))?; + let dmabuf = crate::create_dmabuf_from_bo(&bo); + let params = dmabuf_global.create_params(&qh, ()); + let modifier: u64 = dmabuf.format().modifier.into(); + for (i, handle) in dmabuf.handles().enumerate() { + params.add( + handle, + i as u32, + dmabuf.offsets().nth(i).unwrap_or(0), + dmabuf.strides().nth(i).unwrap_or(0), + (modifier >> 32) as u32, + (modifier & 0xffff_ffff) as u32, + ); + } + let wl = params.create_immed(w, h, fourcc, zwp_linux_buffer_params_v1::Flags::empty(), &qh, ()); + params.destroy(); + SlotBuffer::Gpu { _bo: bo, dmabuf, wl } + } + _ => { + let (format, w, h, stride) = + state.announce_shm.ok_or("no shm fallback announced")?; + let shm = state.shm.clone().ok_or("host compositor lacks wl_shm")?; + let size = (stride * h) as usize; + let fd = memfd_with(&vec![0u8; size])?; + let pool = shm.create_pool(fd.as_fd(), size as i32, &qh, ()); + let wl = pool.create_buffer( + 0, + w, + h, + stride, + WEnum::::from(format).into_result().unwrap_or(wl_shm::Format::Xrgb8888), + &qh, + (), + ); + let file = File::from(fd); + let map = unsafe { memmap2::MmapMut::map_mut(&file) } + .map_err(|e| format!("shm map: {e}"))?; + SlotBuffer::Cpu { _pool: pool, map: Arc::new(map), stride, format, wl } + } + }); + } + free.pop(); + + { + let slot = slots[slot_idx].as_ref().unwrap(); + let wl = match slot { + SlotBuffer::Gpu { wl, .. } => wl, + SlotBuffer::Cpu { wl, .. } => wl, + }; + frame.copy_with_damage(wl); + } + queue.flush().map_err(|e| format!("flush: {e}"))?; + while !state.ready && !state.failed { + queue.blocking_dispatch(&mut state).map_err(|e| format!("dispatch: {e}"))?; + } + frame.destroy(); + if state.failed { + free.push(slot_idx); + consecutive_failures += 1; + if consecutive_failures == 3 { + eprintln!("[HostCapture] repeated screencopy failures; is the output alive?"); + } + std::thread::sleep(std::time::Duration::from_millis(50)); + continue; + } + consecutive_failures = 0; + + let damage = std::mem::take(&mut state.damage); + let out = match slots[slot_idx].as_ref().unwrap() { + SlotBuffer::Gpu { dmabuf, .. } => HostFrame { + slot: slot_idx, + dmabuf: Some(dmabuf.clone()), + cpu: None, + width: fw, + height: fh, + damage, + }, + SlotBuffer::Cpu { map, stride, format, .. } => HostFrame { + slot: slot_idx, + dmabuf: None, + cpu: Some(HostCpuFrame { + map: map.clone(), + stride: *stride as usize, + format: *format, + }), + width: fw, + height: fh, + damage, + }, + }; + if frame_tx.send(out).is_err() { + return Ok(()); + } + } +} diff --git a/pixelflux/src/wayland/keymap.rs b/pixelflux/src/wayland/keymap.rs index 5034484..fe3185e 100644 --- a/pixelflux/src/wayland/keymap.rs +++ b/pixelflux/src/wayland/keymap.rs @@ -40,6 +40,20 @@ pub struct KeymapPolicy { by_sym: HashMap, /// Slot recycle order, oldest bind first. lru: VecDeque, + /// First overlay keycode (xkb numbering); slot i lives at `overlay_first + i`. + overlay_first: u32, + /// Overlay slot count. + overlay_capacity: usize, +} + +/// Keysym one literal character types as: Latin-1 printables map 1:1, `\n` types +/// Return (the raw utf32 table maps it to Linefeed, which no keymap binds), other +/// control characters their `0xffXX` function keysyms, everything else the +/// `0x01000000 | codepoint` Unicode form (0 = unmappable). The keysym then resolves +/// against an ACTIVE keymap, never a hardcoded layout table. +pub fn keysym_for_char(c: char) -> u32 { + let c = if c == '\n' { '\r' } else { c }; + xkb::utf32_to_keysym(c as u32).raw() } /// Compile an XKB_KEYMAP_FORMAT_TEXT_V1 string, or `None` when it does not compile. @@ -97,12 +111,22 @@ pub fn level0_syms(keymap: &xkb::Keymap) -> HashMap { impl KeymapPolicy { /// Placeholder policy before the seat keymap is known; `rebuild_base` fills it in. pub fn empty() -> Self { + Self::with_overlay_range(OVERLAY_FIRST_KEYCODE, OVERLAY_LAST_KEYCODE) + } + + /// Policy with a custom overlay keycode range (inclusive, xkb numbering). The seat + /// uses `empty()`'s above-255 range (pure-Wayland clients resolve it fine); the + /// virtual-keyboard client typing into a nested compositor uses a sub-256 range so + /// XWayland apps under that compositor stay reachable. + pub fn with_overlay_range(first: u32, last: u32) -> Self { Self { base_text: String::new(), base_map: HashMap::new(), slots: Vec::new(), by_sym: HashMap::new(), lru: VecDeque::new(), + overlay_first: first, + overlay_capacity: (last - first + 1) as usize, } } @@ -145,7 +169,7 @@ impl KeymapPolicy { } self.by_sym .get(&keysym) - .map(|&slot| (OVERLAY_FIRST_KEYCODE + slot as u32, 0)) + .map(|&slot| (self.overlay_first + slot as u32, 0)) } /// True when `keysym` resolves at level 0 (base or overlay) — i.e. typable without @@ -203,9 +227,9 @@ impl KeymapPolicy { self.lru.remove(at); } self.lru.push_back(slot); - return (OVERLAY_FIRST_KEYCODE + slot as u32, 0); + return (self.overlay_first + slot as u32, 0); } - let slot = if self.slots.len() < OVERLAY_CAPACITY { + let slot = if self.slots.len() < self.overlay_capacity { self.slots.push(None); self.slots.len() - 1 } else { @@ -220,7 +244,7 @@ impl KeymapPolicy { self.by_sym.insert(sym, slot); self.lru.push_back(slot); *changed = true; - (OVERLAY_FIRST_KEYCODE + slot as u32, 0) + (self.overlay_first + slot as u32, 0) } /// Oldest slot whose keycode is not currently held down; a held keycode must keep its @@ -229,7 +253,7 @@ impl KeymapPolicy { let at = self .lru .iter() - .position(|&slot| !pressed.contains(&(OVERLAY_FIRST_KEYCODE + slot as u32)))?; + .position(|&slot| !pressed.contains(&(self.overlay_first + slot as u32)))?; self.lru.remove(at) } @@ -255,7 +279,7 @@ impl KeymapPolicy { return self.base_text.clone(); }; let old_max: u32 = base[num_at..num_at + num_len].trim().parse().unwrap_or(255); - let need_max = OVERLAY_FIRST_KEYCODE + occupied.last().map(|&(i, _)| i as u32).unwrap_or(0); + let need_max = self.overlay_first + occupied.last().map(|&(i, _)| i as u32).unwrap_or(0); let mut text = String::with_capacity(base.len() + occupied.len() * 48); text.push_str(&base[..num_at]); text.push_str(&old_max.max(need_max).to_string()); @@ -266,7 +290,7 @@ impl KeymapPolicy { }; text.push_str(&rest[..kc_end]); for &(i, _) in &occupied { - text.push_str(&format!("\t = {};\n", i, OVERLAY_FIRST_KEYCODE + i as u32)); + text.push_str(&format!("\t = {};\n", i, self.overlay_first + i as u32)); } let rest = &rest[kc_end..]; let Some(close_at) = rest @@ -368,6 +392,25 @@ mod tests { assert_eq!(p.resolve(held_sym), Some((held_kc, 0))); } + #[test] + fn sub256_overlay_range_overrides_base_keycode_names() { + // The virtual-keyboard client's range collides with keycodes the base + // already names (…); the spliced definitions must win so overlay + // keysyms resolve at their assigned keycodes. + let mut p = KeymapPolicy::with_overlay_range(150, 255); + p.rebuild_base(us_base()); + let (out, changed) = p.bind_many_plain(&[0x1004E2D, 0x61], &HashSet::new()); + assert!(changed); + assert_eq!(out[0], 150); + let km = compile_keymap(&p.keymap_text()).expect("sub-256 overlay keymap compiles"); + let got = km.key_get_syms_by_level(xkb::Keycode::new(out[0]), 0, 0); + assert_eq!(got.len(), 1); + assert_eq!(got[0].raw(), 0x1004E2D); + // 'a' resolves plain in the base without consuming a slot. + assert!(out[1] < 150); + assert_eq!(out[1], p.resolve(0x61).unwrap().0); + } + #[test] fn rebuild_base_keeps_overlay_assignments() { let mut p = policy(); diff --git a/pixelflux/src/wayland/mod.rs b/pixelflux/src/wayland/mod.rs index 08b1720..b9e94a3 100644 --- a/pixelflux/src/wayland/mod.rs +++ b/pixelflux/src/wayland/mod.rs @@ -15,3 +15,11 @@ pub mod frontend; pub mod cursor; /// Seat keymap ownership: base layout plus batched overlay keysym binding. pub mod keymap; +/// Virtual-keyboard client for typing into a nested app compositor's socket. +pub mod vkclient; +/// Shared plumbing for outbound Wayland client connections. +pub mod wlclient; +/// Data-control clipboard client bridging a nested app compositor's selection. +pub mod dcclient; +/// Host-capture mode: capture/inject as a client of an external compositor. +pub mod host; diff --git a/pixelflux/src/wayland/vkclient.rs b/pixelflux/src/wayland/vkclient.rs new file mode 100644 index 0000000..9b2a5fa --- /dev/null +++ b/pixelflux/src/wayland/vkclient.rs @@ -0,0 +1,157 @@ +//! `zwp_virtual_keyboard_v1` client for typing Unicode text into ANOTHER Wayland +//! compositor's socket. +//! +//! pixelflux is normally the compositor, but in a nested deployment (a labwc/kwin +//! session running as a client of pixelflux) the apps live on that inner +//! compositor's socket, and keys injected into pixelflux's own seat resolve +//! against pixelflux's keymap — an overlay the inner compositor never sees. So +//! text is typed here as a client of whichever compositor the apps live under — +//! by Computer-Use actions and by selkies over the `type_text_wayland` ABI — +//! reusing the seat's [`KeymapPolicy`] over a US base: base-reachable characters +//! press their ordinary keycodes, everything else is overlay-bound, one upload +//! per batch. One-shot (connect, type, disconnect): a call carries a whole commit +//! of text, and a fresh connection leaves no stale-socket state to manage. +//! Blocking, off the compositor thread, with every round-trip deadline-bounded so +//! a wedged compositor cannot hang the caller forever. + +use std::collections::HashSet; +use std::os::fd::AsFd; +use std::os::unix::net::UnixStream; +use std::sync::OnceLock; +use std::time::Duration; + +use wayland_client::protocol::{wl_registry, wl_seat}; +use wayland_client::{delegate_noop, Connection, Dispatch, EventQueue, QueueHandle}; +use wayland_protocols_misc::zwp_virtual_keyboard_v1::client::{ + zwp_virtual_keyboard_manager_v1::ZwpVirtualKeyboardManagerV1, + zwp_virtual_keyboard_v1::ZwpVirtualKeyboardV1, +}; + +use crate::wayland::keymap::{compile_rmlvo, keysym_for_char, KeymapPolicy}; +use crate::wayland::wlclient::{bounded_roundtrip, impl_sync_callback, memfd_with, SyncState}; + +/// Overlay keycodes stay under the X11 255 ceiling so XWayland apps under the app +/// compositor can still receive them (the seat's own overlay sits above 255). +const OVERLAY_FIRST_XKB: u32 = 150; +const OVERLAY_LAST_XKB: u32 = 255; +const OVERLAY_SLOTS: usize = (OVERLAY_LAST_XKB - OVERLAY_FIRST_XKB + 1) as usize; +/// wl_keyboard / zwp_virtual_keyboard key events carry evdev codes (xkb - 8). +const EVDEV_OFFSET: u32 = 8; +const KEYMAP_FORMAT_XKB_V1: u32 = 1; + +#[derive(Default)] +struct Globals { + seat: Option, + manager: Option, + sync_done: bool, +} + +impl SyncState for Globals { + fn sync_done_mut(&mut self) -> &mut bool { + &mut self.sync_done + } +} +impl_sync_callback!(Globals); + +impl Dispatch for Globals { + fn event( + state: &mut Self, + registry: &wl_registry::WlRegistry, + event: wl_registry::Event, + _: &(), + _: &Connection, + qh: &QueueHandle, + ) { + if let wl_registry::Event::Global { name, interface, .. } = event { + // Version 1 of each suffices: the seat is only the manager argument. + match interface.as_str() { + "wl_seat" if state.seat.is_none() => { + state.seat = Some(registry.bind(name, 1, qh, ())); + } + "zwp_virtual_keyboard_manager_v1" if state.manager.is_none() => { + state.manager = Some(registry.bind(name, 1, qh, ())); + } + _ => {} + } + } + } +} + +delegate_noop!(Globals: ignore wl_seat::WlSeat); +delegate_noop!(Globals: ZwpVirtualKeyboardManagerV1); +delegate_noop!(Globals: ZwpVirtualKeyboardV1); + +/// The US base keymap text, compiled once per process: selkies types per text +/// commit, and xkbcommon compilation is the expensive part of a call. +pub(crate) fn us_base_text() -> Option<&'static str> { + static CACHE: OnceLock> = OnceLock::new(); + CACHE.get_or_init(|| compile_rmlvo("", "", "us", "", "")).as_deref() +} + +fn upload_keymap( + vk: &ZwpVirtualKeyboardV1, + queue: &mut EventQueue, + text: &str, +) -> Result<(), String> { + let mut data = text.as_bytes().to_vec(); + data.push(0); // compositors parse the mapping as a NUL-terminated string + let fd = memfd_with(&data)?; + vk.keymap(KEYMAP_FORMAT_XKB_V1, fd.as_fd(), data.len() as u32); + queue.flush().map_err(|e| format!("flush keymap: {e}")) +} + +/// One-shot: connect to `socket_path`, type `text` in order, disconnect. Codepoints +/// with no keysym are skipped. Blocking; call off the compositor's calloop thread. +pub fn type_text_to(socket_path: &str, text: &str) -> Result<(), String> { + let stream = + UnixStream::connect(socket_path).map_err(|e| format!("connect {socket_path}: {e}"))?; + let conn = Connection::from_socket(stream).map_err(|e| format!("wayland setup: {e}"))?; + let mut queue = conn.new_event_queue(); + let qh = queue.handle(); + let _registry = conn.display().get_registry(&qh, ()); + let mut state = Globals::default(); + bounded_roundtrip(&conn, &mut queue, &mut state)?; + let seat = state.seat.take().ok_or("app compositor advertises no wl_seat")?; + let manager = state + .manager + .take() + .ok_or("app compositor does not advertise zwp_virtual_keyboard_manager_v1")?; + let vk = manager.create_virtual_keyboard(&seat, &qh, ()); + // Surfaces an "unauthorized" bind error before the first keymap upload. + bounded_roundtrip(&conn, &mut queue, &mut state)?; + + let mut policy = KeymapPolicy::with_overlay_range(OVERLAY_FIRST_XKB, OVERLAY_LAST_XKB); + policy.rebuild_base( + us_base_text().ok_or("us base keymap failed to compile")?.to_string(), + ); + let syms: Vec = text.chars().map(keysym_for_char).filter(|&s| s != 0).collect(); + let none = HashSet::new(); + let mut uploaded = false; + // Chunk bound: at most OVERLAY_SLOTS keysyms per bind call, so a batch can + // never recycle a slot it assigned earlier in the same batch. + for chunk in syms.chunks(OVERLAY_SLOTS) { + let (keycodes, changed) = policy.bind_many_plain(chunk, &none); + // The protocol requires a keymap before the first key event even when the + // whole text resolves in the base. + if changed || !uploaded { + upload_keymap(&vk, &mut queue, &policy.keymap_text())?; + bounded_roundtrip(&conn, &mut queue, &mut state)?; + std::thread::sleep(Duration::from_millis(10)); + uploaded = true; + } + for &kc in &keycodes { + if kc < EVDEV_OFFSET { + continue; // unbindable keysym + } + for pressed in [1u32, 0] { + vk.key(0, kc - EVDEV_OFFSET, pressed); + queue.flush().map_err(|e| format!("flush key: {e}"))?; + std::thread::sleep(Duration::from_millis(2)); + } + } + } + bounded_roundtrip(&conn, &mut queue, &mut state)?; + vk.destroy(); + let _ = queue.flush(); + Ok(()) +} diff --git a/pixelflux/src/wayland/wlclient.rs b/pixelflux/src/wayland/wlclient.rs new file mode 100644 index 0000000..4841627 --- /dev/null +++ b/pixelflux/src/wayland/wlclient.rs @@ -0,0 +1,201 @@ +//! Shared plumbing for pixelflux's outbound Wayland CLIENT connections (the +//! virtual-keyboard typer and the data-control clipboard bridge, both talking to +//! a nested app compositor): socket resolution, deadline-bounded round-trips so a +//! wedged compositor turns into an error instead of a hang, and fd read/write +//! helpers for clipboard pipes. + +use std::os::fd::{AsRawFd, FromRawFd, OwnedFd, RawFd}; +use std::time::{Duration, Instant}; + +use wayland_client::protocol::wl_callback; +use wayland_client::{Connection, Dispatch, EventQueue}; + +pub(crate) const IO_TIMEOUT: Duration = Duration::from_secs(5); + +/// Absolute socket path for a Wayland display name (absolute paths pass through, +/// names join XDG_RUNTIME_DIR). +pub(crate) fn socket_path(name: &str) -> Option { + if name.starts_with('/') { + return Some(name.to_string()); + } + let rt = std::env::var("XDG_RUNTIME_DIR").ok()?; + Some(format!("{}/{}", rt.trim_end_matches('/'), name)) +} + +/// Implemented by every client state so [`bounded_roundtrip`] can flag the sync +/// callback's completion; pair with [`impl_sync_callback`]. +pub(crate) trait SyncState { + fn sync_done_mut(&mut self) -> &mut bool; +} + +/// `Dispatch` for a [`SyncState`] type (a blanket impl would violate +/// the orphan rule, so each state stamps its own). +macro_rules! impl_sync_callback { + ($t:ty) => { + impl wayland_client::Dispatch + for $t + { + fn event( + state: &mut Self, + _: &wayland_client::protocol::wl_callback::WlCallback, + event: wayland_client::protocol::wl_callback::Event, + _: &(), + _: &wayland_client::Connection, + _: &wayland_client::QueueHandle, + ) { + if let wayland_client::protocol::wl_callback::Event::Done { .. } = event { + *crate::wayland::wlclient::SyncState::sync_done_mut(state) = true; + } + } + } + }; +} +pub(crate) use impl_sync_callback; + +/// Block until `fd` is readable or `timeout` passes (false = timed out). +pub(crate) fn wait_readable(fd: RawFd, timeout: Duration) -> Result { + loop { + let mut pfd = libc::pollfd { fd, events: libc::POLLIN, revents: 0 }; + let n = unsafe { libc::poll(&mut pfd, 1, timeout.as_millis().max(1) as libc::c_int) }; + if n < 0 { + let err = std::io::Error::last_os_error(); + if err.raw_os_error() == Some(libc::EINTR) { + continue; + } + return Err(format!("poll: {err}")); + } + return Ok(n > 0); + } +} + +/// `EventQueue::roundtrip` with a deadline: a wedged compositor becomes an error +/// instead of hanging the calling thread (and everything queued behind it). +pub(crate) fn bounded_roundtrip( + conn: &Connection, + queue: &mut EventQueue, + state: &mut S, +) -> Result<(), String> +where + S: SyncState + Dispatch + 'static, +{ + *state.sync_done_mut() = false; + let _cb = conn.display().sync(&queue.handle(), ()); + let deadline = Instant::now() + IO_TIMEOUT; + loop { + queue + .dispatch_pending(state) + .map_err(|e| format!("dispatch: {e}"))?; + if *state.sync_done_mut() { + return Ok(()); + } + queue.flush().map_err(|e| format!("flush: {e}"))?; + let remaining = deadline + .checked_duration_since(Instant::now()) + .ok_or("compositor round-trip timed out")?; + let Some(guard) = conn.prepare_read() else { + continue; + }; + if !wait_readable(guard.connection_fd().as_raw_fd(), remaining)? { + return Err("compositor round-trip timed out".into()); + } + guard.read().map_err(|e| format!("read: {e}"))?; + } +} + +/// Anonymous CLOEXEC memfd holding `data` (keymap uploads, shm-style payloads). +pub(crate) fn memfd_with(data: &[u8]) -> Result { + let name = b"pixelflux-wl\0"; + let fd = + unsafe { libc::memfd_create(name.as_ptr() as *const libc::c_char, libc::MFD_CLOEXEC) }; + if fd < 0 { + return Err(format!("memfd_create: {}", std::io::Error::last_os_error())); + } + let owned = unsafe { OwnedFd::from_raw_fd(fd) }; + let mut written = 0; + while written < data.len() { + let n = unsafe { + libc::write( + owned.as_raw_fd(), + data[written..].as_ptr() as *const libc::c_void, + data.len() - written, + ) + }; + if n < 0 { + return Err(format!("write memfd: {}", std::io::Error::last_os_error())); + } + written += n as usize; + } + Ok(owned) +} + +/// CLOEXEC pipe as (read end, write end). +pub(crate) fn pipe_cloexec() -> Result<(OwnedFd, OwnedFd), String> { + let mut fds = [0i32; 2]; + if unsafe { libc::pipe2(fds.as_mut_ptr(), libc::O_CLOEXEC) } < 0 { + return Err(format!("pipe2: {}", std::io::Error::last_os_error())); + } + Ok(unsafe { (OwnedFd::from_raw_fd(fds[0]), OwnedFd::from_raw_fd(fds[1])) }) +} + +/// Read `fd` to EOF. The deadline is per-chunk (idle), so a large transfer that +/// keeps flowing is never cut off while a stalled writer still errors out. +pub(crate) fn read_fd_to_end(fd: &OwnedFd, idle: Duration) -> Result, String> { + let mut out = Vec::new(); + let mut chunk = [0u8; 65536]; + loop { + if !wait_readable(fd.as_raw_fd(), idle)? { + return Err("clipboard source stalled".into()); + } + let n = unsafe { + libc::read(fd.as_raw_fd(), chunk.as_mut_ptr() as *mut libc::c_void, chunk.len()) + }; + if n < 0 { + let err = std::io::Error::last_os_error(); + if err.raw_os_error() == Some(libc::EINTR) { + continue; + } + return Err(format!("read: {err}")); + } + if n == 0 { + return Ok(out); + } + out.extend_from_slice(&chunk[..n as usize]); + } +} + +/// Write all of `data` to `fd`, tolerating a slow reader up to `idle` per chunk. +/// EPIPE is success-shaped: the paster stopped reading, which is its right. +pub(crate) fn write_fd_all(fd: &OwnedFd, data: &[u8], idle: Duration) -> Result<(), String> { + let mut written = 0; + while written < data.len() { + let mut pfd = libc::pollfd { fd: fd.as_raw_fd(), events: libc::POLLOUT, revents: 0 }; + let n = unsafe { libc::poll(&mut pfd, 1, idle.as_millis().max(1) as libc::c_int) }; + if n < 0 { + let err = std::io::Error::last_os_error(); + if err.raw_os_error() == Some(libc::EINTR) { + continue; + } + return Err(format!("poll: {err}")); + } + if n == 0 { + return Err("clipboard reader stalled".into()); + } + let w = unsafe { + libc::write( + fd.as_raw_fd(), + data[written..].as_ptr() as *const libc::c_void, + data.len() - written, + ) + }; + if w < 0 { + let err = std::io::Error::last_os_error(); + match err.raw_os_error() { + Some(libc::EINTR) => continue, + Some(libc::EPIPE) => return Ok(()), + _ => return Err(format!("write: {err}")), + } + } + written += w as usize; + } + Ok(()) +} From 2cf4bad18c355eccf18b38e091ba0b485f0f3a6a Mon Sep 17 00:00:00 2001 From: Seungmin Kim <8457324+ehfd@users.noreply.github.com> Date: Mon, 20 Jul 2026 07:33:52 +0900 Subject: [PATCH 05/21] fix(wayland): Cleanups --- pixelflux/src/computer_use.rs | 5 +- pixelflux/src/encoders/overlay.rs | 91 ++- pixelflux/src/lib.rs | 274 +++++++-- pixelflux/src/wayland/host.rs | 979 +++++++++++++++++++++++------- pixelflux/src/wayland/wlclient.rs | 53 ++ pixelflux/src/x11/mod.rs | 174 +----- 6 files changed, 1158 insertions(+), 418 deletions(-) diff --git a/pixelflux/src/computer_use.rs b/pixelflux/src/computer_use.rs index 65cb8ec..6755eca 100644 --- a/pixelflux/src/computer_use.rs +++ b/pixelflux/src/computer_use.rs @@ -820,15 +820,14 @@ pub fn set_app_wayland_display(display: Option) { /// Resolve the app compositor socket PATH for CU typing, or None to type on the /// local seat. The ABI value selkies set wins; a standalone CU (no selkies) falls -/// back to PIXELFLUX_APP_WAYLAND_DISPLAY, then SELKIES_APP_WAYLAND_DISPLAY. A value -/// naming pixelflux's own compositor means nothing is nested. +/// back to PIXELFLUX_APP_WAYLAND_DISPLAY. A value naming pixelflux's own +/// compositor means nothing is nested. fn app_wayland_socket_path() -> Option { let name = CU_APP_WAYLAND_DISPLAY .lock() .unwrap() .clone() .or_else(|| std::env::var("PIXELFLUX_APP_WAYLAND_DISPLAY").ok()) - .or_else(|| std::env::var("SELKIES_APP_WAYLAND_DISPLAY").ok()) .filter(|s| !s.is_empty())?; if crate::wait_socket_name(Duration::from_millis(0)).as_deref() == Some(name.as_str()) { return None; diff --git a/pixelflux/src/encoders/overlay.rs b/pixelflux/src/encoders/overlay.rs index 1015c5e..0fd7f1b 100644 --- a/pixelflux/src/encoders/overlay.rs +++ b/pixelflux/src/encoders/overlay.rs @@ -21,7 +21,7 @@ use smithay::{ ImportMem, Renderer, Texture, }, }, - utils::{Point, Transform, Physical}, + utils::{Point, Rectangle, Transform, Physical}, }; use std::path::Path; @@ -59,12 +59,14 @@ pub struct OverlayState { wm_height: u32, wm_pos_x: i32, wm_pos_y: i32, + wm_prev_pos: Option<(i32, i32)>, wm_velocity_x: f64, wm_velocity_y: f64, wm_subpixel_x: f64, wm_subpixel_y: f64, wm_loaded: bool, is_animated: bool, + wm_pixels: Vec, render_buffer: Option, } @@ -75,17 +77,40 @@ impl Default for OverlayState { wm_height: 0, wm_pos_x: 0, wm_pos_y: 0, + wm_prev_pos: None, wm_velocity_x: 2.0, wm_velocity_y: 2.0, wm_subpixel_x: 0.0, wm_subpixel_y: 0.0, wm_loaded: false, is_animated: false, + wm_pixels: Vec::new(), render_buffer: None, } } } +/// Alpha-blend a source pixel (pre-split into r,g,b,a) over a BGRA destination pixel. +/// +/// Overlay pixels are overwhelmingly either fully opaque or fully transparent, and this runs per +/// pixel per frame on the CPU, so the two extremes are special-cased to skip the blend arithmetic +/// entirely: an opaque source (`a == 255`) simply overwrites, a fully transparent source (`a == 0`) +/// is left as-is, and only genuine edge pixels pay for the integer source-over. In every case only +/// the B / G / R bytes are written; the destination's alpha byte is left as the capture delivered it. +#[inline] +pub(crate) fn blend_pixel(dst: &mut [u8], r: u8, g: u8, b: u8, a: u8) { + if a == 255 { + dst[0] = b; + dst[1] = g; + dst[2] = r; + } else if a > 0 { + let ia = 255 - a as u32; + dst[0] = ((b as u32 * a as u32 + dst[0] as u32 * ia) / 255) as u8; + dst[1] = ((g as u32 * a as u32 + dst[1] as u32 * ia) / 255) as u8; + dst[2] = ((r as u32 * a as u32 + dst[2] as u32 * ia) / 255) as u8; + } +} + impl OverlayState { /// Load the watermark image from disk; `output_scale` is the output's fractional /// scale, ceiled to the integer buffer scale of the upload. A failed load clears the overlay. @@ -97,20 +122,81 @@ impl OverlayState { self.wm_loaded = true; let buffer_scale = output_scale.ceil().max(1.0) as i32; + let pixels = rgba.into_vec(); self.render_buffer = Some(MemoryRenderBuffer::from_slice( - &rgba.into_vec(), + &pixels, Fourcc::Abgr8888, (self.wm_width as i32, self.wm_height as i32), buffer_scale, Transform::Normal, None, )); + self.wm_pixels = pixels; } else { self.wm_loaded = false; + self.wm_pixels = Vec::new(); self.render_buffer = None; } } + /// Alpha-blend the watermark into a BGRA frame (row `stride` in bytes) at its + /// current position. Clips per pixel at the frame bounds because the animated + /// position — or a watermark larger than the capture — can leave part of the + /// image off-frame, and only the in-frame portion may be written. + pub fn blend_bgra(&self, frame: &mut [u8], stride: usize, frame_w: i32, frame_h: i32) { + if !self.wm_loaded { + return; + } + let (w, h) = (self.wm_width as i32, self.wm_height as i32); + for y in 0..h { + let ty = self.wm_pos_y + y; + if ty < 0 || ty >= frame_h { + continue; + } + for x in 0..w { + let tx = self.wm_pos_x + x; + if tx < 0 || tx >= frame_w { + continue; + } + let src = ((y * w + x) * 4) as usize; + let (r, g, b, a) = ( + self.wm_pixels[src], + self.wm_pixels[src + 1], + self.wm_pixels[src + 2], + self.wm_pixels[src + 3], + ); + let off = ty as usize * stride + tx as usize * 4; + blend_pixel(&mut frame[off..off + 4], r, g, b, a); + } + } + } + + /// Frame-clipped union of the watermark's current and previous rectangles — + /// the region a damage-gated encoder must repaint after a bounce step moved + /// the image (the vacated area needs repainting as much as the new one). + pub fn damage_rect(&self, frame_w: i32, frame_h: i32) -> Option> { + if !self.wm_loaded { + return None; + } + let (w, h) = (self.wm_width as i32, self.wm_height as i32); + let (mut x0, mut y0) = (self.wm_pos_x, self.wm_pos_y); + let (mut x1, mut y1) = (x0 + w, y0 + h); + if let Some((px, py)) = self.wm_prev_pos { + x0 = x0.min(px); + y0 = y0.min(py); + x1 = x1.max(px + w); + y1 = y1.max(py + h); + } + x0 = x0.max(0); + y0 = y0.max(0); + x1 = x1.min(frame_w); + y1 = y1.min(frame_h); + if x1 <= x0 || y1 <= y0 { + return None; + } + Some(Rectangle::new((x0, y0).into(), (x1 - x0, y1 - y0).into())) + } + /// True once a watermark image has been loaded. pub fn is_active(&self) -> bool { self.wm_loaded @@ -135,6 +221,7 @@ impl OverlayState { let w = self.wm_width as i32; let h = self.wm_height as i32; + self.wm_prev_pos = Some((self.wm_pos_x, self.wm_pos_y)); self.is_animated = matches!(loc, WatermarkLocation::AN); match loc { diff --git a/pixelflux/src/lib.rs b/pixelflux/src/lib.rs index 90abe4d..bee10e7 100644 --- a/pixelflux/src/lib.rs +++ b/pixelflux/src/lib.rs @@ -81,11 +81,13 @@ use smithay::{ element::{ memory::MemoryRenderBufferRenderElement, surface::WaylandSurfaceRenderElement, - AsRenderElements, Wrap, + AsRenderElements, Element, RenderElement, Wrap, }, gles::GlesRenderer, pixman::PixmanRenderer, - Bind, ImportAll, ImportEgl, ImportMem, + sync::SyncPoint, + Bind, ExportMem, Frame as _, ImportAll, ImportDma, ImportEgl, ImportMem, + Renderer as _, }, }, desktop::{space::SpaceRenderElements, Space}, @@ -1544,31 +1546,30 @@ fn start_capture_on_display( ) { use smithay::wayland::fractional_scale::with_fractional_scale; - // Host-capture mode (primary display): connect on first use and point the - // capture thread at the requested size. The compositor keeps running (CU, - // clipboard callbacks, input fallbacks) but its renderer is bypassed. - if display_id == 0 && !settings.wayland_host_display.is_empty() { + // Host-capture mode: connect on first use and point this display's capture + // thread at the requested size. The compositor keeps running (CU, clipboard + // callbacks, input fallbacks) but its renderer is bypassed. + if !settings.wayland_host_display.is_empty() { if state.host.is_none() { - // The compositor's own allocator device: capture buffers come from the - // same render node the encoder imports from (reopened via its live fd, - // since the resolved path string is not retained in auto mode). - let gbm = if state.use_gpu { + // Capture buffers come from the same render node the encoder imports + // from, resolved via its live fd (the path string is not retained in + // auto mode); each capture thread opens its own device handle. + let gbm_path = if state.use_gpu { state.gbm_device.as_ref().and_then(|dev| { use std::os::fd::{AsFd as _, AsRawFd as _}; let fd = dev.as_fd().as_raw_fd(); - std::fs::read_link(format!("/proc/self/fd/{fd}")) - .ok() - .and_then(|p| std::fs::File::options().read(true).write(true).open(p).ok()) - .and_then(|f| GbmDevice::new(f).ok()) + std::fs::read_link(format!("/proc/self/fd/{fd}")).ok() }) } else { None }; - match crate::wayland::host::HostSession::connect(&settings.wayland_host_display, gbm) { + match crate::wayland::host::HostSession::connect(&settings.wayland_host_display, gbm_path) + { Ok(h) => { println!( - "[HostCapture] capturing host compositor '{}'.", - settings.wayland_host_display + "[HostCapture] capturing host compositor '{}' ({} outputs).", + settings.wayland_host_display, + h.output_count() ); state.host = Some(h); } @@ -1579,7 +1580,13 @@ fn start_capture_on_display( } } if let Some(host) = &state.host { - host.start_capture(settings.width, settings.height); + // The node's layout offset rides along so the host's heads mirror + // selkies' union layout (input coordinates already assume it). + if let Some(idx) = state.node_idx_for_id(display_id) { + let pos = state.output_nodes[idx].pos; + host.set_layout(display_id, pos.0, pos.1); + } + host.start_capture(display_id, settings.width, settings.height); } } @@ -1957,6 +1964,80 @@ fn start_capture_on_display( /// display's own encode path, and answer a pending screenshot on the primary. Returns true /// when the tick was skipped because this display's encode pool was still busy (the caller /// then retries shortly instead of waiting a full frame interval). +/// Stamp the watermark element onto `target` in place — no clear, so the +/// captured content underneath stays — returning the draw's sync point for the +/// encoder to wait on. +fn draw_host_watermark( + renderer: &mut GlesRenderer, + overlay: &crate::encoders::overlay::OverlayState, + target: &mut Dmabuf, + size: (i32, i32), +) -> Result { + let elem = overlay + .get_watermark_element(renderer) + .ok_or("watermark element unavailable")?; + let mut fb = renderer.bind(target).map_err(|e| format!("bind: {e:?}"))?; + let mut frame = renderer + .render(&mut fb, (size.0, size.1).into(), Transform::Normal) + .map_err(|e| format!("render: {e:?}"))?; + let dst = elem.geometry(1.0.into()); + let local = Rectangle::from_size(dst.size); + elem.draw(&mut frame, elem.src(), dst, &[local], &[]) + .map_err(|e| format!("draw: {e:?}"))?; + frame.finish().map_err(|e| format!("finish: {e:?}")) +} + +/// Re-compose `src` (the retained host frame) plus the watermark into `target`: +/// the path a moving watermark needs, since re-drawing over the same retained +/// buffer would leave trails. +fn compose_host_watermark( + renderer: &mut GlesRenderer, + overlay: &crate::encoders::overlay::OverlayState, + src: &Dmabuf, + target: &mut Dmabuf, + size: (i32, i32), +) -> Result { + let elem = overlay + .get_watermark_element(renderer) + .ok_or("watermark element unavailable")?; + let tex = renderer + .import_dmabuf(src, None) + .map_err(|e| format!("import: {e:?}"))?; + let mut fb = renderer.bind(target).map_err(|e| format!("bind: {e:?}"))?; + let full: Rectangle = Rectangle::from_size((size.0, size.1).into()); + let mut frame = renderer + .render(&mut fb, (size.0, size.1).into(), Transform::Normal) + .map_err(|e| format!("render: {e:?}"))?; + frame + .render_texture_from_to( + &tex, + Rectangle::from_size((size.0 as f64, size.1 as f64).into()), + full, + &[full], + // Opaque: the capture format's undefined alpha must not blend, it + // would leave the target's previous content (and stamped + // watermarks) underneath. + &[full], + Transform::Normal, + 1.0, + None, + &[], + ) + .map_err(|e| format!("texture: {e:?}"))?; + let dst = elem.geometry(1.0.into()); + let local = Rectangle::from_size(dst.size); + elem.draw(&mut frame, elem.src(), dst, &[local], &[]) + .map_err(|e| format!("draw: {e:?}"))?; + frame.finish().map_err(|e| format!("finish: {e:?}")) +} + +fn warn_once_host_watermark(e: &str) { + static WARNED: std::sync::atomic::AtomicBool = std::sync::atomic::AtomicBool::new(false); + if !WARNED.swap(true, Ordering::Relaxed) { + eprintln!("[HostCapture] watermark compositing failed: {e}"); + } +} + fn render_node_tick( state: &mut AppState, node: &mut wayland::frontend::OutputNode, @@ -2073,27 +2154,49 @@ fn render_node_tick( // Host-capture mode: the host compositor already blitted this display's frame // into one of our buffers (screencopy); adopt it in place of compositing. - let host_mode = state.host.is_some() && node.id == 0; + let host_mode = state.host.as_ref().map(|h| h.has_output_for(node.id)).unwrap_or(false); + if state.host.is_some() && !host_mode { + // No host output backs this display (start_capture already warned): + // produce nothing rather than the compositor's own empty content. + if let Some((id, buf)) = pool_slot.take() { + if let Some(cap) = node.capture.as_ref() { + if let Some(ref pool) = cap.encode_pool { + pool.cancel(id, buf); + } + } + } + return false; + } // Dmabuf handed to the GPU encoder in host mode (from the new or retained frame). let mut host_enc_dmabuf: Option = None; if host_mode { + const RETAINED_OK: u8 = 0; + const RETAINED_NONE: u8 = 1; + const RETAINED_MISMATCH: u8 = 2; + let host_idx = node.id; let gpu_encoder = node .capture .as_ref() .map(|c| c.video_encoder.is_some()) .unwrap_or(false); - let new_frame = state.host.as_ref().unwrap().try_take_frame(); + // The session steps out of `state` while frames are adopted so the + // renderer can composite the watermark / serve screenshot readbacks. + let host = state.host.take().unwrap(); + let new_frame = host.try_take_frame(host_idx); let have_new = new_frame.is_some(); // Streaming mode wants a constant-rate stream (the client's decoder pipeline // is built for it), so re-encode the retained frame every tick like the - // compositor path does. Outside streaming mode, stay damage-driven and only - // re-encode when an IDR is pending (a viewer opening its keyframe gate). + // compositor path does. Outside streaming mode, stay damage-driven, waking + // only for a pending IDR (a viewer opening its keyframe gate), a screenshot + // request, or a bouncing watermark that must keep moving. let streaming = node .capture .as_ref() .map(|c| c.settings.video_streaming_mode) .unwrap_or(false); - if !have_new && !want_idr_for_host && !streaming { + let wm_active = node.overlay_state.is_active(); + let wm_animated = wm_active && node.overlay_state.is_animated(); + if !have_new && !want_idr_for_host && !streaming && !take_screenshot && !wm_animated { if let Some((id, buf)) = pool_slot.take() { if let Some(cap) = node.capture.as_ref() { if let Some(ref pool) = cap.encode_pool { @@ -2101,40 +2204,124 @@ fn render_node_tick( } } } + state.host = Some(host); return false; } if let Some(f) = new_frame { - state.host.as_ref().unwrap().retain_frame(f); + host.retain_frame(host_idx, f); } // Consume the (new or prior) retained frame's content into the pool slot / - // GPU dmabuf. The frame stays retained for the next IDR. - let bad_combo = state.host.as_ref().unwrap().with_retained(|r| { - let Some(f) = r else { return true }; - damage_rects = f.damage.clone(); + // GPU dmabuf. The frame stays retained for the next IDR. The CPU path + // blends the watermark in place; the GPU path composites it below. Damage + // is the fresh blit's; a re-encode of retained content has none of its own. + let mut wm_drawn = false; + let outcome = host.with_retained(host_idx, |r| { + let Some(f) = r else { return RETAINED_NONE }; + damage_rects = if have_new { f.damage.clone() } else { Vec::new() }; if let Some(cpu) = f.cpu.as_ref() { if gpu_encoder { - return true; // software frames, GPU encoder — mismatch + return RETAINED_MISMATCH; // software frames, GPU encoder } if let Some((_, ref mut buf)) = pool_slot { cpu.write_bgra(f.width, f.height, buf); + if wm_active { + node.overlay_state.blend_bgra(buf, (f.width as usize) * 4, f.width, f.height); + wm_drawn = true; + } } cpu.write_bgra(f.width, f.height, &mut node.frame_buffer); + if wm_active { + node.overlay_state + .blend_bgra(&mut node.frame_buffer, (f.width as usize) * 4, f.width, f.height); + } } else if let Some(dmabuf) = f.dmabuf.as_ref() { if !gpu_encoder { - return true; // GPU frames, CPU encoder — mismatch + return RETAINED_MISMATCH; // GPU frames, CPU encoder } host_enc_dmabuf = Some(dmabuf.clone()); } - false + RETAINED_OK }); - if bad_combo { - static WARNED: std::sync::atomic::AtomicBool = - std::sync::atomic::AtomicBool::new(false); - if !WARNED.swap(true, Ordering::Relaxed) { - eprintln!( - "[HostCapture] host frame type and encoder mismatch \ - (GPU host needs a GPU encoder; software host needs a CPU encoder)." - ); + // GPU path: composite the watermark and serve screenshot readbacks with + // the renderer. A bouncing watermark re-composes retained content into + // this display's offscreen target every tick (drawing in place would + // trail); anchored watermarks are stamped once onto each fresh blit and + // ride along with retained re-encodes. + if let Some(src) = host_enc_dmabuf.clone() { + if wm_active { + if let Some(renderer) = state.gles_renderer.as_mut() { + if wm_animated { + if let Some((_, target)) = node.offscreen_buffer.as_mut() { + match compose_host_watermark( + renderer, + &node.overlay_state, + &src, + target, + (width, height), + ) { + Ok(sync) => { + render_sync = Some(sync); + host_enc_dmabuf = Some(target.clone()); + wm_drawn = true; + } + Err(e) => warn_once_host_watermark(&e), + } + } + } else if have_new { + let mut target = src.clone(); + match draw_host_watermark( + renderer, + &node.overlay_state, + &mut target, + (width, height), + ) { + Ok(sync) => { + render_sync = Some(sync); + wm_drawn = true; + } + Err(e) => warn_once_host_watermark(&e), + } + } + } + } + if take_screenshot { + if let Some(renderer) = state.gles_renderer.as_mut() { + let mut shot = host_enc_dmabuf.clone().unwrap_or(src); + match renderer.bind(&mut shot) { + Ok(fb) => { + let rect = Rectangle::new((0, 0).into(), (width, height).into()); + match renderer.copy_framebuffer(&fb, rect, Fourcc::Abgr8888) { + Ok(mapping) => match renderer.map_texture(&mapping) { + Ok(data) => { + let n = data.len().min(node.frame_buffer.len()); + node.frame_buffer[..n].copy_from_slice(&data[..n]); + } + Err(e) => eprintln!("[HostCapture] screenshot map: {e:?}"), + }, + Err(e) => eprintln!("[HostCapture] screenshot copy: {e:?}"), + } + } + Err(e) => eprintln!("[HostCapture] screenshot bind: {e:?}"), + }; + } + } + } + if wm_drawn { + if let Some(rect) = node.overlay_state.damage_rect(width, height) { + damage_rects.push(rect); + } + } + state.host = Some(host); + if outcome != RETAINED_OK { + if outcome == RETAINED_MISMATCH { + static WARNED: std::sync::atomic::AtomicBool = + std::sync::atomic::AtomicBool::new(false); + if !WARNED.swap(true, Ordering::Relaxed) { + eprintln!( + "[HostCapture] host frame type and encoder mismatch \ + (GPU host needs a GPU encoder; software host needs a CPU encoder)." + ); + } } if let Some((id, buf)) = pool_slot.take() { if let Some(cap) = node.capture.as_ref() { @@ -2785,6 +2972,9 @@ fn create_output_on( state.space.map_output(&output, (x, y)); let global = output.create_global::(&state.dh); let damage_tracker = OutputDamageTracker::from_output(&output); + if let Some(host) = state.host.as_ref() { + host.set_layout(id, x, y); + } println!("[Wayland] Output {id} created: {width}x{height} @ ({x}, {y}) scale {scale:.2}."); state.output_nodes.push(wayland::frontend::OutputNode { id, @@ -2862,6 +3052,9 @@ fn reposition_output_on(state: &mut AppState, id: u32, x: i32, y: i32) -> bool { return false; } state.output_nodes[idx].pos = (x, y); + if let Some(host) = state.host.as_ref() { + host.set_layout(id, x, y); + } output.change_current_state(None, None, None, Some((x, y).into())); state.space.map_output(&output, (x, y)); let windows: Vec = state @@ -2888,6 +3081,9 @@ fn destroy_output_on(state: &mut AppState, id: u32) -> bool { } let Some(_) = state.node_idx_for_id(id) else { return false }; stop_capture_on_display(state, id); + if let Some(host) = state.host.as_ref() { + host.idle_output(id); + } wayland_owners().lock().unwrap().remove(&id); // Relocate while the node is still registered so output leave/enter both resolve. let windows: Vec = state diff --git a/pixelflux/src/wayland/host.rs b/pixelflux/src/wayland/host.rs index d0015da..1b3e11f 100644 --- a/pixelflux/src/wayland/host.rs +++ b/pixelflux/src/wayland/host.rs @@ -11,24 +11,31 @@ //! the encoder imports, so no CPU ever touches a frame. Without a GPU the same //! loop degrades to wl_shm buffers feeding the CPU encode pool. //! -//! One [`HostSession`] per backend: the capture loop runs on its own thread and -//! hands ready buffers to the calloop tick over a channel (slots are returned -//! after encode, bounding frames in flight); the input proxies are called from -//! the calloop thread directly (wayland-client proxies are thread-safe). +//! Every host `wl_output` maps to one selkies display id, in registry order: a +//! control thread on the primary connection owns wlr-output-management and +//! applies the whole layout (per-output custom modes plus row positions) as one +//! atomic configuration, while each output gets its own capture connection and +//! thread. Input proxies are called from the calloop thread directly +//! (wayland-client proxies are thread-safe); pointer coordinates arrive in the +//! same union-layout space selkies uses, so the virtual-pointer extent is the +//! layout's bounding box. All capture-side waits poll a wake pipe next to the +//! Wayland socket, so a resize or teardown lands immediately even while the +//! thread is parked in a `copy_with_damage` wait on a static screen. use std::fs::File; -use std::os::fd::AsFd; +use std::os::fd::{AsFd, AsRawFd, OwnedFd, RawFd}; use std::os::unix::net::UnixStream; use std::sync::atomic::{AtomicBool, Ordering}; use std::sync::mpsc::{Receiver, Sender, TryRecvError}; use std::sync::{Arc, Mutex}; +use std::time::{Duration, Instant}; use gbm::{BufferObjectFlags, Device as GbmDevice, Format as GbmFormat}; use smithay::backend::allocator::dmabuf::Dmabuf; use smithay::backend::allocator::Buffer as _; use smithay::utils::{Physical, Rectangle}; use wayland_client::protocol::{wl_output, wl_pointer, wl_registry, wl_seat, wl_shm, wl_shm_pool}; -use wayland_client::{delegate_noop, Connection, Dispatch, Proxy, QueueHandle, WEnum}; +use wayland_client::{delegate_noop, Connection, Dispatch, EventQueue, Proxy, QueueHandle, WEnum}; use wayland_protocols::wp::linux_dmabuf::zv1::client::{ zwp_linux_buffer_params_v1::{self, ZwpLinuxBufferParamsV1}, zwp_linux_dmabuf_v1::ZwpLinuxDmabufV1, @@ -54,16 +61,18 @@ use wayland_protocols_wlr::virtual_pointer::v1::client::{ }; use crate::wayland::wlclient::{ - bounded_roundtrip, impl_sync_callback, memfd_with, socket_path, SyncState, + bounded_roundtrip, drain_pipe, impl_sync_callback, memfd_with, socket_path, wait_readable2, + wake_pipe, wake_write, SyncState, }; const KEYMAP_FORMAT_XKB_V1: u32 = 1; -/// Frames in flight between the capture thread and the encode tick. Two slots: +/// Frames in flight between a capture thread and the encode tick. Two slots: /// one being blitted by the compositor while the previous one is encoded. const SLOTS: usize = 2; /// A frame the compositor finished blitting, ready for the encoder. pub struct HostFrame { + gen: u64, slot: usize, /// GPU path: the filled dmabuf (Arc-backed; the buffer itself stays owned by /// the capture thread and is reused once the slot is released). @@ -129,71 +138,91 @@ impl HostCpuFrame { } } +/// One display's slice of the union layout, as selkies configured it. Slots are +/// keyed by selkies display id ('display2' is id 2 — ids are sparse); active ids +/// map onto host outputs by ascending rank. +#[derive(Clone, Copy, Default)] +struct LayoutSlot { + want: (i32, i32), + pos: (i32, i32), + active: bool, +} + enum ToHost { Start { width: i32, height: i32 }, - Release(usize), - Stop, + /// Slot return; `gen` guards against slots recycled by a renegotiation + /// while the consumer still held the frame. + Release { gen: u64, slot: usize }, + /// Park the capture (display stopped) without ending the thread. + Idle, } -struct HostState { - shm: Option, - dmabuf: Option, - screencopy: Option, +enum CtrlMsg { + Apply(Vec), +} + +// --------------------------------------------------------------------------- +// Control connection: seat + virtual input devices + wlr-output-management. +// --------------------------------------------------------------------------- + +struct CtrlState { seat: Option, vk_mgr: Option, vptr_mgr: Option, - outputs: Vec, + has_screencopy: bool, + outputs: Vec<(wl_output::WlOutput, Option)>, + /// Registry indices in display order (natural name sort — registry + /// announcement order is not guaranteed to match output creation order). + order: Vec, output_mgr: Option, - heads: Vec, + heads: Vec<(ZwlrOutputHeadV1, Option)>, om_serial: Option, - /// Outcome of the pending output configuration: Some(true) applied, - /// Some(false) failed, retried internally on `cancelled`. cfg_result: Option, cfg_cancelled: bool, - // Per-frame capture negotiation/results. - announce_dmabuf: Option<(u32, i32, i32)>, // fourcc, w, h - announce_shm: Option<(u32, i32, i32, i32)>, // format, w, h, stride - buffer_done: bool, - damage: Vec>, - ready: bool, - failed: bool, sync_done: bool, } -impl Default for HostState { +impl Default for CtrlState { fn default() -> Self { Self { - shm: None, - dmabuf: None, - screencopy: None, seat: None, vk_mgr: None, vptr_mgr: None, + has_screencopy: false, outputs: Vec::new(), + order: Vec::new(), output_mgr: None, heads: Vec::new(), om_serial: None, cfg_result: None, cfg_cancelled: false, - announce_dmabuf: None, - announce_shm: None, - buffer_done: false, - damage: Vec::new(), - ready: false, - failed: false, sync_done: false, } } } -impl SyncState for HostState { +/// Natural sort key for an output name: text prefix plus trailing number, so +/// HEADLESS-2 orders before HEADLESS-10. Unnamed outputs keep registry order +/// after every named one. +fn output_order_key(name: Option<&String>, registry_idx: usize) -> (bool, String, u64, usize) { + match name { + Some(n) => { + let digits_at = n.rfind(|c: char| !c.is_ascii_digit()).map(|i| i + 1).unwrap_or(0); + let num = n[digits_at..].parse::().unwrap_or(0); + (false, n[..digits_at].to_string(), num, registry_idx) + } + None => (true, String::new(), 0, registry_idx), + } +} + +impl SyncState for CtrlState { fn sync_done_mut(&mut self) -> &mut bool { &mut self.sync_done } } -impl_sync_callback!(HostState); +impl_sync_callback!(CtrlState); -impl Dispatch for HostState { +impl Dispatch for CtrlState { fn event( state: &mut Self, registry: &wl_registry::WlRegistry, @@ -204,13 +233,6 @@ impl Dispatch for HostState { ) { if let wl_registry::Event::Global { name, interface, version } = event { match interface.as_str() { - "wl_shm" => state.shm = Some(registry.bind(name, 1, qh, ())), - "zwp_linux_dmabuf_v1" if version >= 3 => { - state.dmabuf = Some(registry.bind(name, 3, qh, ())) - } - "zwlr_screencopy_manager_v1" if version >= 3 => { - state.screencopy = Some(registry.bind(name, 3, qh, ())) - } "wl_seat" if state.seat.is_none() => { state.seat = Some(registry.bind(name, 1, qh, ())) } @@ -220,7 +242,12 @@ impl Dispatch for HostState { "zwlr_virtual_pointer_manager_v1" => { state.vptr_mgr = Some(registry.bind(name, version.min(2), qh, ())) } - "wl_output" => state.outputs.push(registry.bind(name, 1, qh, ())), + "zwlr_screencopy_manager_v1" if version >= 3 => state.has_screencopy = true, + "wl_output" => { + let idx = state.outputs.len(); + let out = registry.bind(name, version.min(4), qh, idx); + state.outputs.push((out, None)); + } "zwlr_output_manager_v1" => { state.output_mgr = Some(registry.bind(name, 1, qh, ())) } @@ -230,20 +257,37 @@ impl Dispatch for HostState { } } -delegate_noop!(HostState: ignore wl_seat::WlSeat); -delegate_noop!(HostState: ignore wl_output::WlOutput); -delegate_noop!(HostState: ignore wl_shm::WlShm); -delegate_noop!(HostState: ignore wl_shm_pool::WlShmPool); -delegate_noop!(HostState: ignore wayland_client::protocol::wl_buffer::WlBuffer); -delegate_noop!(HostState: ZwpVirtualKeyboardManagerV1); -delegate_noop!(HostState: ZwpVirtualKeyboardV1); -delegate_noop!(HostState: ZwlrVirtualPointerManagerV1); -delegate_noop!(HostState: ZwlrVirtualPointerV1); -delegate_noop!(HostState: ZwlrScreencopyManagerV1); -delegate_noop!(HostState: ignore ZwlrOutputModeV1); -delegate_noop!(HostState: ZwlrOutputConfigurationHeadV1); +macro_rules! impl_output_name { + ($t:ty) => { + impl Dispatch for $t { + fn event( + state: &mut Self, + _: &wl_output::WlOutput, + event: wl_output::Event, + idx: &usize, + _: &Connection, + _: &QueueHandle, + ) { + if let wl_output::Event::Name { name } = event { + if let Some(o) = state.outputs.get_mut(*idx) { + o.1 = Some(name); + } + } + } + } + }; +} +impl_output_name!(CtrlState); + +delegate_noop!(CtrlState: ignore wl_seat::WlSeat); +delegate_noop!(CtrlState: ZwpVirtualKeyboardManagerV1); +delegate_noop!(CtrlState: ZwpVirtualKeyboardV1); +delegate_noop!(CtrlState: ZwlrVirtualPointerManagerV1); +delegate_noop!(CtrlState: ZwlrVirtualPointerV1); +delegate_noop!(CtrlState: ignore ZwlrOutputModeV1); +delegate_noop!(CtrlState: ZwlrOutputConfigurationHeadV1); -impl Dispatch for HostState { +impl Dispatch for CtrlState { fn event( state: &mut Self, _: &ZwlrOutputManagerV1, @@ -253,18 +297,18 @@ impl Dispatch for HostState { _: &QueueHandle, ) { match event { - zwlr_output_manager_v1::Event::Head { head } => state.heads.push(head), + zwlr_output_manager_v1::Event::Head { head } => state.heads.push((head, None)), zwlr_output_manager_v1::Event::Done { serial } => state.om_serial = Some(serial), _ => {} } } - wayland_client::event_created_child!(HostState, ZwlrOutputManagerV1, [ + wayland_client::event_created_child!(CtrlState, ZwlrOutputManagerV1, [ zwlr_output_manager_v1::EVT_HEAD_OPCODE => (ZwlrOutputHeadV1, ()), ]); } -impl Dispatch for HostState { +impl Dispatch for CtrlState { fn event( state: &mut Self, head: &ZwlrOutputHeadV1, @@ -273,17 +317,25 @@ impl Dispatch for HostState { _: &Connection, _: &QueueHandle, ) { - if let zwlr_output_head_v1::Event::Finished = event { - state.heads.retain(|h| h.id() != head.id()); + match event { + zwlr_output_head_v1::Event::Name { name } => { + if let Some(h) = state.heads.iter_mut().find(|(h, _)| h.id() == head.id()) { + h.1 = Some(name); + } + } + zwlr_output_head_v1::Event::Finished => { + state.heads.retain(|(h, _)| h.id() != head.id()); + } + _ => {} } } - wayland_client::event_created_child!(HostState, ZwlrOutputHeadV1, [ + wayland_client::event_created_child!(CtrlState, ZwlrOutputHeadV1, [ zwlr_output_head_v1::EVT_MODE_OPCODE => (ZwlrOutputModeV1, ()), ]); } -impl Dispatch for HostState { +impl Dispatch for CtrlState { fn event( state: &mut Self, _: &ZwlrOutputConfigurationV1, @@ -301,7 +353,97 @@ impl Dispatch for HostState { } } -impl Dispatch for HostState { +// --------------------------------------------------------------------------- +// Capture connections: one per host output (screencopy + buffer allocation). +// --------------------------------------------------------------------------- + +struct CaptureState { + shm: Option, + dmabuf: Option, + screencopy: Option, + outputs: Vec<(wl_output::WlOutput, Option)>, + // Per-frame capture negotiation/results. + announce_dmabuf: Option<(u32, i32, i32)>, // fourcc, w, h + announce_shm: Option<(u32, i32, i32, i32)>, // format, w, h, stride + buffer_done: bool, + damage: Vec>, + ready: bool, + failed: bool, + sync_done: bool, +} + +impl Default for CaptureState { + fn default() -> Self { + Self { + shm: None, + dmabuf: None, + screencopy: None, + outputs: Vec::new(), + announce_dmabuf: None, + announce_shm: None, + buffer_done: false, + damage: Vec::new(), + ready: false, + failed: false, + sync_done: false, + } + } +} + +impl CaptureState { + fn reset_frame(&mut self) { + self.announce_dmabuf = None; + self.announce_shm = None; + self.buffer_done = false; + self.damage.clear(); + self.ready = false; + self.failed = false; + } +} + +impl SyncState for CaptureState { + fn sync_done_mut(&mut self) -> &mut bool { + &mut self.sync_done + } +} +impl_sync_callback!(CaptureState); +impl_output_name!(CaptureState); + +impl Dispatch for CaptureState { + fn event( + state: &mut Self, + registry: &wl_registry::WlRegistry, + event: wl_registry::Event, + _: &(), + _: &Connection, + qh: &QueueHandle, + ) { + if let wl_registry::Event::Global { name, interface, version } = event { + match interface.as_str() { + "wl_shm" => state.shm = Some(registry.bind(name, 1, qh, ())), + "zwp_linux_dmabuf_v1" if version >= 3 => { + state.dmabuf = Some(registry.bind(name, 3, qh, ())) + } + "zwlr_screencopy_manager_v1" if version >= 3 => { + state.screencopy = Some(registry.bind(name, 3, qh, ())) + } + "wl_output" => { + let idx = state.outputs.len(); + let out = registry.bind(name, version.min(4), qh, idx); + state.outputs.push((out, None)); + } + _ => {} + } + } + } +} + +delegate_noop!(CaptureState: ignore wl_shm::WlShm); +delegate_noop!(CaptureState: ignore wl_shm_pool::WlShmPool); +delegate_noop!(CaptureState: ignore wayland_client::protocol::wl_buffer::WlBuffer); +delegate_noop!(CaptureState: ZwlrScreencopyManagerV1); + +impl Dispatch for CaptureState { fn event( _: &mut Self, _: &ZwpLinuxDmabufV1, @@ -315,7 +457,7 @@ impl Dispatch for HostState { } } -impl Dispatch for HostState { +impl Dispatch for CaptureState { fn event( _: &mut Self, _: &ZwpLinuxBufferParamsV1, @@ -328,7 +470,7 @@ impl Dispatch for HostState { } } -impl Dispatch for HostState { +impl Dispatch for CaptureState { fn event( state: &mut Self, _: &ZwlrScreencopyFrameV1, @@ -375,35 +517,58 @@ enum SlotBuffer { }, } -/// The calloop-side handle: input proxies plus the frame channel. -pub struct HostSession { - conn: Connection, - vk: Option, - vptr: Option, +/// One host output's calloop-side handle: control channel, wake pipe, frames. +struct OutputHandle { to_thread: Sender, + wake: OwnedFd, frames: Receiver, /// Newest frame, kept (slot and all) until replaced so an IDR request on a /// static screen can re-encode current content like compositor mode does. retained: Mutex>, - pub display_size: Arc>, + name: Option, +} + +impl OutputHandle { + fn send(&self, msg: ToHost) { + let _ = self.to_thread.send(msg); + wake_write(self.wake.as_raw_fd()); + } +} + +/// The calloop-side session: input proxies plus one capture handle per output. +pub struct HostSession { + conn: Connection, + vk: Option, + vptr: Option, + ctrl_tx: Sender, + ctrl_wake: OwnedFd, + outputs: Vec, + layout: Mutex>, alive: Arc, } impl HostSession { - /// Connect to `display` and bring up input devices + the capture thread - /// (idle until [`start_capture`]). `gbm` enables the zero-copy path. - pub fn connect(display: &str, gbm: Option>) -> Result { + /// Connect to `display`, bring up input devices, enumerate the host's + /// outputs and spawn one capture thread per output (idle until + /// [`start_capture`]) plus the layout-control thread. `gbm_path` (this + /// process's render node) enables the zero-copy path. + pub fn connect(display: &str, gbm_path: Option) -> Result { let path = socket_path(display).ok_or("XDG_RUNTIME_DIR is unset")?; let stream = UnixStream::connect(&path).map_err(|e| format!("connect {path}: {e}"))?; let conn = Connection::from_socket(stream).map_err(|e| format!("wayland setup: {e}"))?; let mut queue = conn.new_event_queue(); let qh = queue.handle(); let _registry = conn.display().get_registry(&qh, ()); - let mut state = HostState::default(); + let mut state = CtrlState::default(); bounded_roundtrip(&conn, &mut queue, &mut state)?; let seat = state.seat.clone().ok_or("host compositor advertises no wl_seat")?; - state.screencopy.as_ref().ok_or("host compositor lacks zwlr_screencopy_manager_v1")?; + if !state.has_screencopy { + return Err("host compositor lacks zwlr_screencopy_manager_v1 (v3)".into()); + } + if state.outputs.is_empty() { + return Err("host compositor has no wl_output".into()); + } let vk = match &state.vk_mgr { Some(mgr) => { @@ -430,60 +595,165 @@ impl HostSession { None } }; + // Second round-trip: wl_output v4 name events and output-management heads. bounded_roundtrip(&conn, &mut queue, &mut state)?; - let (to_thread, from_main) = std::sync::mpsc::channel::(); - let (frame_tx, frames) = std::sync::mpsc::channel::(); let alive = Arc::new(AtomicBool::new(true)); - let display_size = Arc::new(Mutex::new((0, 0))); - { - let conn = conn.clone(); + let mut order: Vec = (0..state.outputs.len()).collect(); + order.sort_by_key(|&i| output_order_key(state.outputs[i].1.as_ref(), i)); + let names: Vec> = + order.iter().map(|&i| state.outputs[i].1.clone()).collect(); + state.order = order; + println!( + "[HostCapture] host outputs: {}.", + names + .iter() + .enumerate() + .map(|(i, n)| format!("{i}={}", n.as_deref().unwrap_or("?"))) + .collect::>() + .join(", ") + ); + + let mut outputs = Vec::new(); + for (i, name) in names.iter().cloned().enumerate() { + let (wake_rd, wake_wr) = wake_pipe()?; + let (to_thread, from_main) = std::sync::mpsc::channel::(); + let (frame_tx, frames) = std::sync::mpsc::channel::(); + let display = display.to_string(); + let expect = name.clone(); + let gbm_path = gbm_path.clone(); let alive = alive.clone(); - let display_size = display_size.clone(); std::thread::Builder::new() - .name("pf-host-capture".into()) + .name(format!("pf-host-cap{i}")) .spawn(move || { if let Err(e) = - capture_loop(conn, queue, state, gbm, from_main, frame_tx, display_size) + capture_loop(&display, i, expect, gbm_path, from_main, wake_rd, frame_tx) { - eprintln!("[HostCapture] capture loop ended: {e}"); + eprintln!("[HostCapture] output {i} capture ended: {e}"); + } + if i == 0 { + alive.store(false, Ordering::Relaxed); } - alive.store(false, Ordering::Relaxed); }) .map_err(|e| format!("spawn: {e}"))?; + outputs.push(OutputHandle { + to_thread, + wake: wake_wr, + frames, + retained: Mutex::new(None), + name, + }); + } + + let (ctrl_tx, ctrl_rx) = std::sync::mpsc::channel::(); + let (ctrl_wake_rd, ctrl_wake) = wake_pipe()?; + { + let conn = conn.clone(); + std::thread::Builder::new() + .name("pf-host-ctrl".into()) + .spawn(move || control_loop(conn, queue, state, ctrl_rx, ctrl_wake_rd)) + .map_err(|e| format!("spawn: {e}"))?; } - Ok(Self { - conn, - vk, - vptr, - to_thread, - frames, - retained: Mutex::new(None), - display_size, - alive, - }) + + let layout = Mutex::new(std::collections::BTreeMap::new()); + Ok(Self { conn, vk, vptr, ctrl_tx, ctrl_wake, outputs, layout, alive }) + } + + pub fn output_count(&self) -> usize { + self.outputs.len() } - pub fn start_capture(&self, width: i32, height: i32) { - let _ = self.to_thread.send(ToHost::Start { width, height }); + /// The host output backing `display_id`: active display ids map onto host + /// outputs by ascending rank (selkies ids are sparse — 'display2' is id 2). + fn output_index_for(&self, display_id: u32) -> Option { + let layout = self.layout.lock().unwrap(); + let rank = layout + .iter() + .filter(|(_, s)| s.active) + .position(|(id, _)| *id == display_id)?; + (rank < self.outputs.len()).then_some(rank) } - pub fn stop(&self) { - let _ = self.to_thread.send(ToHost::Stop); + /// True when an active capture on `display_id` has a host output behind it. + pub fn has_output_for(&self, display_id: u32) -> bool { + self.output_index_for(display_id).is_some() + } + + /// Record where selkies laid out `display_id` (union coordinates); pushed + /// to the host with the next capture (re)start. + pub fn set_layout(&self, display_id: u32, x: i32, y: i32) { + let mut layout = self.layout.lock().unwrap(); + layout.entry(display_id).or_default().pos = (x, y); + } + + /// Aim `display_id` at `width`x`height`: assigns every active display its + /// host output (by rank), asks the host for all modes and positions in one + /// atomic configuration, and points the capture threads at their sizes; + /// frames gate until the host applies them. + pub fn start_capture(&self, display_id: u32, width: i32, height: i32) { + let assignments = { + let mut layout = self.layout.lock().unwrap(); + let slot = layout.entry(display_id).or_default(); + slot.want = (width, height); + slot.active = true; + let active: Vec<(u32, LayoutSlot)> = layout + .iter() + .filter(|(_, s)| s.active) + .map(|(id, s)| (*id, *s)) + .collect(); + if active.iter().position(|(id, _)| *id == display_id).unwrap_or(usize::MAX) + >= self.outputs.len() + { + eprintln!( + "[HostCapture] display {display_id} has no host output (host has {}); not captured.", + self.outputs.len() + ); + } + active + }; + // Ranks may have shifted (a lower id joined): repoint every assigned + // capture thread; a same-size Start is a no-op for an unaffected one. + let mut by_output: Vec = Vec::new(); + for (rank, (_, slot)) in assignments.iter().enumerate() { + if rank >= self.outputs.len() { + break; + } + self.outputs[rank].send(ToHost::Start { width: slot.want.0, height: slot.want.1 }); + by_output.push(*slot); + } + let _ = self.ctrl_tx.send(CtrlMsg::Apply(by_output)); + wake_write(self.ctrl_wake.as_raw_fd()); + } + + /// Park `display_id`'s capture (its slot leaves the pointer extent). The + /// retained frame goes back to the capture pool — dropping it silently + /// would wedge one of the two slots until the next renegotiation. + pub fn idle_output(&self, display_id: u32) { + let Some(idx) = self.output_index_for(display_id) else { + self.layout.lock().unwrap().entry(display_id).or_default().active = false; + return; + }; + self.layout.lock().unwrap().entry(display_id).or_default().active = false; + self.outputs[idx].send(ToHost::Idle); + if let Some(old) = self.outputs[idx].retained.lock().unwrap().take() { + self.outputs[idx].send(ToHost::Release { gen: old.gen, slot: old.slot }); + } } pub fn alive(&self) -> bool { self.alive.load(Ordering::Relaxed) } - /// Newest ready frame, releasing any staler ones straight back to the pool. - pub fn try_take_frame(&self) -> Option { + /// Newest ready frame for `display_id`, releasing any staler ones straight + /// back to the pool. + pub fn try_take_frame(&self, display_id: u32) -> Option { + let handle = self.outputs.get(self.output_index_for(display_id)?)?; let mut newest: Option = None; loop { - match self.frames.try_recv() { + match handle.frames.try_recv() { Ok(frame) => { if let Some(stale) = newest.replace(frame) { - let _ = self.to_thread.send(ToHost::Release(stale.slot)); + handle.send(ToHost::Release { gen: stale.gen, slot: stale.slot }); } } Err(TryRecvError::Empty) | Err(TryRecvError::Disconnected) => break, @@ -492,22 +762,29 @@ impl HostSession { newest } - /// Return an encoded frame's slot for the host's next blit. - pub fn release_frame(&self, frame: HostFrame) { - let _ = self.to_thread.send(ToHost::Release(frame.slot)); - } - - /// Keep `frame` as the current content (releasing the one it replaces). - pub fn retain_frame(&self, frame: HostFrame) { - let old = self.retained.lock().unwrap().replace(frame); + /// Keep `frame` as `display_id`'s current content (releasing the one it + /// replaces). + pub fn retain_frame(&self, display_id: u32, frame: HostFrame) { + let Some(handle) = self.output_index_for(display_id).and_then(|i| self.outputs.get(i)) + else { + return; + }; + let old = handle.retained.lock().unwrap().replace(frame); if let Some(old) = old { - self.release_frame(old); + handle.send(ToHost::Release { gen: old.gen, slot: old.slot }); } } - /// Run `f` with the retained (current-content) frame, if any. - pub fn with_retained(&self, f: impl FnOnce(Option<&HostFrame>) -> R) -> R { - f(self.retained.lock().unwrap().as_ref()) + /// Run `f` with `display_id`'s retained (current-content) frame, if any. + pub fn with_retained( + &self, + display_id: u32, + f: impl FnOnce(Option<&HostFrame>) -> R, + ) -> R { + match self.output_index_for(display_id).and_then(|i| self.outputs.get(i)) { + Some(handle) => f(handle.retained.lock().unwrap().as_ref()), + None => f(None), + } } /// Upload selkies' managed keymap to the virtual keyboard verbatim. @@ -534,9 +811,25 @@ impl HostSession { let _ = self.conn.flush(); } + /// Union-layout bounding box of every active, output-backed display (the + /// virtual-pointer extent, matching the space selkies injects in). + fn extent(&self) -> (i32, i32) { + let layout = self.layout.lock().unwrap(); + let mut w = 0; + let mut h = 0; + for (rank, (_, slot)) in layout.iter().filter(|(_, s)| s.active).enumerate() { + if rank >= self.outputs.len() { + break; + } + w = w.max(slot.pos.0 + slot.want.0); + h = h.max(slot.pos.1 + slot.want.1); + } + (w, h) + } + pub fn pointer_motion_abs(&self, x: f64, y: f64) { let Some(vp) = &self.vptr else { return }; - let (w, h) = *self.display_size.lock().unwrap(); + let (w, h) = self.extent(); if w <= 0 || h <= 0 { return; } @@ -571,45 +864,125 @@ impl HostSession { } } -impl Drop for HostSession { - fn drop(&mut self) { - let _ = self.to_thread.send(ToHost::Stop); +// Dropping the session drops every channel sender and wake-pipe write end; the +// capture and control threads observe the disconnect (poll sees POLLHUP) and +// exit on their own. + +// --------------------------------------------------------------------------- +// Control thread: pumps the primary connection and applies layout requests. +// --------------------------------------------------------------------------- + +fn control_loop( + conn: Connection, + mut queue: EventQueue, + mut state: CtrlState, + ctrl_rx: Receiver, + wake_rd: OwnedFd, +) { + let qh = queue.handle(); + loop { + let mut pending: Option> = None; + loop { + match ctrl_rx.try_recv() { + Ok(CtrlMsg::Apply(slots)) => pending = Some(slots), + Err(TryRecvError::Empty) => break, + Err(TryRecvError::Disconnected) => return, + } + } + if let Some(slots) = pending { + apply_layout(&conn, &mut queue, &mut state, &qh, &slots); + } + if queue.dispatch_pending(&mut state).is_err() { + return; + } + let _ = queue.flush(); + if let Some(guard) = conn.prepare_read() { + match wait_readable2( + guard.connection_fd().as_raw_fd(), + wake_rd.as_raw_fd(), + Some(Duration::from_secs(1)), + ) { + Ok((wl, wake)) => { + if wake { + drain_pipe(wake_rd.as_raw_fd()); + } + if wl { + let _ = guard.read(); + } else { + drop(guard); + } + } + Err(_) => return, + } + } } } -/// Ask the host (wlr-output-management) to set a custom mode on its first head. -/// Retries once across a `cancelled` (stale serial). False when the host lacks -/// the protocol or refused the mode. -fn resize_host_output( - queue: &mut wayland_client::EventQueue, - state: &mut HostState, - qh: &QueueHandle, - width: i32, - height: i32, -) -> bool { - let Some(mgr) = state.output_mgr.clone() else { return false }; - for _ in 0..2 { - for _ in 0..50 { - if state.om_serial.is_some() && !state.heads.is_empty() { - break; - } - if queue.blocking_dispatch(state).is_err() { - return false; +/// Ask the host (wlr-output-management) to give every active output its wanted +/// mode and layout position in one atomic configuration. Retries across a +/// `cancelled` (stale serial); a refusal only warns — capture threads keep +/// gating on the size they were promised. +fn apply_layout( + conn: &Connection, + queue: &mut EventQueue, + state: &mut CtrlState, + qh: &QueueHandle, + slots: &[LayoutSlot], +) { + let Some(mgr) = state.output_mgr.clone() else { + eprintln!("[HostCapture] host lacks zwlr_output_manager_v1; capture follows the host's own size."); + return; + }; + let deadline = Instant::now() + Duration::from_secs(5); + for _ in 0..3 { + while state.om_serial.is_none() && Instant::now() < deadline { + if !pump_ctrl(conn, queue, state) { + return; } } - let (Some(serial), Some(head)) = (state.om_serial, state.heads.first().cloned()) else { - return false; + let Some(serial) = state.om_serial else { + eprintln!("[HostCapture] output-management serial never arrived; resize skipped."); + return; }; state.cfg_result = None; state.cfg_cancelled = false; let cfg = mgr.create_configuration(serial, qh, ()); - let cfg_head = cfg.enable_head(&head, qh, ()); - cfg_head.set_custom_mode(width, height, 0); + let mut any = false; + for (i, slot) in slots.iter().enumerate() { + if !slot.active { + continue; + } + let want_name = state + .order + .get(i) + .and_then(|&oi| state.outputs.get(oi)) + .and_then(|(_, n)| n.clone()); + let head = match want_name + .as_ref() + .and_then(|n| state.heads.iter().find(|(_, hn)| hn.as_ref() == Some(n))) + .or_else(|| state.heads.get(i)) + { + Some((h, _)) => h.clone(), + None => { + eprintln!("[HostCapture] no output-management head for output {i}; not resized."); + continue; + } + }; + let cfg_head = cfg.enable_head(&head, qh, ()); + cfg_head.set_custom_mode(slot.want.0, slot.want.1, 0); + cfg_head.set_position(slot.pos.0, slot.pos.1); + any = true; + } + if !any { + cfg.destroy(); + return; + } cfg.apply(); let _ = queue.flush(); - while state.cfg_result.is_none() && !state.cfg_cancelled { - if queue.blocking_dispatch(state).is_err() { - return false; + while state.cfg_result.is_none() && !state.cfg_cancelled && Instant::now() < deadline { + if !pump_ctrl(conn, queue, state) { + cfg.destroy(); + return; } } cfg.destroy(); @@ -618,11 +991,40 @@ fn resize_host_output( state.om_serial = None; continue; } - return state.cfg_result == Some(true); + if state.cfg_result != Some(true) { + eprintln!("[HostCapture] host refused the layout; capture follows the host's own size."); + } + return; + } +} + +/// One bounded dispatch step for the control connection (false = connection died). +fn pump_ctrl(conn: &Connection, queue: &mut EventQueue, state: &mut CtrlState) -> bool { + if queue.dispatch_pending(state).is_err() { + return false; + } + let _ = queue.flush(); + let Some(guard) = conn.prepare_read() else { return true }; + match crate::wayland::wlclient::wait_readable( + guard.connection_fd().as_raw_fd(), + Duration::from_millis(200), + ) { + Ok(readable) => { + if readable { + let _ = guard.read(); + } else { + drop(guard); + } + true + } + Err(_) => false, } - false } +// --------------------------------------------------------------------------- +// Capture threads: one connection + screencopy loop per host output. +// --------------------------------------------------------------------------- + fn fourcc_to_gbm(fourcc: u32) -> GbmFormat { // XR24 / AR24; anything else falls back to ARGB (the encoder reads BGRA bytes // and ignores alpha). @@ -632,43 +1034,120 @@ fn fourcc_to_gbm(fourcc: u32) -> GbmFormat { } } +/// What a control-channel drain decided while a capture wait was in progress. +enum Ctl { + None, + Renegotiate, + Idle, + Dead, +} + +enum Pump { + Done, + Control, + Timeout, +} + +/// Dispatch events until `done(state)` holds, waiting on the Wayland socket and +/// the wake pipe together so control messages interrupt any capture wait. The +/// predicate is re-checked after every dispatch, so a wait whose condition was +/// satisfied by already-queued events returns without ever blocking. +fn pump_until( + conn: &Connection, + queue: &mut EventQueue, + state: &mut CaptureState, + wake_rd: RawFd, + timeout: Option, + done: impl Fn(&CaptureState) -> bool, +) -> Result { + let deadline = timeout.map(|t| Instant::now() + t); + loop { + queue.dispatch_pending(state).map_err(|e| format!("dispatch: {e}"))?; + if done(state) { + return Ok(Pump::Done); + } + queue.flush().map_err(|e| format!("flush: {e}"))?; + let remaining = match deadline { + Some(d) => match d.checked_duration_since(Instant::now()) { + Some(r) => Some(r), + None => return Ok(Pump::Timeout), + }, + None => None, + }; + let Some(guard) = conn.prepare_read() else { continue }; + let (wl, wake) = wait_readable2(guard.connection_fd().as_raw_fd(), wake_rd, remaining)?; + if wake { + drop(guard); + drain_pipe(wake_rd); + return Ok(Pump::Control); + } + if wl { + let _ = guard.read(); + } else { + drop(guard); + if deadline.is_some_and(|d| Instant::now() >= d) { + return Ok(Pump::Timeout); + } + } + } +} + fn capture_loop( - _conn: Connection, - mut queue: wayland_client::EventQueue, - mut state: HostState, - gbm: Option>, + display: &str, + index: usize, + expect_name: Option, + gbm_path: Option, from_main: Receiver, + wake_rd: OwnedFd, frame_tx: Sender, - display_size: Arc>, ) -> Result<(), String> { - // Idle until capture starts. - let (mut want_w, mut want_h) = loop { - match from_main.recv() { - Ok(ToHost::Start { width, height }) => break (width, height), - Ok(ToHost::Release(..)) => continue, - Ok(ToHost::Stop) | Err(_) => return Ok(()), + let path = socket_path(display).ok_or("XDG_RUNTIME_DIR is unset")?; + let stream = UnixStream::connect(&path).map_err(|e| format!("connect {path}: {e}"))?; + let conn = Connection::from_socket(stream).map_err(|e| format!("wayland setup: {e}"))?; + let mut queue = conn.new_event_queue(); + let qh = queue.handle(); + let _registry = conn.display().get_registry(&qh, ()); + let mut state = CaptureState::default(); + bounded_roundtrip(&conn, &mut queue, &mut state)?; + // Names (wl_output v4) arrive after the binds from the first round-trip. + bounded_roundtrip(&conn, &mut queue, &mut state)?; + + let screencopy = state.screencopy.clone().ok_or("no zwlr_screencopy_manager_v1")?; + // Select this thread's output by the name the control connection assigned + // it; ordering fallback only when the host names nothing. + let by_name = expect_name.as_ref().and_then(|expect| { + state.outputs.iter().find(|(_, n)| n.as_ref() == Some(expect)).cloned() + }); + let (output, _name) = match by_name { + Some(o) => o, + None => { + let mut order: Vec = (0..state.outputs.len()).collect(); + order.sort_by_key(|&i| output_order_key(state.outputs[i].1.as_ref(), i)); + let oi = *order + .get(index) + .ok_or_else(|| format!("host has no wl_output {index}"))?; + state.outputs[oi].clone() } }; - let qh = queue.handle(); - let output = state.outputs.first().cloned().ok_or("host compositor has no wl_output")?; - let screencopy = state.screencopy.clone().unwrap(); - if !resize_host_output(&mut queue, &mut state, &qh, want_w, want_h) { - eprintln!( - "[HostCapture] host output resize to {want_w}x{want_h} not applied; \ - capture follows the host's own size." - ); - } + let gbm = gbm_path + .as_ref() + .and_then(|p| File::options().read(true).write(true).open(p).ok()) + .and_then(|f| GbmDevice::new(f).ok()); + let wake = wake_rd.as_raw_fd(); let mut slots: Vec> = (0..SLOTS).map(|_| None).collect(); let mut free: Vec = (0..SLOTS).collect(); + let mut gen: u64 = 0; let mut announced: Option<(i32, i32)> = None; let mut warned_mismatch = false; let mut consecutive_failures = 0u32; + let mut want: Option<(i32, i32)> = None; - loop { - // Drain control messages; block only when out of buffers. + 'main: loop { + // Drain control; block while idle or out of slots. loop { - let msg = if free.is_empty() { + let blocking = want.is_none() || free.is_empty(); + let msg = if blocking { match from_main.recv() { Ok(m) => m, Err(_) => return Ok(()), @@ -681,44 +1160,62 @@ fn capture_loop( } }; match msg { - ToHost::Release(slot) => free.push(slot), - ToHost::Stop => return Ok(()), + ToHost::Release { gen: g, slot } => { + if g == gen { + free.push(slot); + } + } + ToHost::Idle => want = None, ToHost::Start { width, height } => { - // A capture reconfigure (client resolution change): resize the - // host output to follow, exactly like compositor mode does. - if (width, height) != (want_w, want_h) { - want_w = width; - want_h = height; + if want != Some((width, height)) { warned_mismatch = false; - if !resize_host_output(&mut queue, &mut state, &qh, want_w, want_h) { - eprintln!( - "[HostCapture] host output resize to {want_w}x{want_h} \ - not applied; capture follows the host's own size." - ); - } } + want = Some((width, height)); } } } + let (want_w, want_h) = match want { + Some(w) => w, + None => continue, + }; - // One frame: negotiate (first time), attach our buffer, wait for damage. - state.announce_dmabuf = None; - state.announce_shm = None; - state.buffer_done = false; - state.damage.clear(); - state.ready = false; - state.failed = false; + // One frame: negotiate, attach our buffer, wait for damage. Control + // messages (resize, idle, teardown) interrupt any of the waits. + state.reset_frame(); let frame = screencopy.capture_output(1, &output, &qh, ()); - while !state.buffer_done && !state.failed { - queue.blocking_dispatch(&mut state).map_err(|e| format!("dispatch: {e}"))?; + loop { + match pump_until(&conn, &mut queue, &mut state, wake, None, |s| { + s.buffer_done || s.failed + })? { + Pump::Done => break, + Pump::Control => match drain_ctl(&from_main, gen, &mut free, &mut want) { + Ctl::None => {} + Ctl::Renegotiate | Ctl::Idle => { + frame.destroy(); + continue 'main; + } + Ctl::Dead => { + frame.destroy(); + return Ok(()); + } + }, + Pump::Timeout => {} + } } if state.failed { frame.destroy(); consecutive_failures += 1; if consecutive_failures == 3 { - eprintln!("[HostCapture] repeated screencopy failures; is the output alive?"); + eprintln!("[HostCapture] output {index}: repeated screencopy failures; is the output alive?"); } - std::thread::sleep(std::time::Duration::from_millis(100)); + let _ = pump_until( + &conn, + &mut queue, + &mut state, + wake, + Some(Duration::from_millis(100)), + |_| false, + ); continue; } @@ -729,19 +1226,20 @@ fn capture_loop( .ok_or("screencopy announced no buffer type")?; if announced != Some((fw, fh)) { announced = Some((fw, fh)); - *display_size.lock().unwrap() = (fw, fh); eprintln!( - "[HostCapture] negotiated: {fw}x{fh} gbm={} dmabuf_global={} dmabuf_announce={:?} shm={:?}", + "[HostCapture] output {index} negotiated: {fw}x{fh} gbm={} dmabuf_global={} dmabuf_announce={:?} shm={:?}", gbm.is_some(), state.dmabuf.is_some(), state.announce_dmabuf, state.announce_shm, ); - // Size change: drop stale buffers. + // Size change: drop stale buffers. The generation bump makes any + // release still in flight for the old buffers a no-op. for s in slots.iter_mut() { *s = None; } free = (0..SLOTS).collect(); + gen += 1; } if (fw, fh) != (want_w, want_h) { // The encoder is configured for (want_w, want_h): hold frames until @@ -750,10 +1248,17 @@ fn capture_loop( if !warned_mismatch { warned_mismatch = true; eprintln!( - "[HostCapture] waiting for host output {want_w}x{want_h} (currently {fw}x{fh})." + "[HostCapture] output {index}: waiting for host {want_w}x{want_h} (currently {fw}x{fh})." ); } - std::thread::sleep(std::time::Duration::from_millis(150)); + let _ = pump_until( + &conn, + &mut queue, + &mut state, + wake, + Some(Duration::from_millis(150)), + |_| false, + ); continue; } warned_mismatch = false; @@ -821,17 +1326,43 @@ fn capture_loop( frame.copy_with_damage(wl); } queue.flush().map_err(|e| format!("flush: {e}"))?; - while !state.ready && !state.failed { - queue.blocking_dispatch(&mut state).map_err(|e| format!("dispatch: {e}"))?; + let mut aborted = false; + loop { + match pump_until(&conn, &mut queue, &mut state, wake, None, |s| s.ready || s.failed)? { + Pump::Done => break, + Pump::Control => match drain_ctl(&from_main, gen, &mut free, &mut want) { + Ctl::None => {} + Ctl::Renegotiate | Ctl::Idle => { + aborted = true; + break; + } + Ctl::Dead => { + frame.destroy(); + return Ok(()); + } + }, + Pump::Timeout => {} + } } frame.destroy(); + if aborted { + free.push(slot_idx); + continue 'main; + } if state.failed { free.push(slot_idx); consecutive_failures += 1; if consecutive_failures == 3 { - eprintln!("[HostCapture] repeated screencopy failures; is the output alive?"); + eprintln!("[HostCapture] output {index}: repeated screencopy failures; is the output alive?"); } - std::thread::sleep(std::time::Duration::from_millis(50)); + let _ = pump_until( + &conn, + &mut queue, + &mut state, + wake, + Some(Duration::from_millis(50)), + |_| false, + ); continue; } consecutive_failures = 0; @@ -839,6 +1370,7 @@ fn capture_loop( let damage = std::mem::take(&mut state.damage); let out = match slots[slot_idx].as_ref().unwrap() { SlotBuffer::Gpu { dmabuf, .. } => HostFrame { + gen, slot: slot_idx, dmabuf: Some(dmabuf.clone()), cpu: None, @@ -847,6 +1379,7 @@ fn capture_loop( damage, }, SlotBuffer::Cpu { map, stride, format, .. } => HostFrame { + gen, slot: slot_idx, dmabuf: None, cpu: Some(HostCpuFrame { @@ -864,3 +1397,35 @@ fn capture_loop( } } } + +/// Drain the control channel from inside a capture wait: releases are applied +/// (generation-checked), and the newest Start/Idle decides the wait's fate. +fn drain_ctl( + rx: &Receiver, + gen: u64, + free: &mut Vec, + want: &mut Option<(i32, i32)>, +) -> Ctl { + let mut out = Ctl::None; + loop { + match rx.try_recv() { + Ok(ToHost::Release { gen: g, slot }) => { + if g == gen { + free.push(slot); + } + } + Ok(ToHost::Start { width, height }) => { + if *want != Some((width, height)) { + *want = Some((width, height)); + out = Ctl::Renegotiate; + } + } + Ok(ToHost::Idle) => { + *want = None; + out = Ctl::Idle; + } + Err(TryRecvError::Empty) => return out, + Err(TryRecvError::Disconnected) => return Ctl::Dead, + } + } +} diff --git a/pixelflux/src/wayland/wlclient.rs b/pixelflux/src/wayland/wlclient.rs index 4841627..7979c96 100644 --- a/pixelflux/src/wayland/wlclient.rs +++ b/pixelflux/src/wayland/wlclient.rs @@ -68,6 +68,50 @@ pub(crate) fn wait_readable(fd: RawFd, timeout: Duration) -> Result, +) -> Result<(bool, bool), String> { + loop { + let mut pfds = [ + libc::pollfd { fd: a, events: libc::POLLIN, revents: 0 }, + libc::pollfd { fd: b, events: libc::POLLIN, revents: 0 }, + ]; + let ms = timeout.map(|t| t.as_millis().max(1) as libc::c_int).unwrap_or(-1); + let n = unsafe { libc::poll(pfds.as_mut_ptr(), 2, ms) }; + if n < 0 { + let err = std::io::Error::last_os_error(); + if err.raw_os_error() == Some(libc::EINTR) { + continue; + } + return Err(format!("poll: {err}")); + } + let hit = |r: libc::c_short| r & (libc::POLLIN | libc::POLLHUP | libc::POLLERR) != 0; + return Ok((hit(pfds[0].revents), hit(pfds[1].revents))); + } +} + +/// Drain every pending byte from a wake pipe without blocking. +pub(crate) fn drain_pipe(fd: RawFd) { + let mut buf = [0u8; 64]; + loop { + let n = unsafe { libc::read(fd, buf.as_mut_ptr() as *mut libc::c_void, buf.len()) }; + if n <= 0 { + return; + } + } +} + +/// One wake byte, non-blocking; a full pipe already guarantees a pending wake. +pub(crate) fn wake_write(fd: RawFd) { + let b = [1u8]; + unsafe { libc::write(fd, b.as_ptr() as *const libc::c_void, 1) }; +} + /// `EventQueue::roundtrip` with a deadline: a wedged compositor becomes an error /// instead of hanging the calling thread (and everything queued behind it). pub(crate) fn bounded_roundtrip( @@ -137,6 +181,15 @@ pub(crate) fn pipe_cloexec() -> Result<(OwnedFd, OwnedFd), String> { Ok(unsafe { (OwnedFd::from_raw_fd(fds[0]), OwnedFd::from_raw_fd(fds[1])) }) } +/// Non-blocking CLOEXEC pipe for wake signalling (read end, write end). +pub(crate) fn wake_pipe() -> Result<(OwnedFd, OwnedFd), String> { + let mut fds = [0i32; 2]; + if unsafe { libc::pipe2(fds.as_mut_ptr(), libc::O_CLOEXEC | libc::O_NONBLOCK) } < 0 { + return Err(format!("pipe2: {}", std::io::Error::last_os_error())); + } + Ok(unsafe { (OwnedFd::from_raw_fd(fds[0]), OwnedFd::from_raw_fd(fds[1])) }) +} + /// Read `fd` to EOF. The deadline is per-chunk (idle), so a large transfer that /// keeps flowing is never cut off while a stalled writer still errors out. pub(crate) fn read_fd_to_end(fd: &OwnedFd, idle: Duration) -> Result, String> { diff --git a/pixelflux/src/x11/mod.rs b/pixelflux/src/x11/mod.rs index 386419c..7fa73e2 100644 --- a/pixelflux/src/x11/mod.rs +++ b/pixelflux/src/x11/mod.rs @@ -33,7 +33,7 @@ use x11rb::protocol::xfixes::ConnectionExt as XfixesExt; use x11rb::protocol::xproto::{ConnectionExt as XprotoExt, ImageFormat}; use x11rb::rust_connection::RustConnection; -use crate::encoders::overlay::WatermarkLocation; +use crate::encoders::overlay::blend_pixel; use crate::encoders::software::EncodedStripe; use crate::pipeline::X11Pipeline; use crate::recording_sink::RecordingSink; @@ -262,28 +262,6 @@ fn grab_frame( Ok(()) } -/// Alpha-blend a source pixel (pre-split into r,g,b,a) over a BGRA destination pixel. -/// -/// Cursor and watermark pixels are overwhelmingly either fully opaque or fully transparent, and this -/// runs per pixel per frame on the CPU, so the two extremes are special-cased to skip the blend -/// arithmetic entirely: an opaque source (`a == 255`) simply overwrites, a fully transparent source -/// (`a == 0`) is left as-is, and only genuine edge pixels pay for the integer source-over. In every -/// case only the B / G / R bytes are written; the destination's alpha byte is left as the capture -/// delivered it. -#[inline] -fn blend_pixel(dst: &mut [u8], r: u8, g: u8, b: u8, a: u8) { - if a == 255 { - dst[0] = b; - dst[1] = g; - dst[2] = r; - } else if a > 0 { - let ia = 255 - a as u32; - dst[0] = ((b as u32 * a as u32 + dst[0] as u32 * ia) / 255) as u8; - dst[1] = ((g as u32 * a as u32 + dst[1] as u32 * ia) / 255) as u8; - dst[2] = ((r as u32 * a as u32 + dst[2] as u32 * ia) / 255) as u8; - } -} - /// Frame-space top-left of the cursor image, given the XFixes hotspot position. /// /// XFixes reports the cursor position at its HOTSPOT; X draws the image with its top-left at @@ -331,147 +309,6 @@ pub(crate) fn overlay_cursor( } } -/// CPU watermark: the raw RGBA pixels (row-major, `w*h*4`) in host memory plus the -/// placement / animation state, blended directly into the captured BGRA frame. -/// -/// The blend is on the CPU because the frame is already sitting in host shm memory before it reaches -/// any encoder, so stamping the overlay there is both the cheapest place to do it and the one place -/// it applies identically regardless of which encoder (hardware or software) runs downstream. The -/// pixels are kept as RGBA straight from `image`'s decode and converted per pixel as they are blended. -struct X11Watermark { - pixels: Vec, - w: i32, - h: i32, - pos_x: i32, - pos_y: i32, - sub_x: f64, - sub_y: f64, - vel_x: f64, - vel_y: f64, - loaded: bool, -} - -impl X11Watermark { - /// Load the watermark image at `path` into host RGBA, or return an unloaded stub. - /// - /// An empty path or any decode failure yields `loaded == false`, which every method treats as a - /// no-op, so a missing or broken watermark simply disables the overlay. The bouncing-animation - /// velocities are seeded here. - fn load(path: &str) -> Self { - let mut wm = Self { - pixels: Vec::new(), - w: 0, - h: 0, - pos_x: 0, - pos_y: 0, - sub_x: 0.0, - sub_y: 0.0, - vel_x: 2.0, - vel_y: 2.0, - loaded: false, - }; - if path.is_empty() { - return wm; - } - if let Ok(img) = image::open(std::path::Path::new(path)) { - let rgba = img.to_rgba8(); - wm.w = rgba.width() as i32; - wm.h = rgba.height() as i32; - wm.pixels = rgba.into_vec(); - wm.loaded = wm.w > 0 && wm.h > 0; - } - wm - } - - /// Set the watermark's top-left placement for this frame from the location enum; a stub - /// (unloaded) watermark returns immediately. - /// - /// The fixed corners and center are direct arithmetic. The animated mode advances a bouncing - /// position and reflects velocity off each frame edge, accumulating in the fractional `sub_x` / - /// `sub_y` rather than the integer `pos_x` / `pos_y`: a sub-pixel-per-frame velocity has to - /// survive between frames or the motion would either stall or snap by whole pixels, so the float - /// carries the remainder and `pos_x` / `pos_y` take its floor for the actual blit. - fn update_position(&mut self, frame_w: i32, frame_h: i32, loc_enum: i32) { - if !self.loaded { - return; - } - match WatermarkLocation::from(loc_enum) { - WatermarkLocation::TL => { - self.pos_x = 0; - self.pos_y = 0; - } - WatermarkLocation::TR => { - self.pos_x = frame_w - self.w; - self.pos_y = 0; - } - WatermarkLocation::BL => { - self.pos_x = 0; - self.pos_y = frame_h - self.h; - } - WatermarkLocation::BR => { - self.pos_x = frame_w - self.w; - self.pos_y = frame_h - self.h; - } - WatermarkLocation::MI => { - self.pos_x = (frame_w - self.w) / 2; - self.pos_y = (frame_h - self.h) / 2; - } - WatermarkLocation::AN => { - self.sub_x += self.vel_x; - self.sub_y += self.vel_y; - if self.sub_x <= 0.0 { - self.sub_x = 0.0; - self.vel_x = self.vel_x.abs(); - } else if self.sub_x + self.w as f64 >= frame_w as f64 { - self.sub_x = (frame_w - self.w) as f64; - self.vel_x = -self.vel_x.abs(); - } - if self.sub_y <= 0.0 { - self.sub_y = 0.0; - self.vel_y = self.vel_y.abs(); - } else if self.sub_y + self.h as f64 >= frame_h as f64 { - self.sub_y = (frame_h - self.h) as f64; - self.vel_y = -self.vel_y.abs(); - } - self.pos_x = self.sub_x as i32; - self.pos_y = self.sub_y as i32; - } - WatermarkLocation::None => (), - } - } - - /// Alpha-blend the loaded watermark into the BGRA frame at its current position; a no-op - /// when unloaded. Clips per pixel at the frame bounds because the animated position — or a - /// watermark larger than the capture — can leave part of the image off-frame, and only the - /// in-frame portion may be written. - fn blend_into(&self, frame: &mut [u8], stride: usize, frame_w: i32, frame_h: i32) { - if !self.loaded { - return; - } - for y in 0..self.h { - let ty = self.pos_y + y; - if ty < 0 || ty >= frame_h { - continue; - } - for x in 0..self.w { - let tx = self.pos_x + x; - if tx < 0 || tx >= frame_w { - continue; - } - let src = ((y * self.w + x) * 4) as usize; - let (r, g, b, a) = ( - self.pixels[src], - self.pixels[src + 1], - self.pixels[src + 2], - self.pixels[src + 3], - ); - let off = ty as usize * stride + tx as usize * 4; - blend_pixel(&mut frame[off..off + 4], r, g, b, a); - } - } - } -} - /// A captured raw BGRA frame held in a pooled shm surface, ready to encode. /// /// It carries the surface pointer plus geometry so the encode thread reads the pixels directly (no @@ -899,7 +736,10 @@ where } let pool = Arc::new(FramePool::new(POOL_N)); - let mut watermark = X11Watermark::load(&settings.watermark_path); + let mut watermark = crate::encoders::overlay::OverlayState::default(); + if !settings.watermark_path.is_empty() { + watermark.load_watermark(&settings.watermark_path, 1.0); + } let enc_pool = pool.clone(); let enc_controls = controls.clone(); @@ -1063,9 +903,9 @@ where } } - if watermark.loaded { + if watermark.is_active() { watermark.update_position(frame_w, frame_h, settings.watermark_location_enum); - watermark.blend_into(buf, stride, frame_w, frame_h); + watermark.blend_bgra(buf, stride, frame_w, frame_h); } let published = pool.publish( From 78e16d088f572b81e90d3452eea3146c0af76786 Mon Sep 17 00:00:00 2001 From: Seungmin Kim <8457324+ehfd@users.noreply.github.com> Date: Mon, 20 Jul 2026 15:03:58 +0900 Subject: [PATCH 06/21] fix(display): Clean up display mechanics --- example/screen_to_browser.py | 11 ++++-- pixelflux/src/lib.rs | 60 +++++++++++++++---------------- pixelflux/src/recorder/mod.rs | 5 ++- pixelflux/src/recording_sink.rs | 41 ++++++++++++++++----- pixelflux/src/wayland/frontend.rs | 16 +++++++++ pixelflux/src/x11/cursor.rs | 54 +++++++++++++++++++++++----- pixelflux/src/x11/mod.rs | 4 +-- 7 files changed, 136 insertions(+), 55 deletions(-) diff --git a/example/screen_to_browser.py b/example/screen_to_browser.py index 6ec8e5f..ff27afa 100644 --- a/example/screen_to_browser.py +++ b/example/screen_to_browser.py @@ -256,9 +256,14 @@ def on_stripe(frame): # Copy fallback: the frame already carries its native wire header. item_to_queue = {'data': bytes(frame), 'owner': None} - asyncio.run_coroutine_threadsafe( - client_queue.put(item_to_queue), g_loop - ) + try: + asyncio.run_coroutine_threadsafe( + client_queue.put(item_to_queue), g_loop + ) + except RuntimeError: + # Loop closed between the check and the call (teardown race); + # dropping the item's references recycles the frame buffer. + pass # --- 4. Register and Start Resources for this Client --- send_task = asyncio.create_task(send_stripes_task(websocket, client_queue)) diff --git a/pixelflux/src/lib.rs b/pixelflux/src/lib.rs index bee10e7..676c758 100644 --- a/pixelflux/src/lib.rs +++ b/pixelflux/src/lib.rs @@ -1340,9 +1340,7 @@ fn wayland_encode_loop(pool: &WlFramePool, cfg: WlEncodeConfig) -> Option = None; @@ -1620,6 +1616,11 @@ fn start_capture_on_display( prior_readback_encoder = teardown_capture(&mut old); } + // Bind only after the old capture is gone: its sink unlinks the socket path in + // Drop, which would strip a fresh bind's filesystem name and leave every later + // recorder connect with ENOENT. + let recording_sink = crate::recording_sink::RecordingSink::try_bind(&settings.recording_socket); + { // Never panic the compositor thread: an output momentarily without a current // mode falls back to the requested geometry so the reconfigure below is a @@ -1943,16 +1944,10 @@ fn start_capture_on_display( log_stream_settings(&settings, 1, cap.video_encoder.as_ref(), false); } drop(prior_readback_encoder); - // Force the keyframe on the now-active path (as RequestIdr does), unconditionally: - // the damage tracker and offscreen buffer stay warm across stop/start, so a - // restarted capture on a static screen otherwise produces no damage, no first - // frame, and no IDR in either path. - if cap.encode_pool.is_some() { - cap.encode_controls.force_idr.store(true, Ordering::Relaxed); - } else { - cap.pending_force_idr = true; - } - cap.needs_full_render = true; + // Force the keyframe unconditionally: the damage tracker and offscreen buffer + // stay warm across stop/start, so a restarted capture on a static screen + // otherwise produces no damage, no first frame, and no IDR in either path. + cap.request_idr(); node.capture = Some(cap); state.output_nodes.insert(node_idx, node); @@ -2091,8 +2086,7 @@ fn render_node_tick( .map(|s| s.should_force_idr()) .unwrap_or(false) { - cap.pending_force_idr = true; - cap.encode_controls.force_idr.store(true, Ordering::Relaxed); + cap.request_idr(); } } let requested_idr = node.capture.as_ref().map(|c| c.pending_force_idr).unwrap_or(false); @@ -2790,9 +2784,7 @@ fn render_node_tick( stripe_height: height, frame_id: cap.frame_counter as i32, }]; if let Some(ref socket) = cap.recording_sink { - for stripe in &stripes { - socket.write_encoded_frame(stripe); - } + socket.write_frame(&stripes, height); } crate::recorder::wayland_tap(node.id, &stripes); // Non-blocking: a full slot parks the frame (delivered @@ -3790,12 +3782,7 @@ fn run_wayland_thread( ThreadCommand::RequestIdr { display_id } => { if let Some(idx) = state.node_idx_for_id(display_id) { if let Some(cap) = state.output_nodes[idx].capture.as_mut() { - if cap.encode_pool.is_some() { - cap.encode_controls.force_idr.store(true, Ordering::Relaxed); - } else { - cap.pending_force_idr = true; - } - cap.needs_full_render = true; + cap.request_idr(); } } } @@ -4761,6 +4748,14 @@ fn want_wayland(settings: &Bound<'_, PyAny>) -> bool { struct ScState { /// 0 = idle, 1 = X11, 2 = Wayland. backend: u8, + /// This capture holds one reference on the shared X11 cursor monitor. Set in + /// the same locked section as `backend = 1` and TAKEN in the same locked + /// section a stop reads the backend, so acquire/release pair exactly per + /// capture — inferring the reference from `backend` alone would let a stop + /// that interleaves with a start release a reference not yet taken (leaking + /// the monitor once the acquire lands). The GIL happens to serialize that + /// window today; the pairing must not depend on it. + cursor_ref: bool, controls: Option>, handle: Option>, cap_thread_id: Option, @@ -4805,7 +4800,7 @@ impl ScreenCapture { /// GIL across the joins would deadlock. A re-entrant stop arriving on the capture, encode, or /// deliver thread cannot join itself, so it detaches and lets the threads exit on the stop flag. fn stop_internal(&self, py: Python<'_>) -> PyResult<()> { - let (handle, deliver_handle, same_thread, backend, controls, wl_display) = { + let (handle, deliver_handle, same_thread, backend, controls, wl_display, cursor_ref) = { let mut st = self.inner.lock().unwrap(); if let Some(c) = &st.controls { c.stop.store(true, Ordering::Relaxed); @@ -4825,6 +4820,7 @@ impl ScreenCapture { let handle = st.handle.take(); let deliver_handle = st.deliver_handle.take(); let backend = st.backend; + let cursor_ref = std::mem::take(&mut st.cursor_ref); let wl_display = st.wl_display; st.backend = 0; st.cap_thread_id = None; @@ -4832,12 +4828,12 @@ impl ScreenCapture { st.encode_tid_rx = None; st.deliver_thread_id = None; st.wl_display = 0; - (handle, deliver_handle, same, backend, controls, wl_display) + (handle, deliver_handle, same, backend, controls, wl_display, cursor_ref) }; if let Some(c) = &controls { live_x11().lock().unwrap().retain(|x| !Arc::ptr_eq(x, c)); } - if backend == 1 { + if cursor_ref { crate::x11::cursor::release(py); } if backend == 2 { @@ -4889,6 +4885,7 @@ impl ScreenCapture { id: NEXT_CAPTURE_ID.fetch_add(1, Ordering::Relaxed), inner: Mutex::new(ScState { backend: 0, + cursor_ref: false, controls: None, handle: None, cap_thread_id: None, @@ -5091,8 +5088,12 @@ impl ScreenCapture { None } }; + // Take the monitor reference BEFORE publishing the capture: a stop that + // observes this capture must always find the reference it is to release. + crate::x11::cursor::acquire(cursor_cap); let mut st = self.inner.lock().unwrap(); st.backend = 1; + st.cursor_ref = true; st.controls = Some(controls); st.handle = Some(handle); st.cap_thread_id = tid; @@ -5101,7 +5102,6 @@ impl ScreenCapture { st.deliver_handle = Some(deliver_handle); st.deliver_thread_id = Some(deliver_thread_id); drop(st); - crate::x11::cursor::acquire(cursor_cap); Ok(()) } diff --git a/pixelflux/src/recorder/mod.rs b/pixelflux/src/recorder/mod.rs index 893657e..8b22581 100644 --- a/pixelflux/src/recorder/mod.rs +++ b/pixelflux/src/recorder/mod.rs @@ -276,7 +276,10 @@ fn offer_frame(shared: &RecShared, tx: &Sender, stripes: &[EncodedStri if s.data.is_empty() { return; } - let offset = if s.data.len() > 10 && s.data[0] == 0x04 { 10 } else { 0 }; + let offset = if s.data.len() >= 10 && s.data[0] == 0x04 { 10 } else { 0 }; + if s.data.len() == offset { + return; + } let pts_us = shared.start.elapsed().as_micros() as u64; match tx.try_send((s.data.clone(), offset, pts_us)) { Ok(()) => { diff --git a/pixelflux/src/recording_sink.rs b/pixelflux/src/recording_sink.rs index 26579a0..ac96c25 100644 --- a/pixelflux/src/recording_sink.rs +++ b/pixelflux/src/recording_sink.rs @@ -59,6 +59,8 @@ pub struct RecordingSink { /// /// [`should_force_idr`]: RecordingSink::should_force_idr client_connected: Arc, + /// One-time notice that the session's H.264 frames are striped and unrecordable. + warned_unrecordable: AtomicBool, } impl RecordingSink { @@ -147,6 +149,7 @@ impl RecordingSink { clients, shutdown, client_connected, + warned_unrecordable: AtomicBool::new(false), }) } @@ -156,21 +159,41 @@ impl RecordingSink { self.client_connected.swap(false, Ordering::Relaxed) } - /// Delivery-layer tap for one encoded stripe. Only H.264 frames (`data_type == 2`) are - /// forwarded; the 10-byte wire header (`0x04` tag) is skipped via the queued offset so the - /// output is a plain Annex-B elementary stream. + /// Delivery-layer tap for one encoded frame. The socket carries a single H.264 + /// elementary stream, so only a lone full-height stripe (`data_type == 2`) is + /// recordable: striped CPU encodes are N independent per-stripe streams, and + /// interleaving them would produce an undecodable file — those are skipped with a + /// one-time notice (live streaming is unaffected). The 10-byte wire header + /// (`0x04` tag) is skipped via the queued offset so consumers receive plain + /// Annex-B. /// - /// Never blocks and never copies: the `Arc` payload is cloned into each client's bounded - /// queue with `try_send`, and a client whose queue is full or whose writer died is dropped. - pub fn write_encoded_frame(&self, stripe: &EncodedStripe) { - if stripe.data.is_empty() || stripe.data_type != 2 { + /// Never blocks and never copies: the `Arc` payload is cloned into each client's + /// bounded queue with `try_send`, and a client whose queue is full or whose + /// writer died is dropped. + pub fn write_frame(&self, stripes: &[EncodedStripe], full_height: i32) { + let mut h264 = stripes + .iter() + .filter(|s| s.data_type == 2 && !s.data.is_empty()); + let Some(stripe) = h264.next() else { return }; + if h264.next().is_some() || stripe.stripe_y_start != 0 || stripe.stripe_height != full_height + { + if !self.warned_unrecordable.swap(true, Ordering::Relaxed) { + eprintln!( + "[recording_sink] WARNING: striped H.264 frames are not recordable \ + (the socket carries one elementary stream); use a full-frame encoder \ + to record this session" + ); + } return; } - let offset = if stripe.data.len() > 10 && stripe.data[0] == 0x04 { + let offset = if stripe.data.len() >= 10 && stripe.data[0] == 0x04 { 10 } else { 0 }; + if stripe.data.len() == offset { + return; + } let mut clients = self.clients.lock().unwrap(); if clients.is_empty() { @@ -242,7 +265,7 @@ mod cost_tests { let mut total_us = 0f64; for _ in 0..n { let t = Instant::now(); - sink.write_encoded_frame(&f); + sink.write_frame(std::slice::from_ref(&f), 720); let us = t.elapsed().as_secs_f64() * 1e6; total_us += us; max_us = max_us.max(us); diff --git a/pixelflux/src/wayland/frontend.rs b/pixelflux/src/wayland/frontend.rs index b56fff8..0e7a447 100644 --- a/pixelflux/src/wayland/frontend.rs +++ b/pixelflux/src/wayland/frontend.rs @@ -228,6 +228,22 @@ pub struct WlCapture { pub last_tick: Option, } +impl WlCapture { + /// Arm a keyframe on whichever path is live, plus a full render so a static + /// screen still produces the frame. Exactly ONE flag is set: the pool encode + /// loop consumes the atomic, every other path consumes `pending_force_idr` — + /// setting both would leave one armed forever (the host-mode idle gate polls + /// them every tick, so a stuck flag disables idle skipping for the session). + pub fn request_idr(&mut self) { + if self.encode_pool.is_some() { + self.encode_controls.force_idr.store(true, Ordering::Relaxed); + } else { + self.pending_force_idr = true; + } + self.needs_full_render = true; + } +} + /// One virtual output and everything sized to it: the Smithay `Output` + its advertised /// global, its layout position, the damage tracker, render targets, and the (at most one) /// capture bound to it. `id` is the Python-facing display key; id 0 is the primary diff --git a/pixelflux/src/x11/cursor.rs b/pixelflux/src/x11/cursor.rs index 394cbe0..970f01a 100644 --- a/pixelflux/src/x11/cursor.rs +++ b/pixelflux/src/x11/cursor.rs @@ -210,9 +210,18 @@ fn monitor_thread(stop: Arc, wake_win: Arc) { } let mut last: Option = fetch_payload(&conn); deliver(last.as_ref()); - // A registration that raced setup may have requested a replay before the wake window - // existed; the current cursor was just delivered, so the request is satisfied. - REPLAY.store(false, Ordering::Release); + // A registration that raced setup may have requested a replay before the wake + // window existed (its wake was skipped). The deliver above satisfied it only if + // the callback was already visible and the fetch produced a payload, so consume + // the flag and re-deliver rather than assume: an unconditional clear leaves that + // callback cursor-less until the next real cursor change. From here on the wake + // window exists, so later requests always reach the loop below. + if REPLAY.swap(false, Ordering::AcqRel) { + if last.is_none() { + last = fetch_payload(&conn); + } + deliver(last.as_ref()); + } loop { let event = match conn.wait_for_event() { Ok(ev) => ev, @@ -315,7 +324,9 @@ fn deliver(payload: Option<&Payload>) { }; if let Some(cb) = cb { let py_bytes = PyBytes::new(py, png); - let _ = cb.call1(py, (*msg_type, py_bytes, *hot_x, *hot_y)); + if let Err(e) = cb.call1(py, (*msg_type, py_bytes, *hot_x, *hot_y)) { + e.print(py); + } } }); } @@ -369,6 +380,12 @@ fn cursor_to_png(img: &GetCursorImageReply, cap: i32) -> (&'static str, Vec, hot_y = (hot_y as f32 * scale) as i32; } crate::unpremultiply_rgba(&mut image); + // A hotspot can lie outside the visible bbox (its neighborhood was cropped as + // fully transparent). Consumers treat the hotspot as an offset INTO the + // bitmap, so clamp to the cropped bounds instead of emitting off-image + // coordinates. + let hot_x = hot_x.clamp(0, image.width() as i32 - 1); + let hot_y = hot_y.clamp(0, image.height() as i32 - 1); let mut png = Vec::new(); match image.write_to(&mut std::io::Cursor::new(&mut png), image::ImageFormat::Png) { Ok(()) => ("png", png, hot_x, hot_y), @@ -406,16 +423,35 @@ mod tests { assert_eq!(t, "hide"); } - /// The image is cropped to its visible bbox and the hotspot re-based to the crop: one - /// opaque pixel at (2,1) with hotspot (3,3) yields a 1x1 PNG with hotspot (1,2). + /// The image is cropped to its visible bbox and the hotspot re-based to the crop: + /// a 2x2 visible block at (1,1)..(2,2) with hotspot (2,2) yields a 2x2 PNG with + /// hotspot (1,1). #[test] fn crop_rebases_hotspot() { let mut px = vec![0u32; 16]; - px[1 * 4 + 2] = 0xFF00_0000; - let (t, data, hx, hy) = cursor_to_png(&reply(4, 4, 3, 3, px), 32); + for (x, y) in [(1, 1), (2, 1), (1, 2), (2, 2)] { + px[y * 4 + x] = 0xFF00_0000; + } + let (t, data, hx, hy) = cursor_to_png(&reply(4, 4, 2, 2, px), 32); assert_eq!(t, "png"); assert!(!data.is_empty()); - assert_eq!((hx, hy), (1, 2)); + assert_eq!((hx, hy), (1, 1)); + } + + /// A hotspot whose neighborhood was cropped away as transparent is clamped into + /// the emitted bitmap — consumers use it as an offset INTO the image, and the + /// Wayland path never emits out-of-bounds hotspots either. + #[test] + fn out_of_bbox_hotspot_clamped() { + let mut px = vec![0u32; 16]; + px[1 * 4 + 2] = 0xFF00_0000; + // Visible pixel at (2,1) only. Hotspot (0,0): rebased (-2,-1) -> (0,0). + let (t, _, hx, hy) = cursor_to_png(&reply(4, 4, 0, 0, px.clone()), 32); + assert_eq!(t, "png"); + assert_eq!((hx, hy), (0, 0)); + // Hotspot (3,3): rebased (1,2) past the 1x1 crop -> clamps to (0,0). + let (_, _, hx, hy) = cursor_to_png(&reply(4, 4, 3, 3, px), 32); + assert_eq!((hx, hy), (0, 0)); } /// Premultiplied color becomes straight alpha in the PNG: a half-alpha pixel stored diff --git a/pixelflux/src/x11/mod.rs b/pixelflux/src/x11/mod.rs index 7fa73e2..4e6a49b 100644 --- a/pixelflux/src/x11/mod.rs +++ b/pixelflux/src/x11/mod.rs @@ -617,9 +617,7 @@ where stripe_count += stripes.len() as u64; // IDR arming is consumed inside X11Pipeline::process; this tap only fans out. if let Some(ref socket) = recording_sink { - for stripe in &stripes { - socket.write_encoded_frame(stripe); - } + socket.write_frame(&stripes, psettings.height); } on_frame(stripes); } From 64daccd33ed58f872c0b1bdeaddea25613528d37 Mon Sep 17 00:00:00 2001 From: Seungmin Kim <8457324+ehfd@users.noreply.github.com> Date: Wed, 22 Jul 2026 00:50:50 +0900 Subject: [PATCH 07/21] fix(wayland): Cursor fix --- pixelflux/src/lib.rs | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/pixelflux/src/lib.rs b/pixelflux/src/lib.rs index 676c758..7c5c699 100644 --- a/pixelflux/src/lib.rs +++ b/pixelflux/src/lib.rs @@ -3526,6 +3526,12 @@ fn run_wayland_thread( state.cursor_callback_set = true; if let Some(icon) = state.current_cursor_icon.clone() { state.send_cursor_image(&icon); + } else { + // No client has set a cursor yet. The render path treats + // None as the default theme cursor, so the first consumer + // must receive that same sprite instead of a blank pointer + // until the first client cursor event. + state.send_cursor_image(&CursorImageStatus::Named(Default::default())); } } ThreadCommand::KeyboardKey { scancode, state: key_state_val } => { From 44d9efee7b42f4dc3b1843b62e24215c18de7af3 Mon Sep 17 00:00:00 2001 From: Seungmin Kim <8457324+ehfd@users.noreply.github.com> Date: Sat, 25 Jul 2026 16:25:40 +0900 Subject: [PATCH 08/21] fix(example): WebSocket accumulation --- example/screen_to_browser.py | 405 +++++++++++++++++++++-------------- 1 file changed, 240 insertions(+), 165 deletions(-) diff --git a/example/screen_to_browser.py b/example/screen_to_browser.py index ff27afa..ba7ced7 100644 --- a/example/screen_to_browser.py +++ b/example/screen_to_browser.py @@ -9,111 +9,44 @@ This script demonstrates the pixelflux library's instance-safe capabilities. It can handle multiple WebSocket clients, each with its own independent screen capture session. The capture region can be controlled via the URL hash. + +The capture callback never blocks and never buffers more than the +bounded per-client queue. When a client cannot drain the stream, the backlog +is dropped and H.264 delivery gates per stripe row until the next keyframe +(requested from the encoder at most once per second), so a slow consumer +costs bounded memory and resumes on a decodable frame. """ # Standard library imports import asyncio import os import mimetypes +import time +import urllib.parse import websockets import websockets.asyncio.server as ws_async -import threading # Third-party library imports from pixelflux import CaptureSettings, ScreenCapture, ensure_wayland_display # ============================================================================== # --- BASE CONFIGURATION SETTINGS --- -# These settings will be used as a template for each new connection. -# Modify the parameters below to test different capture and encoding options. +# These settings are applied to a fresh CaptureSettings for each new +# connection. Modify the parameters below to test different capture and +# encoding options. # ============================================================================== HTTP_PORT = 9001 WS_PORT = 9000 -# Create a default template for capture settings. -base_capture_settings = CaptureSettings() - -# --- Debugging --- -# Enable/disable the continuous FPS and settings log printed to the console. -base_capture_settings.debug_logging = True - -# --- Core Capture --- -base_capture_settings.capture_width = 1920 -base_capture_settings.capture_height = 1080 -base_capture_settings.capture_x = 0 # This can be overridden by the URL -base_capture_settings.capture_y = 0 -base_capture_settings.target_fps = 60.0 -base_capture_settings.capture_cursor = False - -# --- Encoding Mode --- -# Sets the output codec. 0 for JPEG, 1 for H.264. -base_capture_settings.output_mode = 1 -# Force CPU encoding and ignore hardware encoders -base_capture_settings.use_cpu = False - -# --- Zero-copy frames --- -# The native callback delivers a StripeFrame (buffer-protocol object owning the C -# buffer). Every mode now emits its wire header natively (0x03 JPEG / 0x04 H.264), -# so memoryview(frame) is sent with no copy or re-frame on any mode. The buffer -# protocol pins the buffer alive across every Python version (no PEP 688). -ZERO_COPY = True - -# --- H.264 Quality Settings --- -# Constant Rate Factor (0-51, lower is better quality & higher bitrate). -# Good values are typically 18-28. -base_capture_settings.video_crf = 25 -# CRF for H.264 paintover on static content. Used if lower (better) than video_crf. -base_capture_settings.video_paintover_crf = 18 -# Number of high-quality H.264 frames to send in a burst when a paintover is triggered. -base_capture_settings.video_paintover_burst_frames = 5 -# Use I444 (full color) instead of I420. Better quality, higher CPU/bandwidth. -base_capture_settings.video_fullcolor = False -# Encode full frames instead of just changed stripes. -base_capture_settings.video_fullframe = False -# Flag the stream to be in streaming mode to bypass all vnc logic -base_capture_settings.video_streaming_mode = False -# Encoder device index: -2 = auto-detect (lets AUTO_GPU pick), -1 = software encoding, 0+ = specific GPU. -base_capture_settings.encode_node_index = -2 -# Switches to CBR mode and ignores CRF value. Used in conjunction with video_bitrate_kbps. -base_capture_settings.video_cbr_mode = False -# Target bitrate in kbps for CBR mode. Required when video_cbr_mode is enabled. -base_capture_settings.video_bitrate_kbps = 4000 -# Optional VBV buffer size in kilobits for custom buffer size. -base_capture_settings.video_vbv_multiplier = 1.5 # VBV as a multiple of one frame's bit budget (0 = auto policy). -# Allow pixelflux to adjust its capture width and height. Overrides provided width and height when enabled. -base_capture_settings.auto_adjust_screen_capture_size = True -# - -# --- Change Detection & Optimization --- -# Use a higher quality setting for static regions that haven't changed for a while. -base_capture_settings.use_paint_over_quality = True -# Number of frames of no motion in a stripe to trigger a high-quality "paint-over". -base_capture_settings.paint_over_trigger_frames = 15 -# Consecutive changes to a stripe to trigger a "damaged" state (uses base quality). -base_capture_settings.damage_block_threshold = 10 -# Number of frames a stripe stays "damaged" after being triggered. -base_capture_settings.damage_block_duration = 30 - -# --- JPEG Quality Settings --- -# Quality of jpegs under motion -base_capture_settings.jpeg_quality = 40 -# Quality of jpegs on static content paintovers -base_capture_settings.paint_over_jpeg_quality = 90 - -# --- Watermarking --- -# The path MUST be a byte string (b"") and point to a valid PNG file. -#base_capture_settings.watermark_path = b"/path/to/image.png" -# Sets the watermark location on the screen. Default is 0 (disabled). -# Options: 0:None, 1:TopLeft, 2:TopRight, 3:BottomLeft, 4:BottomRight, 5:Middle, 6:Animated -base_capture_settings.watermark_location_enum = 0 - -# --- Recording --- -# When this is set to a valid path (string) will enable a unix socket for recording -# i.e. '/tmp/test' can be recorded with "ffmpeg -f h264 -i unix:///tmp/test -c:v copy test.h264" -# For a clean recording the stream might need a re-encode i.e.: -# "ffmpeg -f h264 -framerate 60 -i unix:///tmp/test -c:v libx264 -preset fast -crf 23 -pix_fmt yuv420p test.mp4" -# This option enables IDR frames every 30 frames and on socket connection -base_capture_settings.recording_socket = None +# Send-side bounds, mirroring selkies' per-client video relay: the queue holds +# at most ~2 seconds of stream at 60 fps; keyframe requests triggered by +# overflow are rate-limited per client. +VIDEO_QUEUE_MAXSIZE = 120 +IDR_REQUEST_FLOOR_SECONDS = 1.0 +# A send that makes no progress for this long marks the client abandoned; the +# connection is closed rather than letting a dead socket linger (drops are +# already handled upstream by the bounded queue). +SEND_TIMEOUT_SECONDS = 1.0 # ============================================================================== # --- ENVIRONMENT OVERRIDES (the same variables selkies accepts) --- @@ -128,41 +61,128 @@ def _env(name, legacy=None): value = os.environ.get(legacy) return value or "" -# Backend: force Wayland/X11; left unset, pixelflux follows WAYLAND_DISPLAY. _wayland = _env("SELKIES_WAYLAND", "PIXELFLUX_WAYLAND").lower() -if _wayland: - base_capture_settings.use_wayland = _wayland in ("1", "true", "yes", "on") -# Compositor render node: an explicit path wins over auto-GPU selection. _render_dri = _env("SELKIES_RENDER_DRI", "DRINODE") -if _render_dri: - base_capture_settings.render_node_path = _render_dri.encode("utf-8") -# "true" (the default) = first GPU; any other token = first GPU matching a vendor -# name, kernel driver name, devicetree prefix, or raw PCI vendor id; "false" disables. -base_capture_settings.auto_gpu = _env("SELKIES_AUTO_GPU", "AUTO_GPU") or "true" -# Encoder node (VA-API/NVENC device), distinct from the render node. _encode_dri = _env("SELKIES_ENCODE_DRI", "DRI_NODE") -if _encode_dri: - base_capture_settings.encode_node_path = _encode_dri.encode("utf-8") - _idx = _encode_dri.rsplit("renderD", 1)[-1] - if _idx.isdigit(): - base_capture_settings.encode_node_index = int(_idx) - 128 _recording = _env("SELKIES_RECORDING_SOCKET", "PIXELFLUX_RECORDING_SOCKET") -if _recording: - base_capture_settings.recording_socket = _recording -# Compositor cursor-theme size in pixels (Wayland backend). _cursor_size = _env("SELKIES_CURSOR_SIZE", "XCURSOR_SIZE") -if _cursor_size.isdigit(): - base_capture_settings.cursor_size = int(_cursor_size) +_auto_gpu = _env("SELKIES_AUTO_GPU", "AUTO_GPU") or "true" + +def build_capture_settings(): + """Return a fresh CaptureSettings for one client. + + Each connection gets its own object: sessions started concurrently must + not see another client's overrides (e.g. the per-URL capture_x below). + """ + cs = CaptureSettings() + + # --- Debugging --- + # Enable/disable the continuous FPS and settings log printed to the console. + cs.debug_logging = True + + # --- Core Capture --- + cs.capture_width = 1920 + cs.capture_height = 1080 + cs.capture_x = 0 # This can be overridden by the URL + cs.capture_y = 0 + cs.target_fps = 60.0 + cs.capture_cursor = False + + # --- Encoding Mode --- + # Sets the output codec. 0 for JPEG, 1 for H.264. + cs.output_mode = 1 + # Force CPU encoding and ignore hardware encoders + cs.use_cpu = False + + # --- H.264 Quality Settings --- + # Constant Rate Factor (0-51, lower is better quality & higher bitrate). + # Good values are typically 18-28. + cs.video_crf = 25 + # CRF for H.264 paintover on static content. Used if lower (better) than video_crf. + cs.video_paintover_crf = 18 + # Number of high-quality H.264 frames to send in a burst when a paintover is triggered. + cs.video_paintover_burst_frames = 5 + # Use I444 (full color) instead of I420. Better quality, higher CPU/bandwidth. + cs.video_fullcolor = False + # Encode full frames instead of just changed stripes. + cs.video_fullframe = False + # Flag the stream to be in streaming mode to bypass all vnc logic + cs.video_streaming_mode = False + # Encoder device index: -2 = auto-detect (lets AUTO_GPU pick), -1 = software encoding, 0+ = specific GPU. + cs.encode_node_index = -2 + # Switches to CBR mode and ignores CRF value. Used in conjunction with video_bitrate_kbps. + cs.video_cbr_mode = False + # Target bitrate in kbps for CBR mode. Required when video_cbr_mode is enabled. + cs.video_bitrate_kbps = 4000 + # Optional VBV buffer size in kilobits for custom buffer size. + cs.video_vbv_multiplier = 1.5 # VBV as a multiple of one frame's bit budget (0 = auto policy). + # Allow pixelflux to adjust its capture width and height. Overrides provided width and height when enabled. + cs.auto_adjust_screen_capture_size = True + + # --- Change Detection & Optimization --- + # Use a higher quality setting for static regions that haven't changed for a while. + cs.use_paint_over_quality = True + # Number of frames of no motion in a stripe to trigger a high-quality "paint-over". + cs.paint_over_trigger_frames = 15 + # Consecutive changes to a stripe to trigger a "damaged" state (uses base quality). + cs.damage_block_threshold = 10 + # Number of frames a stripe stays "damaged" after being triggered. + cs.damage_block_duration = 30 + + # --- JPEG Quality Settings --- + # Quality of jpegs under motion + cs.jpeg_quality = 40 + # Quality of jpegs on static content paintovers + cs.paint_over_jpeg_quality = 90 + + # --- Watermarking --- + # The path MUST be a byte string (b"") and point to a valid PNG file. + #cs.watermark_path = b"/path/to/image.png" + # Sets the watermark location on the screen. Default is 0 (disabled). + # Options: 0:None, 1:TopLeft, 2:TopRight, 3:BottomLeft, 4:BottomRight, 5:Middle, 6:Animated + cs.watermark_location_enum = 0 + + # --- Recording --- + # When this is set to a valid path (string) will enable a unix socket for recording + # i.e. '/tmp/test' can be recorded with "ffmpeg -f h264 -i unix:///tmp/test -c:v copy test.h264" + # For a clean recording the stream might need a re-encode i.e.: + # "ffmpeg -f h264 -framerate 60 -i unix:///tmp/test -c:v libx264 -preset fast -crf 23 -pix_fmt yuv420p test.mp4" + # This option enables IDR frames every 30 frames and on socket connection + cs.recording_socket = None + + # --- Environment overrides --- + # Backend: force Wayland/X11; left unset, pixelflux follows WAYLAND_DISPLAY. + if _wayland: + cs.use_wayland = _wayland in ("1", "true", "yes", "on") + # Compositor render node: an explicit path wins over auto-GPU selection. + if _render_dri: + cs.render_node_path = _render_dri.encode("utf-8") + # "true" (the default) = first GPU; any other token = first GPU matching a vendor + # name, kernel driver name, devicetree prefix, or raw PCI vendor id; "false" disables. + cs.auto_gpu = _auto_gpu + # Encoder node (VA-API/NVENC device), distinct from the render node. + if _encode_dri: + cs.encode_node_path = _encode_dri.encode("utf-8") + _idx = _encode_dri.rsplit("renderD", 1)[-1] + if _idx.isdigit(): + cs.encode_node_index = int(_idx) - 128 + if _recording: + cs.recording_socket = _recording + # Compositor cursor-theme size in pixels (Wayland backend). + if _cursor_size.isdigit(): + cs.cursor_size = int(_cursor_size) + + return cs # Wayland: bring the compositor socket up now, before the capture starts, so apps # launched alongside this script can already connect to WAYLAND_DISPLAY. -if getattr(base_capture_settings, "use_wayland", False): +if _wayland in ("1", "true", "yes", "on"): _dim = lambda name: int(os.environ.get(name) or 0) if str(os.environ.get(name) or "").isdigit() else 0 ensure_wayland_display( width=_dim("SELKIES_MANUAL_WIDTH"), height=_dim("SELKIES_MANUAL_HEIGHT"), render_node=_render_dri, - auto_gpu=base_capture_settings.auto_gpu, + auto_gpu=_auto_gpu, cursor_size=int(_cursor_size) if _cursor_size.isdigit() else -1, ) @@ -175,7 +195,6 @@ def _env(name, legacy=None): # The key is the WebSocket connection object. # The value is another dictionary containing the client's capture module, queue, and task. ACTIVE_CLIENTS = {} -CLIENT_LOCK = threading.Lock() # Lock for thread-safe modifications to ACTIVE_CLIENTS. async def send_stripes_task(websocket, queue): """ @@ -184,22 +203,29 @@ async def send_stripes_task(websocket, queue): """ print(f"Send task started for client {websocket.remote_address}.") try: - # This loop will run until the connection is closed, - # which will raise a ConnectionClosed exception. while True: item = await queue.get() try: - # item == {'data': , 'owner': }. Keeping - # `item` (hence the StripeFrame) referenced for the whole send keeps the C - # buffer alive until the send releases the view, so zero-copy is safe. - await websocket.send(item['data']) + # item == {'data': , 'owner': }. Keeping + # `item` (hence the StripeFrame) referenced for the whole send keeps + # the C buffer alive until the send releases the view, so zero-copy + # is safe. The timeout is liveness-only: overflow is handled by the + # bounded queue upstream, so a send stuck this long means the client + # is gone, not merely slow. + await asyncio.wait_for(websocket.send(item['data']), + SEND_TIMEOUT_SECONDS) + except asyncio.TimeoutError: + print(f"Client {websocket.remote_address} abandoned (send " + f"stalled > {SEND_TIMEOUT_SECONDS}s); closing.") + await websocket.close(code=1001, reason='Send timeout') + break finally: queue.task_done() except websockets.exceptions.ConnectionClosed: # This is the expected, clean way to exit the loop when a client disconnects. print(f"Connection closed for {websocket.remote_address}. Send task stopping.") - + except asyncio.CancelledError: # This happens when the main handler cancels us during cleanup. print(f"Send task was cancelled for {websocket.remote_address}.") @@ -207,7 +233,7 @@ async def send_stripes_task(websocket, queue): except Exception as e: # Catch any other unexpected errors. print(f"[ERROR] Send task for client {websocket.remote_address} failed unexpectedly: {e}") - + finally: print(f"Send task for {websocket.remote_address} has finished.") @@ -221,45 +247,87 @@ async def websocket_handler(websocket): client_module = None send_task = None - # Keep a reference to the callback to prevent it from being garbage collected - on_stripe = None try: # --- 1. Configure Capture for this Specific Client --- - client_settings = base_capture_settings + client_settings = build_capture_settings() try: x_offset = int(path.strip('/')) client_settings.capture_x = x_offset print(f"Client {client_id} requested custom capture at x={x_offset}.") except (ValueError, TypeError): print(f"Client {client_id} using default capture at x=0.") - client_settings.capture_x = 0 # --- 2. Create Resources for this Client --- client_module = ScreenCapture() - client_queue = asyncio.Queue(maxsize=120) + client_queue = asyncio.Queue(maxsize=VIDEO_QUEUE_MAXSIZE) + # live_rows: stripe rows (wire y_start) whose H.264 chain is intact + # for this client. A row's deltas are deliverable only after its IDR; + # an overflow drop clears the set, gating every row until a keyframe + # re-anchors it. One frame can mix IDR and delta stripes (per-stripe + # encoder re-init), so gating is per row, not per frame — the same + # rule as selkies' video relay. Full-frame encoders always emit row 0. + relay_state = {'live_rows': set(), 'last_idr_req': 0.0} # --- 3. Create a unique callback (closure) for this client --- - # This function "closes over" client_queue and g_loop, giving it access - # without needing global lookups or user_data. + # Runs on the native capture thread: it must never block, so it hands + # the frame to the loop and the loop-side enqueue applies the bounded + # relay rules. The memoryview aliases the C buffer and pins the + # StripeFrame alive until the send (or a drop) releases the item. def on_stripe(frame): """Callback invoked by pixelflux with a StripeFrame per video stripe.""" if not (len(frame) > 0 and g_loop and not g_loop.is_closed()): return - if ZERO_COPY: - # Zero-copy: memoryview aliases the C buffer and pins the StripeFrame - # alive. The queued item holds both, so the frame (hence its buffer) - # outlives the send until the view is released. - item_to_queue = {'data': memoryview(frame), 'owner': frame} - else: - # Copy fallback: the frame already carries its native wire header. - item_to_queue = {'data': bytes(frame), 'owner': None} + item_to_queue = {'data': memoryview(frame), 'owner': frame} + + def request_idr_throttled(): + now = time.monotonic() + if now - relay_state['last_idr_req'] >= IDR_REQUEST_FLOOR_SECONDS: + relay_state['last_idr_req'] = now + client_module.request_idr_frame() + + def enqueue(): + data = item_to_queue['data'] + # Wire prefixes: 0x04 = H.264 (byte 1 = encoded picture type, + # 0x01 = IDR; bytes 4:6 = stripe y_start big-endian), 0x03 = + # JPEG. JPEG is never gated: every frame is independently + # decodable, a dropped one is simply repainted by the next. + is_h264 = len(data) >= 10 and data[0] == 0x04 + is_idr = is_h264 and data[1] == 0x01 + row = ((data[4] << 8) | data[5]) if is_h264 else 0 + if is_h264 and not is_idr and row not in relay_state['live_rows']: + # Undecodable until this row re-anchors; ask for a + # keyframe in case none is already on the way. + request_idr_throttled() + return + try: + client_queue.put_nowait(item_to_queue) + if is_idr: + relay_state['live_rows'].add(row) + except asyncio.QueueFull: + # The socket is slower than the stream: drop the whole + # backlog rather than buffer it, then recover via a + # keyframe. Dropping the item references releases the + # underlying StripeFrame buffers. + while not client_queue.empty(): + try: + client_queue.get_nowait() + client_queue.task_done() + except asyncio.QueueEmpty: + break + relay_state['live_rows'].clear() + if is_h264 and not is_idr: + request_idr_throttled() + else: + # Keyframes and JPEG frames go into the just-cleared + # queue so the client resumes immediately. + client_queue.put_nowait(item_to_queue) + if is_idr: + relay_state['live_rows'].add(row) try: - asyncio.run_coroutine_threadsafe( - client_queue.put(item_to_queue), g_loop - ) + g_loop.call_soon_threadsafe(enqueue) except RuntimeError: # Loop closed between the check and the call (teardown race); # dropping the item's references recycles the frame buffer. @@ -292,12 +360,12 @@ def on_stripe(frame): finally: # --- 7. Clean Up Resources for this Specific Client --- print(f"Cleaning up resources for client {client_id}...") - + if send_task and not send_task.done(): send_task.cancel() try: await send_task except asyncio.CancelledError: pass - + if client_module: loop = asyncio.get_running_loop() await loop.run_in_executor(None, client_module.stop_capture) @@ -305,6 +373,20 @@ def on_stripe(frame): ACTIVE_CLIENTS.pop(websocket, None) print(f"Cleanup complete for client {client_id}. Active clients: {len(ACTIVE_CLIENTS)}") +def _resolve_static_path(script_dir, request_path): + """Return the real path of request_path under script_dir, or None if it + escapes. realpath canonicalizes '..'/symlinks and the root+os.sep boundary + rejects sibling-prefix dirs; the path is URL-decoded first.""" + decoded = urllib.parse.unquote(request_path) + root = os.path.realpath(script_dir) + try: + requested = os.path.realpath(os.path.join(root, decoded.lstrip('/'))) + except ValueError: + return None # e.g. embedded NUL byte ("%00") + if requested != root and not requested.startswith(root + os.sep): + return None + return requested + async def handle_http_request(reader, writer): """Handle HTTP requests by serving static files from the script directory.""" try: @@ -314,7 +396,7 @@ async def handle_http_request(reader, writer): parts = request_line.split() if len(parts) < 2 or parts[0] != b'GET': - writer.write(b'HTTP/1.1 405 Method Not Allowed\r\n\r\n') + writer.write(b'HTTP/1.1 405 Method Not Allowed\r\nContent-Length: 0\r\nConnection: close\r\n\r\n') return path = parts[1].decode().split('#')[0] # Ignore hash part @@ -322,33 +404,34 @@ async def handle_http_request(reader, writer): path = '/index.html' script_dir = os.path.dirname(os.path.abspath(__file__)) - full_path = os.path.join(script_dir, path.lstrip('/')) - - # Security check to prevent directory traversal attacks - if not os.path.realpath(full_path).startswith(os.path.realpath(script_dir)): - writer.write(b'HTTP/1.1 403 Forbidden\r\n\r\n') + full_path = _resolve_static_path(script_dir, path) + + # Security check: reject directory traversal / escapes outside script_dir. + if full_path is None: + writer.write(b'HTTP/1.1 403 Forbidden\r\nContent-Length: 0\r\nConnection: close\r\n\r\n') return if os.path.isfile(full_path): with open(full_path, 'rb') as f: content = f.read() content_type = mimetypes.guess_type(full_path)[0] or 'application/octet-stream' - headers = f'HTTP/1.1 200 OK\r\nContent-Type: {content_type}\r\nContent-Length: {len(content)}\r\n\r\n' + headers = f'HTTP/1.1 200 OK\r\nContent-Type: {content_type}\r\nContent-Length: {len(content)}\r\nConnection: close\r\n\r\n' writer.write(headers.encode()) writer.write(content) else: - writer.write(b'HTTP/1.1 404 Not Found\r\n\r\n') + writer.write(b'HTTP/1.1 404 Not Found\r\nContent-Length: 0\r\nConnection: close\r\n\r\n') except Exception as e: print(f"[HTTP Error] {e}") + writer.write(b'HTTP/1.1 500 Internal Server Error\r\nContent-Length: 0\r\nConnection: close\r\n\r\n') finally: - if not writer.is_closing(): - try: - await writer.drain() - except ConnectionResetError: - pass # Client closed connection before we could finish. - finally: - writer.close() + # The client may already be gone; draining/closing a dead connection + # must not raise out of the handler. + try: + await writer.drain() + except (ConnectionError, OSError): + pass + writer.close() async def main(): @@ -369,26 +452,18 @@ async def main(): await asyncio.Event().wait() except OSError as e: print(f"[FATAL] Could not start server (is port {WS_PORT} in use?): {e}") - except KeyboardInterrupt: - print("\nShutdown signal received.") finally: print("Shutting down all client connections...") - # Create a list of cleanup tasks for all active clients - cleanup_tasks = [] - with CLIENT_LOCK: - clients_to_clean = list(ACTIVE_CLIENTS.keys()) - - for ws in clients_to_clean: - # Closing the websocket connection will trigger its handler's finally block - cleanup_tasks.append(ws.close(code=1001, reason='Server shutting down')) - + # Closing each websocket connection triggers its handler's finally block. + cleanup_tasks = [ws.close(code=1001, reason='Server shutting down') + for ws in list(ACTIVE_CLIENTS.keys())] if cleanup_tasks: await asyncio.gather(*cleanup_tasks, return_exceptions=True) if ws_server: ws_server.close() await ws_server.wait_closed() - + http_server.close() await http_server.wait_closed() print("All servers and connections closed. Goodbye.") From 6a84fa73b1d28cf61b1d5e8624ea82fe22d59eb4 Mon Sep 17 00:00:00 2001 From: Seungmin Kim <8457324+ehfd@users.noreply.github.com> Date: Sun, 26 Jul 2026 11:17:40 +0900 Subject: [PATCH 09/21] fix: License patterns --- MANIFEST.in | 1 + 1 file changed, 1 insertion(+) diff --git a/MANIFEST.in b/MANIFEST.in index a606bf7..0127132 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -5,6 +5,7 @@ recursive-include pixelflux/nvcodec-sys * recursive-exclude pixelflux *.so *.pyc prune pixelflux/target prune pixelflux/nvcodec-sys/target +include LICENSE include pyproject.toml include README.md include setup.py From 64acb7c8cf411e1b8d77ecedd17670ee7881b42b Mon Sep 17 00:00:00 2001 From: Seungmin Kim <8457324+ehfd@users.noreply.github.com> Date: Thu, 30 Jul 2026 00:16:45 +0900 Subject: [PATCH 10/21] fix: Performance optimizations --- pixelflux/src/lib.rs | 21 +++++++++++++++++++-- pixelflux/src/wayland/frontend.rs | 11 +++++++---- pixelflux/src/wayland/wlclient.rs | 26 +++++++++++++++++++++++--- 3 files changed, 49 insertions(+), 9 deletions(-) diff --git a/pixelflux/src/lib.rs b/pixelflux/src/lib.rs index 7c5c699..8780900 100644 --- a/pixelflux/src/lib.rs +++ b/pixelflux/src/lib.rs @@ -213,6 +213,24 @@ fn get_shm_usage_bytes() -> u64 { total_size } +/// Memory reads for the throttle watchdog, answered from a 100 ms cache: the +/// render timer re-arms at 1 ms while an encode pool is saturated, and a +/// per-tick /proc read plus a full /dev/shm directory walk would spin the +/// syscall count of an otherwise deliberate busy-wait. The watchdog is a +/// hysteresis and tolerates the staleness. +fn get_cached_mem_usage() -> (usize, u64) { + static CACHE: std::sync::Mutex> = std::sync::Mutex::new(None); + let mut guard = CACHE.lock().unwrap_or_else(|e| e.into_inner()); + if let Some((at, rss, shm)) = *guard { + if at.elapsed() < Duration::from_millis(100) { + return (rss, shm); + } + } + let sampled = (Instant::now(), get_process_rss_bytes(), get_shm_usage_bytes()); + *guard = Some(sampled); + (sampled.1, sampled.2) +} + fn calculate_memory_threshold(width: i32, height: i32) -> usize { let frame_size = (width.max(0) as usize) .saturating_mul(height.max(0) as usize) @@ -3912,8 +3930,7 @@ fn run_wayland_thread( let loop_start_time = Instant::now(); state.space.refresh(); - let current_rss = get_process_rss_bytes(); - let shm_usage = get_shm_usage_bytes(); + let (current_rss, shm_usage) = get_cached_mem_usage(); // The threshold follows the largest active capture (fallback: primary settings). let (max_w, max_h) = state .output_nodes diff --git a/pixelflux/src/wayland/frontend.rs b/pixelflux/src/wayland/frontend.rs index 0e7a447..954997b 100644 --- a/pixelflux/src/wayland/frontend.rs +++ b/pixelflux/src/wayland/frontend.rs @@ -1304,7 +1304,8 @@ impl SelectionHandler for AppState { } /// A client pastes the Python-owned selection: stream the stored bytes into the client's - /// fd on a spawned thread, since the receiving pipe may backpressure. + /// fd on a spawned thread, since the receiving pipe may backpressure. Bounded by an + /// idle deadline: a client that never reads its paste fd must not pin the thread. fn send_selection( &mut self, ty: SelectionTarget, @@ -1318,9 +1319,11 @@ impl SelectionHandler for AppState { } let payload = user_data.clone(); std::thread::spawn(move || { - use std::io::Write; - let mut f = std::fs::File::from(fd); - let _ = f.write_all(&payload.1); + let _ = crate::wayland::wlclient::write_fd_all( + &fd, + &payload.1, + crate::wayland::wlclient::IO_TIMEOUT, + ); }); } } diff --git a/pixelflux/src/wayland/wlclient.rs b/pixelflux/src/wayland/wlclient.rs index 7979c96..f56717a 100644 --- a/pixelflux/src/wayland/wlclient.rs +++ b/pixelflux/src/wayland/wlclient.rs @@ -218,10 +218,30 @@ pub(crate) fn read_fd_to_end(fd: &OwnedFd, idle: Duration) -> Result, St /// Write all of `data` to `fd`, tolerating a slow reader up to `idle` per chunk. /// EPIPE is success-shaped: the paster stopped reading, which is its right. +/// +/// The fd is switched to non-blocking for the transfer (original flags are +/// restored on every exit path): a single blocking write() of a large payload +/// would stay parked in the kernel once the peer stops draining mid-transfer, +/// and the poll deadline could never fire. With O_NONBLOCK, write() returns +/// EAGAIN instead and the idle deadline bounds the whole transfer. pub(crate) fn write_fd_all(fd: &OwnedFd, data: &[u8], idle: Duration) -> Result<(), String> { + let raw = fd.as_raw_fd(); + let old_flags = unsafe { libc::fcntl(raw, libc::F_GETFL) }; + if old_flags < 0 { + return Err(format!("fcntl F_GETFL: {}", std::io::Error::last_os_error())); + } + if unsafe { libc::fcntl(raw, libc::F_SETFL, old_flags | libc::O_NONBLOCK) } < 0 { + return Err(format!("fcntl F_SETFL: {}", std::io::Error::last_os_error())); + } + let result = write_fd_all_nb(raw, data, idle); + unsafe { libc::fcntl(raw, libc::F_SETFL, old_flags) }; + result +} + +fn write_fd_all_nb(raw: i32, data: &[u8], idle: Duration) -> Result<(), String> { let mut written = 0; while written < data.len() { - let mut pfd = libc::pollfd { fd: fd.as_raw_fd(), events: libc::POLLOUT, revents: 0 }; + let mut pfd = libc::pollfd { fd: raw, events: libc::POLLOUT, revents: 0 }; let n = unsafe { libc::poll(&mut pfd, 1, idle.as_millis().max(1) as libc::c_int) }; if n < 0 { let err = std::io::Error::last_os_error(); @@ -235,7 +255,7 @@ pub(crate) fn write_fd_all(fd: &OwnedFd, data: &[u8], idle: Duration) -> Result< } let w = unsafe { libc::write( - fd.as_raw_fd(), + raw, data[written..].as_ptr() as *const libc::c_void, data.len() - written, ) @@ -243,7 +263,7 @@ pub(crate) fn write_fd_all(fd: &OwnedFd, data: &[u8], idle: Duration) -> Result< if w < 0 { let err = std::io::Error::last_os_error(); match err.raw_os_error() { - Some(libc::EINTR) => continue, + Some(libc::EINTR) | Some(libc::EAGAIN) => continue, Some(libc::EPIPE) => return Ok(()), _ => return Err(format!("write: {err}")), } From b3c893ad542490f70ef2a9d3fcd1fe7982090380 Mon Sep 17 00:00:00 2001 From: Seungmin Kim <8457324+ehfd@users.noreply.github.com> Date: Fri, 31 Jul 2026 14:25:29 +0900 Subject: [PATCH 11/21] fix: Various issues --- pixelflux/src/computer_use.rs | 22 ++- pixelflux/src/encoders/nvenc.rs | 17 +-- pixelflux/src/encoders/oh264.rs | 4 +- pixelflux/src/encoders/overlay.rs | 17 +++ pixelflux/src/encoders/vaapi.rs | 16 ++- pixelflux/src/lib.rs | 222 +++++++++++++++++++++++------ pixelflux/src/wayland/dcclient.rs | 14 ++ pixelflux/src/wayland/frontend.rs | 5 +- pixelflux/src/wayland/wlclient.rs | 19 ++- pixelflux/src/x11/mod.rs | 223 ++++++++++++++++++++++++++---- 10 files changed, 466 insertions(+), 93 deletions(-) diff --git a/pixelflux/src/computer_use.rs b/pixelflux/src/computer_use.rs index 6755eca..e8930e6 100644 --- a/pixelflux/src/computer_use.rs +++ b/pixelflux/src/computer_use.rs @@ -21,6 +21,7 @@ use std::sync::{Mutex, OnceLock}; use std::thread; use std::time::Duration; use std::io::Cursor; +use std::io::Read; use smithay::input::keyboard::xkb; @@ -917,8 +918,15 @@ pub fn run_cu_server(addr: String) { let mut last_backend = ""; for mut request in server.incoming_requests() { + // A CU body is a single input command: cap its size so a hostile client + // of the (unauthenticated) endpoint cannot exhaust memory with a giant POST. + const MAX_CU_BODY: u64 = 4 * 1024 * 1024; let mut body = String::new(); - if let Err(e) = request.as_reader().read_to_string(&mut body) { + if let Err(e) = request + .as_reader() + .take(MAX_CU_BODY + 1) + .read_to_string(&mut body) + { let _ = request.respond(tiny_http::Response::from_string(format!( "{{\"error\":\"{}\"}}", e )) @@ -929,6 +937,18 @@ pub fn run_cu_server(addr: String) { continue; } + if body.len() as u64 > MAX_CU_BODY { + let resp = tiny_http::Response::from_string( + "{\"error\":\"request body too large\"}".to_string(), + ) + .with_status_code(413) + .with_header( + "Content-Type: application/json".parse::().unwrap() + ); + let _ = request.respond(resp); + continue; + } + if let Some(json) = handle_record_endpoint(request.url(), &body) { let _ = request.respond( tiny_http::Response::from_string(json) diff --git a/pixelflux/src/encoders/nvenc.rs b/pixelflux/src/encoders/nvenc.rs index a74c52a..1eca638 100644 --- a/pixelflux/src/encoders/nvenc.rs +++ b/pixelflux/src/encoders/nvenc.rs @@ -533,22 +533,7 @@ impl Drop for NvencEncoder { /// level is fixed from frame 1 (with the profile held at High) the stream never bumps its level /// mid-GOP — which would force some hardware decoders to re-initialize. Above 6.2's limits there is /// no higher level, so it returns 6.2 as a best effort. -fn min_h264_level(width: u32, height: u32, fps: u32) -> u32 { - let mbs = (width as u64).div_ceil(16) * (height as u64).div_ceil(16); - let mbps = mbs * fps.max(1) as u64; - const LEVELS: [(u32, u64, u64); 4] = [ - (52, 36864, 2073600), - (60, 139264, 4177920), - (61, 139264, 8355840), - (62, 139264, 16711680), - ]; - for &(level, max_fs, max_mbps) in &LEVELS { - if mbs <= max_fs && mbps <= max_mbps { - return level; - } - } - 62 -} +use super::min_h264_level; impl NvencEncoder { /// Resolve EGL at runtime rather than link against it, so one binary boots even on hosts diff --git a/pixelflux/src/encoders/oh264.rs b/pixelflux/src/encoders/oh264.rs index 4a59df2..a766d5e 100644 --- a/pixelflux/src/encoders/oh264.rs +++ b/pixelflux/src/encoders/oh264.rs @@ -806,6 +806,8 @@ mod rebuild_cost { let first_ms = t.elapsed().as_secs_f64() * 1000.0; assert!(!out.is_empty()); println!("openh264 1080p init={init_ms:.1}ms first_frame={first_ms:.1}ms"); - assert!(init_ms < 100.0, "OpenH264 init unexpectedly slow: {init_ms}ms"); + // Headroom for heavily-loaded runners: the assertion exists to catch + // pathological rebuilds (seconds), not a few ms of host contention. + assert!(init_ms < 250.0, "OpenH264 init unexpectedly slow: {init_ms}ms"); } } diff --git a/pixelflux/src/encoders/overlay.rs b/pixelflux/src/encoders/overlay.rs index 0fd7f1b..e0be09c 100644 --- a/pixelflux/src/encoders/overlay.rs +++ b/pixelflux/src/encoders/overlay.rs @@ -111,6 +111,23 @@ pub(crate) fn blend_pixel(dst: &mut [u8], r: u8, g: u8, b: u8, a: u8) { } } +/// Source-over compositing for PREMULTIPLIED sources (XFixes/Xcursor pixels are +/// premultiplied by format definition): dst = src + dst*(1-a). The straight-alpha +/// `blend_pixel` on premultiplied input multiplies alpha in twice and darkens +/// every translucent pixel (visible dark fringes on the composited cursor). +pub(crate) fn blend_pixel_premultiplied(dst: &mut [u8], r: u8, g: u8, b: u8, a: u8) { + if a == 255 { + dst[0] = b; + dst[1] = g; + dst[2] = r; + } else if a > 0 { + let ia = 255 - a as u32; + dst[0] = (b as u32 + dst[0] as u32 * ia / 255) as u8; + dst[1] = (g as u32 + dst[1] as u32 * ia / 255) as u8; + dst[2] = (r as u32 + dst[2] as u32 * ia / 255) as u8; + } +} + impl OverlayState { /// Load the watermark image from disk; `output_scale` is the output's fractional /// scale, ceiled to the integer buffer scale of the upload. A failed load clears the overlay. diff --git a/pixelflux/src/encoders/vaapi.rs b/pixelflux/src/encoders/vaapi.rs index 77e09bf..2e7f810 100644 --- a/pixelflux/src/encoders/vaapi.rs +++ b/pixelflux/src/encoders/vaapi.rs @@ -294,7 +294,8 @@ impl VaapiEncoder { let width = settings.width; let height = settings.height; - let fps = settings.target_fps as i32; + // AVRational.den must stay nonzero: a sub-1 fps setting would truncate to 0. + let fps = (settings.target_fps as i32).max(1); unsafe { let mut drm_device_ctx: *mut ff::AVBufferRef = ptr::null_mut(); @@ -436,7 +437,11 @@ impl VaapiEncoder { } set_opt(&mut opts, "async_depth", "1"); set_opt(&mut opts, "profile", "high"); - set_opt(&mut opts, "level", "4.1"); + set_opt( + &mut opts, + "level", + &super::min_h264_level(width as u32, height as u32, fps as u32).to_string(), + ); let ret = ff::avcodec_open2(encoder_ctx, codec, &mut opts); ff::av_dict_free(&mut opts); @@ -699,7 +704,12 @@ impl VaapiEncoder { } set_opt(&mut opts, "async_depth", "1"); set_opt(&mut opts, "profile", "high"); - set_opt(&mut opts, "level", "4.1"); + set_opt( + &mut opts, + "level", + &super::min_h264_level(self.width as u32, self.height as u32, self.fps.max(1) as u32) + .to_string(), + ); let ret = ff::avcodec_open2(self.encoder_ctx, self.codec, &mut opts); ff::av_dict_free(&mut opts); diff --git a/pixelflux/src/lib.rs b/pixelflux/src/lib.rs index 8780900..e6ff962 100644 --- a/pixelflux/src/lib.rs +++ b/pixelflux/src/lib.rs @@ -139,6 +139,26 @@ pub mod encoders { pub mod software; /// VA-API hardware H.264 encoder for Intel / AMD GPUs via FFmpeg. pub mod vaapi; + /// Minimum H.264 level for a stream of these dimensions at this frame rate, on the + /// 5.2-6.2 conformance ladder every hardware encoder here shares. NVENC's startup + /// clamp historically also covers all smaller sizes; keeping ONE ladder avoids two + /// encoders disagreeing on the same geometry. + pub(crate) fn min_h264_level(width: u32, height: u32, fps: u32) -> u32 { + let mbs = (width as u64).div_ceil(16) * (height as u64).div_ceil(16); + let mbps = mbs * fps.max(1) as u64; + const LEVELS: [(u32, u64, u64); 4] = [ + (52, 36864, 2073600), + (60, 139264, 4177920), + (61, 139264, 8355840), + (62, 139264, 16711680), + ]; + for &(level, max_fs, max_mbps) in &LEVELS { + if mbs <= max_fs && mbps <= max_mbps { + return level; + } + } + 62 + } /// Size the CBR VBV/HRD buffer so rate control has enough slack to hold quality steady /// without letting end-to-end latency drift upward. @@ -1554,6 +1574,89 @@ fn stop_capture_on_display(state: &mut AppState, display_id: u32) { /// the new logical size, resolve the encode path (zero-copy vs readback), and spawn the /// delivery (and readback-mode encode) threads. The single-display behavior of the former /// global StartCapture is preserved exactly for display 0. +/// Bring up the readback encode path (pixman readback → pool → encode thread) for a +/// capture whose zero-copy session is absent or just died. Mirrors the start-capture +/// bootstrap: u64::MAX content generations mark every pool slot stale so each one is +/// read back before its first publish, whatever the damage says. +fn bootstrap_readback_pool( + cap: &mut wayland::frontend::WlCapture, + display_id: u32, + use_gpu: bool, + try_gpu: bool, + prior: Option, +) { + let Some(deliver_tx) = cap.deliver_tx.clone() else { + return; + }; + let settings = cap.settings.clone(); + let pool = Arc::new(WlFramePool::new( + WL_POOL_SURFACES, + (settings.width.max(0) as usize) * (settings.height.max(0) as usize) * 4, + )); + cap.pool_last_render = vec![0; WL_POOL_SURFACES]; + cap.render_seq = 0; + cap.pool_content_gen = vec![u64::MAX; WL_POOL_SURFACES]; + cap.content_gen = 0; + let c = &cap.encode_controls; + c.bitrate_kbps.store(settings.video_bitrate_kbps, Ordering::Relaxed); + c.vbv_mult_milli.store( + (settings.video_vbv_multiplier * 1000.0).round() as i32, + Ordering::Relaxed, + ); + c.fps_milli.store( + (settings.target_fps.max(1.0) * 1000.0) as u64, + Ordering::Relaxed, + ); + let cfg = WlEncodeConfig { + settings: settings.clone(), + display_id, + use_gpu, + try_gpu, + prior, + recording_sink: cap.recording_sink.clone(), + deliver_tx, + controls: cap.encode_controls.clone(), + stats: cap.encode_stats.clone(), + }; + let pool2 = pool.clone(); + cap.encode_join = Some( + thread::Builder::new() + .name(format!("wl-encode-{display_id}")) + .spawn(move || wayland_encode_loop(&pool2, cfg)) + .expect("failed to spawn wl-encode thread"), + ); + cap.encode_pool = Some(pool); +} + +/// Rebuild a broken zero-copy hardware session with the startup construction (driver +/// match, EGL display hand-over); VAAPI stays excluded under fullcolor exactly like +/// startup. `None` means unrecoverable — the caller demotes to readback. +fn rebuild_zerocopy_encoder( + cap: &wayland::frontend::WlCapture, + state: &mut AppState, +) -> Option { + let settings = &cap.settings; + let encode_driver = get_gpu_driver(settings.encode_node_index.max(0)); + if encode_driver.contains("nvidia") { + let egl_display = state + .gles_renderer + .as_ref() + .map(|r| r.egl_context().display().get_display_handle().handle) + .unwrap_or(std::ptr::null()); + NvencEncoder::new(settings, egl_display) + .ok() + .map(GpuEncoder::Nvenc) + } else if !settings.video_fullcolor { + VaapiEncoder::new(settings).ok().map(GpuEncoder::Vaapi) + } else { + None + } +} + +/// Consecutive encode failures before the zero-copy path recovers (~0.5s at 60fps): +/// a hiccup outlasts it, anything longer starts to look like a dead session. +const HW_ERROR_RECOVERY_THRESHOLD: u32 = 30; + fn start_capture_on_display( state: &mut AppState, display_id: u32, @@ -1882,6 +1985,7 @@ fn start_capture_on_display( pending_force_idr: false, needs_full_render: true, last_tick: None, + hw_error_streak: 0, }; { @@ -1914,47 +2018,13 @@ fn start_capture_on_display( } if cap.video_encoder.is_none() { - if let Some(deliver_tx) = cap.deliver_tx.clone() { - let pool = Arc::new(WlFramePool::new( - WL_POOL_SURFACES, - (settings.width.max(0) as usize) * (settings.height.max(0) as usize) * 4, - )); - cap.pool_last_render = vec![0; WL_POOL_SURFACES]; - cap.render_seq = 0; - // u64::MAX marks every slot stale so each one is read back before its - // first publish, whatever the damage says. - cap.pool_content_gen = vec![u64::MAX; WL_POOL_SURFACES]; - cap.content_gen = 0; - let c = &cap.encode_controls; - c.bitrate_kbps.store(settings.video_bitrate_kbps, Ordering::Relaxed); - c.vbv_mult_milli.store( - (settings.video_vbv_multiplier * 1000.0).round() as i32, - Ordering::Relaxed, - ); - c.fps_milli.store( - (settings.target_fps.max(1.0) * 1000.0) as u64, - Ordering::Relaxed, - ); - let cfg = WlEncodeConfig { - settings: settings.clone(), - display_id, - use_gpu: state.use_gpu, - try_gpu: gpu_intent && (!state.use_gpu || different_gpu), - prior: prior_readback_encoder.take(), - recording_sink: cap.recording_sink.clone(), - deliver_tx, - controls: cap.encode_controls.clone(), - stats: cap.encode_stats.clone(), - }; - let pool2 = pool.clone(); - cap.encode_join = Some( - thread::Builder::new() - .name(format!("wl-encode-{display_id}")) - .spawn(move || wayland_encode_loop(&pool2, cfg)) - .expect("failed to spawn wl-encode thread"), - ); - cap.encode_pool = Some(pool); - } + bootstrap_readback_pool( + &mut cap, + display_id, + state.use_gpu, + gpu_intent && (!state.use_gpu || different_gpu), + prior_readback_encoder.take(), + ); } else { cap.encode_stats.n_stripes.store(1, Ordering::Relaxed); *cap.encode_stats.desc.lock().unwrap() = @@ -2700,6 +2770,27 @@ fn render_node_tick( } if let Some(cap) = node.capture.as_mut() { + // A dead encode thread (panic, unexpected exit) cannot drain the pool; + // every publish would park the slot and each tick would silently skip + // while is_capturing still reports true. Rebuild the readback path in + // place, reusing a cleanly handed-back encoder session if any. + if cap.encode_join.as_ref().is_some_and(|j| j.is_finished()) { + let prior = cap + .encode_join + .take() + .map(|j| j.join().ok().flatten()) + .flatten(); + if let Some(pool) = cap.encode_pool.take() { + pool.shutdown(); + } + let s = &cap.settings; + let try_gpu = s.output_mode == 1 + && !s.use_openh264 + && !(s.use_cpu || s.encode_node_index == -1); + eprintln!("[Wayland] encode thread died; rebuilding the readback path."); + bootstrap_readback_pool(cap, node.id, state.use_gpu, try_gpu, prior); + cap.request_idr(); + } if is_memory_throttling { if cap.encode_pool.is_none() { cap.frame_counter = cap.frame_counter.wrapping_add(1); @@ -2792,6 +2883,7 @@ fn render_node_tick( }; if let Ok(data) = result { + cap.hw_error_streak = 0; if !data.is_empty() { frame_out = true; cap.encode_stats.frames.fetch_add(1, Ordering::Relaxed); @@ -2818,6 +2910,34 @@ fn render_node_tick( } } else if let Err(e) = result { eprintln!("HW Encode Error: {}", e); + cap.hw_error_streak = cap.hw_error_streak.saturating_add(1); + if cap.hw_error_streak == HW_ERROR_RECOVERY_THRESHOLD { + // The zero-copy session persistently fails after having + // worked (driver hiccup, CUDA pressure from a co-tenant): + // rebuild the session once, else demote to the readback + // path. Streaming black frames forever is not an option. + match rebuild_zerocopy_encoder(cap, state) { + Some(enc) => { + cap.video_encoder = Some(enc); + cap.pending_force_idr = true; + eprintln!("[Wayland] zero-copy HW encoder rebuilt after repeated encode errors."); + } + None => { + eprintln!("[Wayland] zero-copy HW encoder unrecoverable; demoting to readback encode."); + cap.video_encoder = None; + // Mirror the startup intent: readback still + // tries the GPU unless the operator opted out. + let s = &cap.settings; + let try_gpu = s.output_mode == 1 + && !s.use_openh264 + && !(s.use_cpu || s.encode_node_index == -1); + bootstrap_readback_pool( + cap, node.id, state.use_gpu, try_gpu, None, + ); + } + } + cap.hw_error_streak = 0; + } } } // An unserved request stays armed: on an infinite GOP an IDR lost to an @@ -3529,11 +3649,20 @@ fn run_wayland_thread( } else { vec![mime.clone()] }; + let payload = std::sync::Arc::new((mime, data)); smithay::wayland::selection::data_device::set_data_device_selection( + &state.dh, + &state.seat.clone(), + mimes.clone(), + payload.clone(), + ); + // Middle-click parity with the X11 clipboard bridge: the same + // offer backs the primary selection too. + smithay::wayland::selection::primary_selection::set_primary_selection( &state.dh, &state.seat.clone(), mimes, - std::sync::Arc::new((mime, data)), + payload, ); // The selection is compositor-owned now; a later SetClipboardCallback // must not try to re-read a client source that no longer holds it. @@ -5529,10 +5658,16 @@ impl ScreenCapture { /// atexit sweep. impl Drop for ScreenCapture { fn drop(&mut self) { - if let Ok(st) = self.inner.lock() { + if let Ok(mut st) = self.inner.lock() { if let Some(c) = &st.controls { c.stop.store(true, Ordering::Relaxed); } + // Pair the cursor-monitor acquire from start_capture: a dropped + // handle that never got stop_capture would pin the refcount and its + // monitor thread forever. + if std::mem::take(&mut st.cursor_ref) && !crate::PY_SHUTDOWN.load(Ordering::Relaxed) { + Python::attach(crate::x11::cursor::release); + } } } } @@ -5667,6 +5802,7 @@ fn _stop_all_captures(py: Python<'_>) { py.detach(crate::recorder::finalize_on_exit); *PENDING_CURSOR_CALLBACK.lock().unwrap() = None; crate::x11::cursor::shutdown(); + crate::wayland::dcclient::unwatch_all(); if let Some(slot) = LIVE_X11.get() { for c in slot.lock().unwrap().iter() { c.stop.store(true, Ordering::Relaxed); diff --git a/pixelflux/src/wayland/dcclient.rs b/pixelflux/src/wayland/dcclient.rs index 0756662..286b139 100644 --- a/pixelflux/src/wayland/dcclient.rs +++ b/pixelflux/src/wayland/dcclient.rs @@ -329,6 +329,17 @@ pub(crate) fn watch(socket_path: &str, callback: Py) -> Result<(), String Ok(()) } +/// Stop every clipboard watch (process teardown sweep; watches are NOT tied to +/// captures, so the global stop helper must reach them explicitly). +pub(crate) fn unwatch_all() { + let mut reg = WATCHERS.lock().unwrap(); + if let Some(map) = reg.as_mut() { + for (_, handle) in map.drain() { + handle.stop.store(true, Ordering::Relaxed); + } + } +} + /// Stop the watch on `socket_path` (no-op when none is running). pub(crate) fn unwatch(socket_path: &str) { let mut reg = WATCHERS.lock().unwrap(); @@ -350,6 +361,9 @@ fn watch_loop(socket_path: &str, callback: Py, stop: &AtomicBool) -> Resu .and_then(|o| state.offer_mimes.get(&o.id()).cloned()) .unwrap_or_default(); if !mimes.is_empty() { + if crate::PY_SHUTDOWN.load(Ordering::Relaxed) { + break; + } Python::attach(|py| { if let Err(e) = callback.call1(py, (mimes,)) { e.print(py); diff --git a/pixelflux/src/wayland/frontend.rs b/pixelflux/src/wayland/frontend.rs index 954997b..ac46388 100644 --- a/pixelflux/src/wayland/frontend.rs +++ b/pixelflux/src/wayland/frontend.rs @@ -226,6 +226,9 @@ pub struct WlCapture { /// Last tick this capture actually rendered; paces per-display fps under the one /// shared render timer (which fires at the fastest active capture's rate). pub last_tick: Option, + /// Consecutive zero-copy encode failures: past the recovery threshold the session + /// is rebuilt once, then the capture demotes to the readback path. + pub hw_error_streak: u32, } impl WlCapture { @@ -1314,7 +1317,7 @@ impl SelectionHandler for AppState { _seat: Seat, user_data: &Self::SelectionUserData, ) { - if ty != SelectionTarget::Clipboard { + if ty != SelectionTarget::Clipboard && ty != SelectionTarget::Primary { return; } let payload = user_data.clone(); diff --git a/pixelflux/src/wayland/wlclient.rs b/pixelflux/src/wayland/wlclient.rs index f56717a..31a5fd6 100644 --- a/pixelflux/src/wayland/wlclient.rs +++ b/pixelflux/src/wayland/wlclient.rs @@ -193,9 +193,25 @@ pub(crate) fn wake_pipe() -> Result<(OwnedFd, OwnedFd), String> { /// Read `fd` to EOF. The deadline is per-chunk (idle), so a large transfer that /// keeps flowing is never cut off while a stalled writer still errors out. pub(crate) fn read_fd_to_end(fd: &OwnedFd, idle: Duration) -> Result, String> { + read_fd_to_end_capped(fd, idle, READ_CAP_MAX) +} + +/// Same as `read_fd_to_end` but bounded: a hostile or stuck source can otherwise +/// stream unbounded bytes into memory (the compositor-internal read caps at the +/// same 64 MiB the selkies clipboard protocol allows). +const READ_CAP_MAX: usize = 64 * 1024 * 1024; + +pub(crate) fn read_fd_to_end_capped( + fd: &OwnedFd, + idle: Duration, + cap: usize, +) -> Result, String> { let mut out = Vec::new(); let mut chunk = [0u8; 65536]; loop { + if out.len() >= cap { + return Err(format!("clipboard source exceeds the {cap} byte cap")); + } if !wait_readable(fd.as_raw_fd(), idle)? { return Err("clipboard source stalled".into()); } @@ -212,7 +228,8 @@ pub(crate) fn read_fd_to_end(fd: &OwnedFd, idle: Duration) -> Result, St if n == 0 { return Ok(out); } - out.extend_from_slice(&chunk[..n as usize]); + let take = (cap - out.len()).min(n as usize); + out.extend_from_slice(&chunk[..take]); } } diff --git a/pixelflux/src/x11/mod.rs b/pixelflux/src/x11/mod.rs index 4e6a49b..98370cf 100644 --- a/pixelflux/src/x11/mod.rs +++ b/pixelflux/src/x11/mod.rs @@ -33,7 +33,7 @@ use x11rb::protocol::xfixes::ConnectionExt as XfixesExt; use x11rb::protocol::xproto::{ConnectionExt as XprotoExt, ImageFormat}; use x11rb::rust_connection::RustConnection; -use crate::encoders::overlay::blend_pixel; +use crate::encoders::overlay::blend_pixel_premultiplied; use crate::encoders::software::EncodedStripe; use crate::pipeline::X11Pipeline; use crate::recording_sink::RecordingSink; @@ -195,20 +195,25 @@ impl ShmSurface { } } +/// Clamp a capture offset to the live root so a grab always has at least one root pixel in +/// range: a past-end offset fails every `shm_get_image` with BadMatch and would burn out the +/// failure budget with no chance of recovery. The upper clamp also keeps protocol-range roots +/// (INT16 coordinates) from truncating a wider value through the `as i16` cast. +fn clamp_offset(x: i32, root: u16) -> i16 { + let upper = (root as i32).saturating_sub(1).min(i16::MAX as i32 - 1); + x.max(0).min(upper) as i16 +} + /// Resolve the capture dimensions `shm_get_image` will read, bounded so the region can never -/// run past the live root from the capture offset. -/// -/// The bound is the reason this function exists: a grab that ran off the root edge would fail the -/// request outright, so the size is always clamped to what is actually there. With auto-adjust (or -/// an unset width/height `<= 0`) the capture tracks the full root minus the capture offset; -/// otherwise the requested size is clamped to what is available from that offset. `capture_x` / -/// `capture_y` are floored at `>= 0` exactly as `shm_get_image` consumes them, and saturating -/// subtraction plus a final `u16` clamp (minimum 2) keep pathological settings from overflowing or -/// collapsing to an unusable surface. H.264 (`output_mode == 1`) requires even dimensions, so both -/// are then rounded down to a multiple of two. +/// run past the live root from the (clamped) capture offset: with auto-adjust (or unset +/// width/height `<= 0`) the capture tracks the full root minus the offset, otherwise the +/// requested size is clamped to what is available from it. Saturating subtraction plus a final +/// `u16` clamp (minimum 2) keep pathological settings from overflowing or collapsing to an +/// unusable surface. H.264 (`output_mode == 1`) requires even dimensions, so both are then +/// rounded down to a multiple of two. fn resolve_dims(root_w: u16, root_h: u16, s: &RustCaptureSettings) -> (u16, u16) { - let cap_x = s.capture_x.max(0); - let cap_y = s.capture_y.max(0); + let cap_x = clamp_offset(s.capture_x, root_w) as i32; + let cap_y = clamp_offset(s.capture_y, root_h) as i32; let avail_w = (root_w as i32).saturating_sub(cap_x).max(2); let avail_h = (root_h as i32).saturating_sub(cap_y).max(2); let mut w = if s.auto_adjust_screen_capture_size || s.width <= 0 { @@ -230,6 +235,75 @@ fn resolve_dims(root_w: u16, root_h: u16, s: &RustCaptureSettings) -> (u16, u16) (w as u16, h as u16) } +/// Rebuild the X channel + capture surfaces after a connection-level failure (Xorg restart, +/// VT switch), refreshing the offset/dims state against the live root. The pool is drained +/// first so the encode thread can never be reading a surface as it is destroyed, and its +/// generation is bumped so the encoder rebuilds against the new layout. Every fallible step +/// completes before the swap, and the old channel/surfaces are destroyed best-effort after +/// it (their server is likely gone; the local detaches still run, and the kernel frees +/// each old segment once nothing maps it). +#[allow(clippy::too_many_arguments)] +fn try_rebuild_channel( + conn: &mut RustConnection, + root: &mut u32, + rsettings: &RustCaptureSettings, + cap_x: &mut i16, + cap_y: &mut i16, + cap_w: &mut u16, + cap_h: &mut u16, + pool_n: usize, + pool: &FramePool, + surfaces: &mut Vec, + stop: &AtomicBool, +) -> Result<(), String> { + let (new_conn, new_screen) = + x11rb::connect(None).map_err(|e| format!("X11 reconnect failed: {e}"))?; + new_conn + .shm_query_version() + .map_err(|e| format!("shm_query_version: {e}"))? + .reply() + .map_err(|e| format!("XShm unavailable on reconnect: {e}"))?; + new_conn + .xfixes_query_version(5, 0) + .map_err(|e| format!("xfixes_query_version: {e}"))? + .reply() + .map_err(|e| format!("XFixes unavailable on reconnect: {e}"))?; + let new_root = new_conn.setup().roots[new_screen].root; + let geo = new_conn + .get_geometry(new_root) + .map_err(|e| format!("get_geometry: {e}"))? + .reply() + .map_err(|e| format!("get_geometry reply on reconnect: {e}"))?; + if !pool.drain_for_resize(pool_n, stop) { + return Err("pool drain interrupted by stop during reconnect".to_string()); + } + let (fw, fh) = resolve_dims(geo.width, geo.height, rsettings); + let mut fresh: Vec = Vec::with_capacity(pool_n); + for _ in 0..pool_n { + match ShmSurface::create(&new_conn, fw, fh) { + Ok(s) => fresh.push(s), + Err(e) => { + for s in fresh.iter_mut() { + s.destroy(&new_conn); + } + return Err(e); + } + } + } + for s in surfaces.iter_mut() { + s.destroy(conn); + } + *conn = new_conn; + *root = new_root; + *cap_x = clamp_offset(rsettings.capture_x, geo.width); + *cap_y = clamp_offset(rsettings.capture_y, geo.height); + *cap_w = fw; + *cap_h = fh; + *surfaces = fresh; + pool.bump_generation(); + Ok(()) +} + /// Grab one frame of the capture region into `surface` with a single XShm round-trip. /// /// One `shm_get_image` request is atomic — the server never interleaves another client @@ -273,9 +347,10 @@ pub(crate) fn cursor_image_origin(x: i16, y: i16, xhot: u16, yhot: u16, cap_x: i (x as i32 - xhot as i32 - cap_x, y as i32 - yhot as i32 - cap_y) } -/// Composite the XFixes cursor (ARGB `u32` per pixel) onto the BGRA frame with its top-left -/// at `(img_x, img_y)`, blending each pixel through `blend_pixel` with per-pixel bounds clipping so -/// an image straddling a frame edge writes only its in-frame portion. +/// Composite the XFixes cursor (ARGB `u32` per pixel, premultiplied by the XFixes +/// format definition) onto the BGRA frame with its top-left at `(img_x, img_y)`, +/// blending each pixel through `blend_pixel_premultiplied` with per-pixel bounds +/// clipping so an image straddling a frame edge writes only its in-frame portion. #[allow(clippy::too_many_arguments)] pub(crate) fn overlay_cursor( frame: &mut [u8], @@ -304,7 +379,7 @@ pub(crate) fn overlay_cursor( let g = ((px >> 8) & 0xFF) as u8; let b = (px & 0xFF) as u8; let off = ty as usize * stride + tx as usize * 4; - blend_pixel(&mut frame[off..off + 4], r, g, b, a); + blend_pixel_premultiplied(&mut frame[off..off + 4], r, g, b, a); } } } @@ -689,9 +764,9 @@ pub fn run_capture( where F: FnMut(Vec) + Send + 'static, { - let (conn, screen_num) = + let (mut conn, screen_num) = x11rb::connect(None).map_err(|e| format!("X11 connect failed: {e}"))?; - let root = conn.setup().roots[screen_num].root; + let mut root = conn.setup().roots[screen_num].root; let root_depth = conn.setup().roots[screen_num].root_depth; let bpp = conn @@ -724,14 +799,18 @@ where let mut rsettings = settings.clone(); let (mut cap_w, mut cap_h) = resolve_dims(geo.width, geo.height, &rsettings); - let mut cap_x = rsettings.capture_x.max(0) as i16; - let mut cap_y = rsettings.capture_y.max(0) as i16; + let mut cap_x = clamp_offset(rsettings.capture_x, geo.width); + let mut cap_y = clamp_offset(rsettings.capture_y, geo.height); const POOL_N: usize = 3; let mut surfaces: Vec = Vec::with_capacity(POOL_N); for _ in 0..POOL_N { surfaces.push(ShmSurface::create(&conn, cap_w, cap_h)?); } + // Consecutive channel rebuilds without one good grab in between: geometry + // errors can burn out the grab budget repeatedly under a dead region, so + // recovery only gets a bounded number of channel rebuilds. + let mut reconnect_streak = 0u32; let pool = Arc::new(FramePool::new(POOL_N)); let mut watermark = crate::encoders::overlay::OverlayState::default(); @@ -746,7 +825,19 @@ where crate::boost_thread_priority(-10); let _ = encode_tid_tx.send(thread::current().id()); let mut on_frame = on_frame; - encode_loop(&enc_pool, &enc_controls, &enc_settings, &mut on_frame); + // An encoder panic must never wedge the capture: without the pool shutdown + // the capture thread would block in publish forever while is_capturing still + // reports true on this thread's handle. + let guard_pool = enc_pool.clone(); + let guard_controls = enc_controls.clone(); + let result = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| { + encode_loop(&enc_pool, &enc_controls, &enc_settings, &mut on_frame); + })); + if result.is_err() { + eprintln!("[pixelflux x11] encode thread panicked; shutting the pool to fail the capture"); + guard_pool.shutdown(); + guard_controls.stop.store(true, Ordering::Relaxed); + } }); let mut next_frame = Instant::now(); @@ -784,9 +875,9 @@ where // settings) would glue resolve_dims back to the ROOT size, undoing the // re-target on the next geometry poll. w/h <= 0 keep following the root. rsettings.auto_adjust_screen_capture_size = nw <= 0 || nh <= 0; - cap_x = nx.max(0) as i16; - cap_y = ny.max(0) as i16; if let Some(g) = conn.get_geometry(root).ok().and_then(|c| c.reply().ok()) { + cap_x = clamp_offset(nx, g.width); + cap_y = clamp_offset(ny, g.height); let (fw, fh) = resolve_dims(g.width, g.height, &rsettings); if fw != cap_w || fh != cap_h { if !pool.drain_for_resize(POOL_N, &controls.stop) { @@ -816,6 +907,10 @@ where if (settings.auto_adjust_screen_capture_size || grab_failures > 0) && geometry_check <= 0 { geometry_check = GEOMETRY_POLL_FRAMES; if let Some(g) = conn.get_geometry(root).ok().and_then(|c| c.reply().ok()) { + // A root shrink can strand even a previously-valid offset past + // the new edge; re-clamp it against the live root. + cap_x = clamp_offset(rsettings.capture_x, g.width); + cap_y = clamp_offset(rsettings.capture_y, g.height); let (nw, nh) = resolve_dims(g.width, g.height, &rsettings); if nw != cap_w || nh != cap_h { if !pool.drain_for_resize(POOL_N, &controls.stop) { @@ -847,8 +942,11 @@ where Some(i) => i, None => break, }; - let surface = &mut surfaces[idx]; - if let Err(e) = grab_frame(&conn, root, surface, cap_x, cap_y, cap_w, cap_h) { + let grab_result = { + let surface = &mut surfaces[idx]; + grab_frame(&conn, root, surface, cap_x, cap_y, cap_w, cap_h) + }; + if let Err(e) = grab_result { // Most likely an external root resize between geometry polls made // the grab run past the root edge: return the surface, re-poll // geometry immediately, and only give up if it never recovers. @@ -856,11 +954,44 @@ where grab_failures += 1; geometry_check = 0; if grab_failures > 120 { - return Err(e); + // A burn-out usually means the X connection itself is gone + // (Xorg restart, VT switch): every request fails until the + // server is back. Rebuild the channel a few times before + // declaring the capture dead — a recovered server resumes the + // stream where a dead thread would leave a black screen until + // the Python watchdog restarts everything. + let mut recovered = false; + for _attempt in 0..5 { + if controls.stop.load(Ordering::Relaxed) { + break; + } + std::thread::sleep(Duration::from_secs(1)); + match try_rebuild_channel( + &mut conn, &mut root, &rsettings, + &mut cap_x, &mut cap_y, &mut cap_w, &mut cap_h, + POOL_N, &pool, &mut surfaces, &controls.stop, + ) { + Ok(()) => { recovered = true; break; } + Err(re) => { + eprintln!("[pixelflux x11] reconnect attempt failed: {re}"); + } + } + } + if !recovered { + return Err(format!("capture could not recover: {e}")); + } + grab_failures = 0; + geometry_check = 0; + reconnect_streak += 1; + if reconnect_streak >= 4 { + return Err("capture recovered its channel but the region never grabbed again".to_string()); + } } continue; } grab_failures = 0; + reconnect_streak = 0; + let surface = &mut surfaces[idx]; let frame_w = cap_w as i32; let frame_h = cap_h as i32; @@ -1114,6 +1245,44 @@ mod cursor_tests { assert_eq!(px(5, 5), 0, "no pixel at the un-offset (hotspot) corner"); } + #[test] + fn offset_clamps_into_root() { + assert_eq!(clamp_offset(100, 1920), 100); + assert_eq!(clamp_offset(-50, 1920), 0); + // Past-end offsets keep one root pixel instead of failing forever. + assert_eq!(clamp_offset(100000, 1920), 1919); + // A protocol-max root still maps into i16 X coordinates. + assert_eq!(clamp_offset(40000, u16::MAX), 32766); + } + + #[test] + fn resolve_dims_survives_past_end_offset() { + let mut s = RustCaptureSettings::default(); + s.capture_x = 100000; + s.capture_y = 100000; + s.width = 1920; + s.height = 1080; + s.output_mode = 1; + let (w, h) = resolve_dims(1920, 1080, &s); + assert!(w >= 2 && h >= 2, "a past-end offset now degrades to a small grab, not death"); + assert_eq!(w % 2, 0); + assert_eq!(h % 2, 0); + } + + /// `overlay_cursor` treats XFixes pixels as premultiplied: half-alpha at half + /// intensity stays at that intensity over black (the straight-alpha formula + /// would multiply alpha twice and darken it). + #[test] + fn overlay_blends_premultiplied_pixels() { + let stride = 4 * 4; + let mut frame = vec![0u8; stride * 4]; + // a=128, g=128 (premultiplied half-intensity), b=64. + let pixels = [0x8000_8040u32; 4]; + overlay_cursor(&mut frame, stride, 4, 4, 2, 2, &pixels, 0, 0); + assert_eq!(frame[0], 64, "blue channel: dst = 64 over black"); + assert_eq!(frame[1], 128, "premultiplied half-alpha stays 128, not darkened to 64"); + } + /// `overlay_cursor` clips a negative origin at the frame edge: with the origin at (-1,-1) /// only the in-frame quadrant is written, landing the cursor's bottom-right pixel at (0,0). #[test] From b5ae4b74eb6b019d501c92b4ce7ea1b9de479797 Mon Sep 17 00:00:00 2001 From: Seungmin Kim <8457324+ehfd@users.noreply.github.com> Date: Sat, 1 Aug 2026 13:12:36 +0900 Subject: [PATCH 12/21] fix: Various findings --- pixelflux/src/encoders/nvenc.rs | 12 ++-- pixelflux/src/lib.rs | 109 +++++++++++++++++++++++++----- pixelflux/src/wayland/cursor.rs | 98 +++++++++++++++++++++++---- pixelflux/src/wayland/frontend.rs | 2 + 4 files changed, 186 insertions(+), 35 deletions(-) diff --git a/pixelflux/src/encoders/nvenc.rs b/pixelflux/src/encoders/nvenc.rs index 1eca638..faddfb0 100644 --- a/pixelflux/src/encoders/nvenc.rs +++ b/pixelflux/src/encoders/nvenc.rs @@ -934,11 +934,11 @@ impl NvencEncoder { NV_ENC_VUI_VIDEO_FORMAT::NV_ENC_VUI_VIDEO_FORMAT_UNSPECIFIED; config.encodeCodecConfig.h264Config.h264VUIParameters.colourDescriptionPresentFlag = 1; config.encodeCodecConfig.h264Config.h264VUIParameters.colourPrimaries = - NV_ENC_VUI_COLOR_PRIMARIES::NV_ENC_VUI_COLOR_PRIMARIES_BT709; + NV_ENC_VUI_COLOR_PRIMARIES::NV_ENC_VUI_COLOR_PRIMARIES_SMPTE170M; config.encodeCodecConfig.h264Config.h264VUIParameters.transferCharacteristics = - NV_ENC_VUI_TRANSFER_CHARACTERISTIC::NV_ENC_VUI_TRANSFER_CHARACTERISTIC_BT709; + NV_ENC_VUI_TRANSFER_CHARACTERISTIC::NV_ENC_VUI_TRANSFER_CHARACTERISTIC_SMPTE170M; config.encodeCodecConfig.h264Config.h264VUIParameters.colourMatrix = - NV_ENC_VUI_MATRIX_COEFFS::NV_ENC_VUI_MATRIX_COEFFS_BT709; + NV_ENC_VUI_MATRIX_COEFFS::NV_ENC_VUI_MATRIX_COEFFS_SMPTE170M; config.encodeCodecConfig.h264Config.chromaFormatIDC = if is_444 { 3 } else { 1 }; config.encodeCodecConfig.h264Config.h264VUIParameters.videoFullRangeFlag = if is_444 { 1 } else { 0 }; @@ -1616,7 +1616,11 @@ impl NvencEncoder { } /// Encode a host ARGB frame by uploading it straight into the ARGB input surface, with no - /// CPU-side colour conversion. + /// CPU-side colour conversion. NVENC performs its hardware ARGB->YUV conversion with the + /// BT.601 matrix while the session's VUI is written BT.709 (a slight core-visual shift vs + /// the BT.709 CPU path): the VUI is reinterpreted by decoders but cannot be made to match + /// the nvenc hardware conversion — a 601->709 CPU prepass is intentionally skipped here to + /// keep this path zero-copy. /// /// The packed rows are copied host→device into the registered ARGB surface and NVENC's hardware /// CSC produces the YUV. Bytes must be in NVENC ARGB order (`B,G,R,A` in memory) — the host BGRA diff --git a/pixelflux/src/lib.rs b/pixelflux/src/lib.rs index e6ff962..75c9011 100644 --- a/pixelflux/src/lib.rs +++ b/pixelflux/src/lib.rs @@ -388,6 +388,7 @@ pub struct LiveTunables { pub video_streaming_mode: bool, pub keyframe_interval_s: f64, pub capture_cursor: bool, + pub cursor_size_cap: i32, } impl LiveTunables { @@ -404,6 +405,7 @@ impl LiveTunables { video_streaming_mode: s.video_streaming_mode, keyframe_interval_s: s.keyframe_interval_s, capture_cursor: s.capture_cursor, + cursor_size_cap: s.cursor_size_cap, } } @@ -418,6 +420,7 @@ impl LiveTunables { s.video_paintover_burst_frames = self.video_paintover_burst_frames; s.video_streaming_mode = self.video_streaming_mode; s.keyframe_interval_s = self.keyframe_interval_s; + s.cursor_size_cap = self.cursor_size_cap; s.capture_cursor = self.capture_cursor; } } @@ -499,15 +502,27 @@ pub(crate) fn extract_settings(settings: &Bound<'_, PyAny>) -> PyResult i32 { + if v <= 0 { 0 } else { v.min(MAX_CAPTURE_DIM) } + }; + let sanitize_fps = |v: f64| -> f64 { + if v.is_finite() && v > 0.0 { v.min(MAX_FPS) } else { DEFAULT_FPS } + }; + let sanitize_scale = |v: f64| -> f64 { + if v.is_finite() && v > 0.0 { v.min(MAX_SCALE) } else { 1.0 } + }; + Ok(RustCaptureSettings { - width: settings.getattr("capture_width")?.extract()?, - height: settings.getattr("capture_height")?.extract()?, - scale, + width: sanitize_dim(settings.getattr("capture_width")?.extract()?), + height: sanitize_dim(settings.getattr("capture_height")?.extract()?), + scale: sanitize_scale(scale), capture_x: settings.getattr("capture_x")?.extract()?, capture_y: settings.getattr("capture_y")?.extract()?, - target_fps: settings.getattr("target_fps")?.extract()?, - jpeg_quality: settings.getattr("jpeg_quality")?.extract()?, - paint_over_jpeg_quality: settings.getattr("paint_over_jpeg_quality")?.extract()?, + target_fps: sanitize_fps(settings.getattr("target_fps")?.extract()?), + jpeg_quality: settings.getattr("jpeg_quality")?.extract::()?.clamp(1, 100), + paint_over_jpeg_quality: settings.getattr("paint_over_jpeg_quality")?.extract::()?.clamp(1, 100), use_paint_over_quality: settings.getattr("use_paint_over_quality")?.extract()?, paint_over_trigger_frames: settings.getattr("paint_over_trigger_frames")?.extract()?, damage_block_threshold: settings.getattr("damage_block_threshold")?.extract()?, @@ -1043,6 +1058,14 @@ impl WlEncodeControls { /// publish slot, so a deeper pool would only add latency (staler frames), never overlap. const WL_POOL_SURFACES: usize = 2; +/// Upper bounds for Python-supplied capture geometry: keeps a hostile or buggy setting from +/// turning into a multi-GB `vec![]` allocation that would abort the process. 16384 covers every +/// real display wall; the fps/scale bounds keep timing math finite and sane. +const MAX_CAPTURE_DIM: i32 = 16384; +const MAX_FPS: f64 = 1000.0; +const DEFAULT_FPS: f64 = 60.0; +const MAX_SCALE: f64 = 8.0; + /// Shared capture stats: whichever thread owns the encoders counts frames/stripes and /// composes `desc` + `n_stripes` (the encoder half of the 1 s debug log line); the calloop /// log loads, prints and resets the counters. @@ -3298,8 +3321,20 @@ fn run_wayland_thread( let width: i32 = if initial_width > 0 { initial_width } else { 1024 }; let height: i32 = if initial_height > 0 { initial_height } else { 768 }; - let mut event_loop = EventLoop::::try_new().expect("Unable to create event_loop"); - let display: Display = Display::new().unwrap(); + let mut event_loop = match EventLoop::::try_new() { + Ok(l) => l, + Err(e) => { + eprintln!("[Wayland] compositor thread aborting: event loop init failed: {e}"); + return; + } + }; + let display: Display = match Display::new() { + Ok(d) => d, + Err(e) => { + eprintln!("[Wayland] compositor thread aborting: display init failed: {e}"); + return; + } + }; let dh: DisplayHandle = display.handle(); unsafe { if let Ok(lib) = libloading::Library::new("libwayland-server.so.0") { @@ -3461,7 +3496,7 @@ fn run_wayland_thread( ..RustCaptureSettings::default() }, cursor_callback_set: false, - cursor_tx: wayland::cursor::spawn_cursor_worker(cursor_size), + cursor_tx: wayland::cursor::spawn_cursor_worker(cursor_size, 32), clipboard_callback: None, pending_clipboard_read: None, current_selection_mime: None, @@ -3972,6 +4007,7 @@ fn run_wayland_thread( } ThreadCommand::UpdateTunables { display_id, tunables: t } => { state.render_cursor_on_framebuffer = t.capture_cursor; + let _ = state.cursor_tx.send(CursorJob::SetSizeCap(t.cursor_size_cap)); if display_id == 0 { t.apply_to(&mut state.settings); } @@ -4030,7 +4066,13 @@ fn run_wayland_thread( }) .unwrap(); - let source = ListeningSocketSource::new_auto().unwrap(); + let source = match ListeningSocketSource::new_auto() { + Ok(s) => s, + Err(e) => { + eprintln!("[Wayland] compositor thread aborting: could not bind a wayland-N socket (XDG_RUNTIME_DIR unset/full?): {e}"); + return; + } + }; let socket_name = source.socket_name().to_string_lossy().into_owned(); println!("[Wayland] Socket listening on: {:?}", socket_name); std::env::set_var("WAYLAND_DISPLAY", &socket_name); @@ -4147,14 +4189,16 @@ fn run_wayland_thread( .iter() .filter_map(|n| n.capture.as_ref().map(|c| c.settings.target_fps)) .fold(0.0f64, f64::max); - let fps = (if is_memory_throttling { + let raw_fps = if is_memory_throttling { 5.0 } else if max_fps > 0.0 { max_fps } else { state.settings.target_fps - }) - .max(1.0); + }; + // Settings are sanitized at the Python boundary; this final guard keeps a + // non-finite value from ever reaching Duration::from_secs_f64 (panics on NaN). + let fps = if raw_fps.is_finite() && raw_fps > 0.0 { raw_fps.min(MAX_FPS) } else { DEFAULT_FPS }; let target_frame_duration = Duration::from_secs_f64(1.0 / fps); let wait_duration = target_frame_duration.saturating_sub(work_elapsed); let final_wait = if wait_duration.as_millis() < 1 { Duration::from_millis(1) } else { wait_duration }; @@ -4165,7 +4209,11 @@ fn run_wayland_thread( event_loop .handle() .insert_source(Generic::new(display, Interest::READ, Mode::Level), |_, display, _state| { - unsafe { display.get_mut().dispatch_clients(_state).unwrap(); } + // A single misbehaving client must not take the compositor (and every other + // session) down with it. + if let Err(e) = unsafe { display.get_mut().dispatch_clients(_state) } { + eprintln!("[Wayland] client dispatch error: {e:?}"); + } Ok(PostAction::Continue) }) .unwrap(); @@ -4173,10 +4221,10 @@ fn run_wayland_thread( crate::computer_use::register_wayland_backend(command_tx.clone()); crate::computer_use::spawn_cu_from_env(); - event_loop.run(None, &mut state, |state| { + let _ = event_loop.run(None, &mut state, |state| { state.process_pending_clipboard_read(); - state.dh.flush_clients().unwrap(); - }).unwrap(); + let _ = state.dh.flush_clients(); + }); } /// Zero-copy encoded-frame handoff to Python. Owns the encoded `Vec` and @@ -5115,6 +5163,10 @@ impl ScreenCapture { return Ok(()); } + // A fresh start proves the interpreter is alive: clear the teardown flag the atexit + // sweep may have set (the Wayland start path already does the same), or the delivery + // thread below would drop every frame. + PY_SHUTDOWN.store(false, Ordering::Relaxed); let mut rs = rs; if rs.encode_node_index < -1 { let auto_gpu = settings @@ -5662,6 +5714,29 @@ impl Drop for ScreenCapture { if let Some(c) = &st.controls { c.stop.store(true, Ordering::Relaxed); } + // A Wayland capture is owned by this instance: release it, or a GC'd handle + // would leave the compositor encoding and delivering forever. + if st.backend == 2 && !crate::PY_SHUTDOWN.load(Ordering::Relaxed) { + let did = st.wl_display; + let owned = { + let mut owners = wayland_owners().lock().unwrap(); + if owners.get(&did) == Some(&self.id) { + owners.remove(&did); + true + } else { + false + } + }; + if owned { + if let Some(slot) = WAYLAND_BACKEND.get() { + if let Some(be) = slot.lock().unwrap().as_ref() { + Python::attach(|py| { + let _ = be.bind(py).borrow().stop_capture(did); + }); + } + } + } + } // Pair the cursor-monitor acquire from start_capture: a dropped // handle that never got stop_capture would pin the refcount and its // monitor thread forever. diff --git a/pixelflux/src/wayland/cursor.rs b/pixelflux/src/wayland/cursor.rs index e442603..15525e7 100644 --- a/pixelflux/src/wayland/cursor.rs +++ b/pixelflux/src/wayland/cursor.rs @@ -29,6 +29,9 @@ pub enum CursorJob { SetCallback(Py), /// Reload the worker's theme handle at a new pixel size; later `Named` jobs render at it. SetSize(i32), + /// Cap the longest delivered cursor edge; larger images are downscaled like the X11 + /// XFixes monitor does (parity: `CaptureSettings.cursor_size_cap` was X11-only). + SetSizeCap(i32), Named { name: &'static str }, Hide, /// wl_shm cursor sprite: raw pool bytes plus the sub-image descriptor. @@ -57,10 +60,11 @@ pub enum CursorJob { /// Spawn the cursor delivery worker; returns its job channel. The worker owns its own /// theme handle, the PNG cache, and the Python callback for the life of the process (like the /// compositor thread itself); `PY_SHUTDOWN` gates every Python call. -pub fn spawn_cursor_worker(cursor_size: i32) -> std::sync::mpsc::Sender { +pub fn spawn_cursor_worker(cursor_size: i32, size_cap: i32) -> std::sync::mpsc::Sender { let (tx, rx) = std::sync::mpsc::channel::(); let _ = std::thread::Builder::new().name("wl-cursor".into()).spawn(move || { let mut helper = Cursor::load(cursor_size); + let mut cap = size_cap; let mut cache: HashMap> = HashMap::new(); let mut callback: Option> = None; while let Ok(job) = rx.recv() { @@ -73,8 +77,15 @@ pub fn spawn_cursor_worker(cursor_size: i32) -> std::sync::mpsc::Sender { + cap = c; + continue; + } CursorJob::Named { name } => match helper.get_png_data(name) { - Some((png, x, y)) => ("png", png, x as i32, y as i32), + Some((png, x, y)) => { + let (png, x, y) = cap_cursor_png(png, x as i32, y as i32, cap); + ("png", png, x, y) + } None => ("error", Vec::new(), 0, 0), }, CursorJob::Hide => ("hide", Vec::new(), 0, 0), @@ -88,18 +99,22 @@ pub fn spawn_cursor_worker(cursor_size: i32) -> std::sync::mpsc::Sender { - let png = cached_png(&mut cache, hash, || { - encode_shm_cursor(width, height, stride, offset, opaque, &bytes) - }); - ("png", png, hot_x, hot_y) - } - CursorJob::Gles { hash, width, height, bytes, hot_x, hot_y } => { - let png = cached_png(&mut cache, hash, || { - encode_gles_cursor(width, height, &bytes) - }); - ("png", png, hot_x, hot_y) - } + } => capped_job( + &mut cache, + hash, + cap, + hot_x, + hot_y, + || encode_shm_cursor(width, height, stride, offset, opaque, &bytes), + ), + CursorJob::Gles { hash, width, height, bytes, hot_x, hot_y } => capped_job( + &mut cache, + hash, + cap, + hot_x, + hot_y, + || encode_gles_cursor(width, height, &bytes), + ), }; // A sprite whose pixels could not be read yields empty data; suppressing it // preserves the consumer's last cursor instead of blanking it (only an @@ -143,6 +158,61 @@ fn cached_png( png } +/// Downscale a cursor PNG so its longest edge is at most `cap` (<= 0 = uncapped), scaling +/// the hotspot with it. Mirrors the X11 XFixes monitor's `cursor_size_cap` handling so a +/// Wayland session ships cursor payloads no larger than an X11 one. +fn cap_cursor_png(png: Vec, hot_x: i32, hot_y: i32, cap: i32) -> (Vec, i32, i32) { + if cap <= 0 || png.is_empty() { + return (png, hot_x, hot_y); + } + let img = match image::load_from_memory(&png) { + Ok(i) => i.to_rgba8(), + Err(_) => return (png, hot_x, hot_y), + }; + let (w, h) = (img.width(), img.height()); + let longest = w.max(h); + if longest <= cap as u32 || longest == 0 { + return (png, hot_x, hot_y); + } + let scale = cap as f64 / longest as f64; + let nw = ((w as f64 * scale).round() as u32).max(1); + let nh = ((h as f64 * scale).round() as u32).max(1); + let resized = image::imageops::resize( + &img, + nw, + nh, + image::imageops::FilterType::Lanczos3, + ); + let mut out = Vec::new(); + if resized + .write_to(&mut IoCursor::new(&mut out), image::ImageFormat::Png) + .is_err() + { + return (png, hot_x, hot_y); + } + ( + out, + (hot_x as f64 * scale).round() as i32, + (hot_y as f64 * scale).round() as i32, + ) +} + +/// Cache wrapper for sprite jobs: encode (or reuse) the full-size PNG, then apply the +/// size cap. `cap_cursor_png` is idempotent for already-capped images, so cache hits +/// only pay a cheap dimension check. +fn capped_job( + cache: &mut HashMap>, + hash: u64, + cap: i32, + hot_x: i32, + hot_y: i32, + encode: impl FnOnce() -> Vec, +) -> (&'static str, Vec, i32, i32) { + let png = cached_png(cache, hash, encode); + let (png, sx, sy) = cap_cursor_png(png, hot_x, hot_y, cap); + ("png", png, sx, sy) +} + /// Convert a wl_shm BGRA/XRGB sprite sub-image to a straight-alpha PNG. Stride/offset are /// clamped non-negative with checked arithmetic so a garbage descriptor skips pixels instead of /// panicking; sprites larger than 128x128 are ignored (never a hardware cursor). diff --git a/pixelflux/src/wayland/frontend.rs b/pixelflux/src/wayland/frontend.rs index ac46388..f446e42 100644 --- a/pixelflux/src/wayland/frontend.rs +++ b/pixelflux/src/wayland/frontend.rs @@ -1253,6 +1253,8 @@ const CLIPBOARD_MIME_PREFERENCE: &[&str] = &[ "image/jpeg", "image/webp", "image/bmp", + "image/svg+xml", + "image/svg", "text/plain;charset=utf-8", "UTF8_STRING", "text/plain", From dab8ef768e01c5252bf176d563557f19f96f96a5 Mon Sep 17 00:00:00 2001 From: Seungmin Kim <8457324+ehfd@users.noreply.github.com> Date: Mon, 3 Aug 2026 17:11:54 +0900 Subject: [PATCH 13/21] fix: Address issues --- pixelflux/src/lib.rs | 9 ++++++++- pixelflux/src/x11/mod.rs | 9 ++++++++- 2 files changed, 16 insertions(+), 2 deletions(-) diff --git a/pixelflux/src/lib.rs b/pixelflux/src/lib.rs index 75c9011..38ba8aa 100644 --- a/pixelflux/src/lib.rs +++ b/pixelflux/src/lib.rs @@ -1524,7 +1524,14 @@ fn log_stream_settings( log_msg.push_str(" Streaming"); } - log_msg.push_str(&format!(" | CRF: {}", settings.video_crf)); + if settings.video_cbr_mode { + log_msg.push_str(&format!(" | CBR {}", settings.video_bitrate_kbps)); + } else { + log_msg.push_str(&format!(" | CRF: {}", settings.video_crf)); + if settings.video_bitrate_kbps > 0 { + log_msg.push_str(&format!(" | VBV: {} kbps", settings.video_bitrate_kbps)); + } + } if settings.use_paint_over_quality { log_msg.push_str(&format!( diff --git a/pixelflux/src/x11/mod.rs b/pixelflux/src/x11/mod.rs index 98370cf..da701ff 100644 --- a/pixelflux/src/x11/mod.rs +++ b/pixelflux/src/x11/mod.rs @@ -642,7 +642,14 @@ where if psettings.output_mode == 0 { log_msg.push_str(&format!(" | Mode: JPEG | Quality: {}", psettings.jpeg_quality)); } else { - log_msg.push_str(&format!(" | Mode: H264 | CRF: {}", psettings.video_crf)); + if psettings.video_cbr_mode { + log_msg.push_str(&format!(" | Mode: H264 CBR {}", psettings.video_bitrate_kbps)); + } else { + log_msg.push_str(&format!(" | Mode: H264 | CRF: {}", psettings.video_crf)); + if psettings.video_bitrate_kbps > 0 { + log_msg.push_str(&format!(" | VBV: {} kbps", psettings.video_bitrate_kbps)); + } + } if psettings.video_fullcolor { log_msg.push_str(" | Colorspace: I444 (Full Range)"); } else { From 68b849239edb3e301a183a73e985b52365fa5789 Mon Sep 17 00:00:00 2001 From: Seungmin Kim <8457324+ehfd@users.noreply.github.com> Date: Wed, 5 Aug 2026 00:32:01 +0900 Subject: [PATCH 14/21] feat: Make GPL optional --- README.md | 36 +++++++++++++++++++++++------- pixelflux/Cargo.toml | 10 ++++++++- pixelflux/src/encoders/software.rs | 30 +++++++++++++++++++++++++ pixelflux/src/lib.rs | 19 ++++++++++++++++ pixelflux/src/pipeline.rs | 36 ++++++++++++++++++++++++++---- setup.py | 30 ++++++++++++++++++++++++- 6 files changed, 147 insertions(+), 14 deletions(-) diff --git a/README.md b/README.md index 37f7f27..8657206 100644 --- a/README.md +++ b/README.md @@ -8,11 +8,11 @@ This module provides a Python interface to a high-performance capture library supporting both **X11** and **Wayland** environments. It captures pixel data, detects changes, and encodes modified stripes into JPEG or H.264. -It supports CPU-based encoding (x264, JPEG) as well as hardware-accelerated H.264 encoding via NVIDIA's NVENC and VA-API for Intel/AMD GPUs. Both backends share a **zero-copy pipeline** that minimizes copies and latency end to end. +It supports CPU-based encoding (x264 JPEG, or the BSD-licensed OpenH264) as well as hardware-accelerated H.264 encoding via NVIDIA's NVENC and VA-API for Intel/AMD GPUs. **About "zero copy":** the Wayland GPU path is truly zero-copy (dmabuf frames flow GBM → encoder without touching system RAM). The X11 path copies **exactly once**: the X server renders each frame into a shared-memory surface (`XShmGetImage`); the encoder threads then read that mapped surface **in place** and pass the encoded bytes to Python through the buffer protocol without any further copies. ## Installation -pixelflux is a single self-contained **Rust** extension (no C/C++ sources) compiled during installation. Both the X11 and Wayland backends, all encoders, and the Python API live in it. +pixelflux is a single self-contained **Rust** extension compiled during installation. Both the X11 and Wayland backends, all encoders, and the Python API live in it. (`libjpeg-turbo` and `openh264` C sources are vendored and built by their Rust `-sys` crates, so `cmake` and `nasm` are required, but no system copies of those libraries are used.) ### 1. Prerequisites @@ -34,7 +34,6 @@ sudo apt-get install -y \ libavcodec-dev \ libavutil-dev \ libx264-dev \ - libturbojpeg0-dev \ libgbm-dev \ libdrm-dev \ libwayland-dev \ @@ -43,7 +42,13 @@ sudo apt-get install -y \ libva-dev ``` -> **Notes:** the FFmpeg bindings (`ffmpeg-sys-next` 8.1) work with any system **FFmpeg 6.0–8.1** (only `avcodec`/`avfilter` are used, for `h264_vaapi`); on distros shipping an older FFmpeg, install a newer build and point `PKG_CONFIG_PATH` at it. X11 capture uses pure-Rust XCB (no `libX11`/`libxcb`/`Xfixes` dev packages needed); colorspace conversion is pure-Rust and the NVENC/CUDA libraries are loaded at runtime (no compile-time NVIDIA packages). +> **Notes:** the FFmpeg bindings (`ffmpeg-sys-next` 8.1) work with any system **FFmpeg 6.0–8.1 built LGPL-only** (only `avcodec`/`avfilter` are used, for `h264_vaapi`; `--enable-gpl` is deliberately never passed to upstream FFmpeg builds); on distros shipping an older FFmpeg, install a newer build and point `PKG_CONFIG_PATH` at it. `libjpeg-turbo` is vendored and built statically by its crate — **no `libturbojpeg` system package is needed** (only `cmake` + `nasm`). X11 capture uses pure-Rust XCB; colorspace conversion is pure-Rust and the NVENC/CUDA libraries are loaded at runtime (no compile-time NVIDIA packages). +> +> **GPL component (`libx264`):** striped software H.264 uses the system `libx264` (GPL-2.0+), which is the only GPL-licensed dependency **of pixelflux itself**. It is enabled **by default**; to build without it, set `PIXELFLUX_ENABLE_GPL=0` (or `=false`) before `pip install`. You then lose the striped/4:4:4 software H.264 path (the default CPU H.264 path) and `libx264-dev` is no longer required; **kept** are all JPEG modes, the BSD-licensed OpenH264 full-frame software H.264 (`use_openh264 = True`), NVENC, and VA-API. A notice is printed at install time whether GPL components are enabled or not. +> +> **Caveat (transitively-linked x264):** the extension links the *system* FFmpeg (`libavcodec`/`libavfilter`) for VA-API, and many distro FFmpeg builds (e.g. Ubuntu/Debian's) are themselves compiled with `--enable-libx264`, so their `libavcodec` drags `libx264` in as a transitive shared-library dependency even when pixelflux was built GPL-free. pixelflux contains no x264 code in that case (verified: no `x264` symbols or `NEEDED` entries), for a deployment that must be x264-free end to end, use an FFmpeg built without `--enable-libx264` (the project's wheel CI builds FFmpeg n8.1 LGPL-only, and the project's AppImage bundles such a variant). +> +> **Official wheels are always GPL-enabled** (x264 enabled, LGPL-only FFmpeg); the `PIXELFLUX_ENABLE_GPL=0` path is for verified license-minimal source builds. The AppImage distribution bundles the LGPL-only FFmpeg variant so the optional GPL-free posture holds end-to-end. ### 2. Hardware Acceleration (Optional but Recommended) * **NVIDIA (NVENC):** The library detects the NVIDIA driver at runtime. No extra compile-time packages are needed. @@ -56,6 +61,8 @@ sudo apt-get install -y \ pip install pixelflux ``` +Prebuilt wheels are published on the GitHub Releases page (`manylinux_2_28` and `musllinux`, x86_64 and aarch64, CPython 3.9–3.14). PyPI serves them automatically on supported platforms; on other platforms pip builds from source with the prerequisites above. + **Option B: Install from local source** ```bash # From the root of the project repository @@ -113,6 +120,7 @@ settings.capture_y = 0 settings.capture_cursor = True settings.target_fps = 60.0 settings.scale = 1.0 # Fractional scaling (Wayland only) +settings.wayland_host_display = "" # Capture from an EXTERNAL wlroots compositor instead of the built-in one (host-capture mode) # --- Encoding Mode --- # 0 for JPEG, 1 for H.264 @@ -134,6 +142,8 @@ settings.video_paintover_burst_frames = 5 # Number of high-quality fram settings.video_fullcolor = False # Use I444/full color (High 4:4:4) instead of I420. Supported by software encoding and NVENC. settings.video_fullframe = True # Encode full frames (required for HW accel) instead of just changed stripes settings.video_streaming_mode = False # Bypass all VNC logic and work like a normal video encoder, higher constant CPU usage for fullscreen gaming/videos +settings.use_openh264 = False # Use Cisco OpenH264 (BSD-licensed) for software full-frame H.264 instead of x264 (GPL). Required for software H.264 when built with PIXELFLUX_ENABLE_GPL=0 +settings.keyframe_interval_s = 0.0 # Periodic keyframe interval in seconds (0 = keyframes only on demand/paint-over) settings.video_cbr_mode = False # Switches to CBR mode and ignores CRF value. Used in conjunction with video_bitrate_kbps. settings.video_bitrate_kbps = 4000 # Target bitrate for CBR mode. Required when video_cbr_mode is enabled. settings.video_vbv_multiplier = 1.5 # Optional CBR VBV size as a multiple of one frame's bit budget (0 = auto: 1.5, or 3 with periodic keyframes). @@ -157,6 +167,8 @@ settings.render_node_path = None settings.omit_stripe_headers = False # --- Change Detection & Optimization --- +settings.video_min_qp = 0 # CBR QP clamps: 0 = encoder default; max bounds the quality floor, min bounds bit waste on easy content +settings.video_max_qp = 0 settings.use_paint_over_quality = True # Enable paint-over/IDR requests for static regions settings.paint_over_trigger_frames = 15 # Frames of no motion to trigger paint-over settings.damage_block_threshold = 10 # Consecutive changes to trigger "damaged" state @@ -165,6 +177,7 @@ settings.damage_block_duration = 30 # Frames a stripe stays "damaged" # --- Watermarking --- # Must be a bytes object. The path to your PNG image. settings.watermark_path = b"/path/to/your/watermark.png" +settings.cursor_size_cap = 0 # Cap out-of-band hardware-cursor PNGs to this longest edge (<= 0 = uncapped) # 0:None, 1:TopLeft, 2:TopRight, 3:BottomLeft, 4:BottomRight, 5:Middle, 6:Animated settings.watermark_location_enum = 4 ``` @@ -239,6 +252,10 @@ The Wayland backend implements a **Zero-Copy** architecture for hardware encodin **Performance Note:** Software (Pixman) rendering, the absence of a hardware encoder, or utilizing a render node different from the encoding node will force a "Readback" fallback, copying pixels to the CPU and breaking the zero-copy chain (higher latency and CPU load). A watermark does **not** force readback — on the GPU path it is composited into the frame before encoding. +## Built-in MP4 Recorder + +For convenience, the extension ships its own fragmented-MP4 muxer (no `avformat` dependency) with the `start_recording(...)`, `stop_recording()`, and `recording_status()` Python functions, controllable through the `PIXELFLUX_RECORD*` environment variables. Recording taps the encoded full-frame H.264 stream, and HTTP endpoints allow remote trigger/stop/status. + ## Recording Sink The capture session can output the raw H.264 video stream directly to a Unix domain socket for external recording. @@ -260,9 +277,9 @@ ffmpeg -f h264 -i unix:///tmp/pixelflux_record -c:v copy test.h264 ffmpeg -f h264 -framerate 60 -i unix:///tmp/pixelflux_record -c:v libx264 -preset fast -crf 23 -pix_fmt yuv420p test.mp4 ``` -## Computer Use Interface (Wayland) +## Computer Use Interface (X11 and Wayland) -The Wayland backend implements the [Anthropic Computer Use specification](https://github.com/anthropics/claude-quickstarts/tree/main/computer-use-demo), providing an HTTP API for AI agents to control the desktop. Enable it by setting the `PIXELFLUX_CU` environment variable to the port the server should listen on: +Both backends implement the [Anthropic Computer Use specification](https://github.com/anthropics/claude-quickstarts/tree/main/computer-use-demo), providing an HTTP API for AI agents to control the desktop. On Wayland the compositor injects input natively; on X11 the existing display is driven through XTEST with root-window screenshots. Enable it by setting the `PIXELFLUX_CU` environment variable to the port the server should listen on: ```bash export PIXELFLUX_CU=5000 @@ -437,7 +454,7 @@ install at build or runtime beyond the driver. * **X11:** XShm capture via pure-Rust XCB, with XFixes cursor and watermark compositing. * **Wayland:** Modern, secure, headless compositor based on [Smithay](https://github.com/Smithay/smithay). * **Flexible Encoding:** - * **Software:** x264 (H.264, incl. 4:4:4) and JPEG with multi-threaded striping. + * **Software:** x264 (H.264, incl. 4:4:4 — GPL, toggleable) and JPEG with multi-threaded striping, plus BSD-licensed OpenH264 full-frame software H.264. * **Hardware:** NVIDIA NVENC (incl. High 4:4:4, ARGB-direct BT.709, multi-GPU containers, API-version negotiation) and VA-API (Intel/AMD, VA-VPP convert) with Zero-Copy support. * **Driver-aware GPU auto-selection** via the `auto_gpu` setting. * **Zero-Copy Frames (X11 & Wayland):** the native frame object (buffer protocol) hands the encoded buffer to Python with no copy, on every supported Python version (3.9–3.14). @@ -446,13 +463,16 @@ install at build or runtime beyond the driver. * **Paint-Over:** Automatically improves quality for static regions. * **Damage Throttling:** Limits processing during high-motion scenes. * **On-demand keyframes:** `request_idr_frame()` forces an IDR for reconnecting clients. -* **Input Handling:** Built-in input injection for mouse and keyboard (Wayland). +* **Input Handling:** Built-in input injection for mouse and keyboard (Wayland; XTEST on X11 via Computer Use). * **Cursor Compositing:** Hardware cursor planes or software rendering options. * **Dynamic Watermarking:** Overlay PNGs with static positioning or DVD-screensaver style animation. * **Recording Sink:** Direct Unix socket output of full-frame H.264 streams for local capture. +* **Built-in MP4 Recorder:** Crash-safe fragmented-MP4 recording without any FFmpeg `avformat` dependency. * **AI Agent Control:** Computer Use API to dump screenshots and drive all facets of a desktop environment. ## License This project is licensed under the **Mozilla Public License Version 2.0**. A copy of the MPL 2.0 can be found at https://mozilla.org/MPL/2.0/. + +Note that the default build links the GPL-2.0+ `libx264` for striped software H.264; build with `PIXELFLUX_ENABLE_GPL=0` to exclude every GPL-licensed component (the FFmpeg bindings are used LGPL-only, and `openh264` is BSD-licensed). diff --git a/pixelflux/Cargo.toml b/pixelflux/Cargo.toml index 76fac70..9af43ec 100644 --- a/pixelflux/Cargo.toml +++ b/pixelflux/Cargo.toml @@ -19,7 +19,9 @@ crossbeam-channel = "0.5" calloop = "0.14" turbojpeg = "1.3" rayon = "1.10" -x264-sys = "0.2.2" +# Striped software H.264 (GPL-2.0+ libx264). Optional, enabled by the default `gpl` +# feature; build with `--no-default-features` (or PIXELFLUX_ENABLE_GPL=0) to exclude it. +x264-sys = { version = "0.2.2", optional = true } # BSD-licensed Cisco OpenH264 as an alternative software H.264 encoder; builds the # vendored C source (needs a C toolchain + nasm), no runtime download. openh264-sys2 # is used for the raw set_option live-bitrate path the safe wrapper doesn't expose. @@ -49,11 +51,17 @@ base64 = "0.22" serde = { version = "1", features = ["derive"] } serde_json = "1" tiny_http = "0.12" +cfg-if = "1" [dependencies.nvcodec-sys] path = "nvcodec-sys" features = ["cuda"] +[features] +default = ["gpl"] +## GPL-licensed components (GPL-2.0+ libx264 for striped software H.264). +gpl = ["dep:x264-sys"] + [dependencies.smithay] git = "https://github.com/Smithay/smithay" rev = "ca932e042fa9ad150605c150a86275b85f9ad5b3" diff --git a/pixelflux/src/encoders/software.rs b/pixelflux/src/encoders/software.rs index f25afd6..550b65b 100644 --- a/pixelflux/src/encoders/software.rs +++ b/pixelflux/src/encoders/software.rs @@ -14,7 +14,9 @@ use crate::RustCaptureSettings; use rayon::prelude::*; use smithay::utils::{Physical, Rectangle}; +#[cfg(feature = "gpl")] use std::ffi::CString; +#[cfg(feature = "gpl")] use std::ptr; use std::sync::Arc; use yuv::{BufferStoreMut, YuvConversionMode, YuvPlanarImageMut, YuvRange, YuvStandardMatrix}; @@ -152,6 +154,7 @@ thread_local! { /// state and corrupt the heap. The lock is deliberately held only around open and close, never /// around `x264_encoder_encode`, so serializing setup costs nothing in the hot per-stripe encode /// path where the real parallelism lives. +#[cfg(feature = "gpl")] static X264_OPEN_CLOSE_LOCK: std::sync::Mutex<()> = std::sync::Mutex::new(()); /// One long-lived libx264 session for a stripe, holding the raw `x264_t` handle alongside a @@ -165,6 +168,7 @@ static X264_OPEN_CLOSE_LOCK: std::sync::Mutex<()> = std::sync::Mutex::new(()); /// chosen at open and gates which of the live reconfigures apply. The manual `Send` impl exists only /// because a raw pointer is not `Send` by default and the handle must move onto the rayon stripe /// workers; `Drop` closes it under the global open/close lock for the same reason that lock exists. +#[cfg(feature = "gpl")] pub struct H264EncoderWrapper { encoder: *mut x264_sys::x264_t, pub width: i32, @@ -179,8 +183,10 @@ pub struct H264EncoderWrapper { full_range: bool, } +#[cfg(feature = "gpl")] unsafe impl Send for H264EncoderWrapper {} +#[cfg(feature = "gpl")] impl Drop for H264EncoderWrapper { fn drop(&mut self) { if !self.encoder.is_null() { @@ -191,6 +197,7 @@ impl Drop for H264EncoderWrapper { } } +#[cfg(feature = "gpl")] impl H264EncoderWrapper { /// Open an x264 encoder tuned for real-time screen streaming, or `None` on failure. /// @@ -491,10 +498,14 @@ impl H264EncoderWrapper { pub struct StripeState { pub no_motion_frame_count: u32, pub paint_over_sent: bool, + #[cfg(feature = "gpl")] pub h264_encoder: Option, pub h264_burst_frames_remaining: i32, + #[cfg(feature = "gpl")] pub y_buf: Vec, + #[cfg(feature = "gpl")] pub u_buf: Vec, + #[cfg(feature = "gpl")] pub v_buf: Vec, pub packet_buf: Vec, pub last_hash: u64, @@ -677,6 +688,7 @@ pub struct EncodedStripe { /// conversion band each, since the parallelism there already comes from encoding the stripes /// concurrently. #[allow(clippy::too_many_arguments)] +#[cfg_attr(not(feature = "gpl"), allow(unused_assignments, unused_mut, unused_variables))] pub fn encode_cpu( stripes: &mut Vec, raw_pixels: &[u8], @@ -786,6 +798,7 @@ pub fn encode_cpu( let video_crf = settings.video_crf; let video_po_crf = settings.video_paintover_crf; let video_burst = settings.video_paintover_burst_frames; + #[cfg(feature = "gpl")] let video_fullcolor = settings.video_fullcolor; let video_streaming = settings.video_streaming_mode; let jpeg_q = settings.jpeg_quality; @@ -793,12 +806,16 @@ pub fn encode_cpu( let trigger_frames = settings.paint_over_trigger_frames; let use_paint_over = settings.use_paint_over_quality; let burst_crf = if use_paint_over && video_po_crf < video_crf { video_po_crf } else { video_crf }; + #[cfg(feature = "gpl")] let target_fps = settings.target_fps; let omit_headers = settings.omit_stripe_headers; let damage_block_threshold = settings.damage_block_threshold; let damage_block_duration = settings.damage_block_duration as i32; + #[cfg(feature = "gpl")] let video_cbr = settings.video_cbr_mode; + #[cfg(feature = "gpl")] let video_bitrate = settings.video_bitrate_kbps; + #[cfg(feature = "gpl")] let video_vbv = (crate::encoders::vbv_bits( (video_bitrate.max(0) as u32).saturating_mul(1000), target_fps, @@ -809,11 +826,13 @@ pub fn encode_cpu( // Full-frame x264 threads follow the same policy as the OpenH264 encoder: // one fewer than the cores (headroom for the capture thread), clamped to // [1, 4] to match the four-slice ceiling below. + #[cfg(feature = "gpl")] let h264_threads = if n_processing_stripes == 1 { num_cores.saturating_sub(1).clamp(1, 4) as i32 } else { 1 }; + #[cfg(feature = "gpl")] let csc_bands = 1; let stripe_body = |(i, stripe_state): (usize, &mut StripeState)| -> Option { @@ -938,6 +957,8 @@ pub fn encode_cpu( }) }) } else { + cfg_if::cfg_if! { + if #[cfg(feature = "gpl")] { let needs_reinit = if let Some(ref enc) = stripe_state.h264_encoder { enc.width != width_usize as i32 || enc.height != actual_height as i32 @@ -1035,6 +1056,15 @@ pub fn encode_cpu( } else { None } + } else { + static GPL_OFF_LOGGED: std::sync::atomic::AtomicBool = + std::sync::atomic::AtomicBool::new(false); + if !GPL_OFF_LOGGED.swap(true, std::sync::atomic::Ordering::Relaxed) { + eprintln!("[software] software H.264 encoding unavailable: pixelflux was built without GPL components (libx264 disabled). Use JPEG, OpenH264 (`use_openh264 = true`), or a hardware encoder."); + } + None + } + } } } else { None diff --git a/pixelflux/src/lib.rs b/pixelflux/src/lib.rs index 38ba8aa..5ac3683 100644 --- a/pixelflux/src/lib.rs +++ b/pixelflux/src/lib.rs @@ -1130,12 +1130,23 @@ fn build_readback_encoders( return (None, Some(e)); } None => { + #[cfg(feature = "gpl")] eprintln!("[Wayland] OpenH264 init failed; falling back to x264 software."); + #[cfg(not(feature = "gpl"))] + eprintln!("[Wayland] OpenH264 init failed; no software H.264 encoder is available in this GPL-free build."); return (None, None); } } } if !try_gpu { + #[cfg(not(feature = "gpl"))] + if settings.output_mode == 1 && !settings.use_openh264 { + println!("[Wayland] pixelflux was built without GPL components (libx264 disabled); substituting the BSD-licensed OpenH264 software encoder."); + if let Some(e) = crate::encoders::oh264::Openh264Encoder::new(settings) { + return (None, Some(e)); + } + eprintln!("[Wayland] OpenH264 init failed; no software H.264 encoder is available in this GPL-free build."); + } return (None, None); } let encode_driver = get_gpu_driver(settings.encode_node_index.max(0)); @@ -1177,6 +1188,14 @@ fn build_readback_encoders( } } } + #[cfg(not(feature = "gpl"))] + if settings.output_mode == 1 && !settings.use_openh264 { + println!("[Wayland] pixelflux was built without GPL components (libx264 disabled); substituting the BSD-licensed OpenH264 software encoder."); + if let Some(e) = crate::encoders::oh264::Openh264Encoder::new(settings) { + return (None, Some(e)); + } + eprintln!("[Wayland] OpenH264 init failed; no software H.264 encoder is available in this GPL-free build."); + } (None, None) } diff --git a/pixelflux/src/pipeline.rs b/pixelflux/src/pipeline.rs index 26afaee..4c4960f 100644 --- a/pixelflux/src/pipeline.rs +++ b/pixelflux/src/pipeline.rs @@ -155,6 +155,31 @@ enum X11Encoder { Openh264(Openh264Encoder), } +/// Software H.264 encoder used when hardware encoders are unavailable or the CPU is forced. +/// With GPL components this is the striped x264 path (its state lives per-stripe inside +/// `encode_cpu`, so no persistent object exists here); without them (the crate built with the +/// `gpl` feature disabled) the BSD-licensed OpenH264 full-frame encoder is substituted instead. +#[cfg(feature = "gpl")] +fn software_h264_encoder(settings: &RustCaptureSettings) -> X11Encoder { + let _ = settings; + X11Encoder::None +} + +#[cfg(not(feature = "gpl"))] +fn software_h264_encoder(settings: &RustCaptureSettings) -> X11Encoder { + if settings.output_mode != 1 { + return X11Encoder::None; + } + println!("[x11] pixelflux was built without GPL components (libx264 disabled); substituting the BSD-licensed OpenH264 software encoder."); + match Openh264Encoder::new(settings) { + Some(e) => X11Encoder::Openh264(e), + None => { + eprintln!("[x11] OpenH264 init failed; no software H.264 encoder is available in this GPL-free build."); + X11Encoder::None + } + } +} + /// Everything the X11 host-ARGB path has to remember between frames. /// /// Unlike the Wayland backend, X11 capture has no compositor to report what changed, so this @@ -194,7 +219,10 @@ impl X11Pipeline { match Openh264Encoder::new(&settings) { Some(e) => X11Encoder::Openh264(e), None => { + #[cfg(feature = "gpl")] eprintln!("[x11] OpenH264 init failed; falling back to software x264"); + #[cfg(not(feature = "gpl"))] + eprintln!("[x11] OpenH264 init failed; no software H.264 encoder is available in this GPL-free build."); X11Encoder::None } } @@ -205,7 +233,7 @@ impl X11Pipeline { if !encode_driver.is_empty() && !encode_driver.contains("nvidia") { if settings.video_fullcolor { println!("[x11] 4:4:4 full-color requested. VAAPI does not support this profile reliably. Falling back to CPU."); - X11Encoder::None + software_h264_encoder(&settings) } else { println!("[x11] Initializing Unified VAAPI Encoder..."); match VaapiEncoder::new_host(&settings) { @@ -215,7 +243,7 @@ impl X11Pipeline { } Err(err) => { eprintln!("[x11] Failed to init VAAPI: {err}. Falling back to CPU."); - X11Encoder::None + software_h264_encoder(&settings) } } } @@ -228,7 +256,7 @@ impl X11Pipeline { } Err(err) => { eprintln!("[x11] Failed to init NVENC: {err}. Falling back to CPU."); - X11Encoder::None + software_h264_encoder(&settings) } } } @@ -236,7 +264,7 @@ impl X11Pipeline { if settings.output_mode == 1 { println!("[x11] No GPU Encoder available -> Using CPU Software Encoding."); } - X11Encoder::None + software_h264_encoder(&settings) }; Self { settings, diff --git a/setup.py b/setup.py index a18493b..916e1c1 100644 --- a/setup.py +++ b/setup.py @@ -2,12 +2,39 @@ # License, v. 2.0. If a copy of the MPL was not distributed with this # file, You can obtain one at https://mozilla.org/MPL/2.0/. +import os +import sys + from setuptools import setup from setuptools_rust import Binding, RustExtension, Strip with open("README.md", "r", encoding="utf-8") as fh: long_description = fh.read() +# GPL components (GPL-2.0+ libx264, used for striped software H.264) are enabled by +# default. Set PIXELFLUX_ENABLE_GPL=0 (or "false"/"no") to build without them; the +# BSD-licensed OpenH264 encoder is then substituted for software H.264. +_enable_gpl = os.environ.get("PIXELFLUX_ENABLE_GPL", "1").strip().lower() not in ( + "0", + "false", + "no", + "off", +) +if _enable_gpl: + print( + "NOTICE: pixelflux is being built WITH GPL-licensed components " + "(GPL-2.0+ libx264 for striped software H.264), which is the default. " + "Set PIXELFLUX_ENABLE_GPL=0 to exclude every GPL-licensed component.", + file=sys.stderr, + ) +else: + print( + "NOTICE: pixelflux is being built WITHOUT GPL-licensed components " + "(PIXELFLUX_ENABLE_GPL=0): libx264 is excluded and OpenH264 (BSD) " + "substitutes for software H.264.", + file=sys.stderr, + ) + setup( name="pixelflux", version="2.0.0", @@ -29,7 +56,8 @@ "pixelflux/Cargo.toml", binding=Binding.PyO3, debug=False, - strip=Strip.All + strip=Strip.All, + args=([] if _enable_gpl else ["--no-default-features"]), ) ], From 4637025b3a4b76614eb3d80f8784f5279c76018f Mon Sep 17 00:00:00 2001 From: Seungmin Kim <8457324+ehfd@users.noreply.github.com> Date: Wed, 5 Aug 2026 03:08:08 +0900 Subject: [PATCH 15/21] fix: Add Cargo.lock --- .gitignore | 1 - pixelflux/Cargo.lock | 2587 ++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 2587 insertions(+), 1 deletion(-) create mode 100644 pixelflux/Cargo.lock diff --git a/.gitignore b/.gitignore index 5e02635..f71a8a8 100644 --- a/.gitignore +++ b/.gitignore @@ -76,4 +76,3 @@ cython_debug/ .ruff_cache/ .pypirc codeto* -Cargo.lock diff --git a/pixelflux/Cargo.lock b/pixelflux/Cargo.lock new file mode 100644 index 0000000..598b15d --- /dev/null +++ b/pixelflux/Cargo.lock @@ -0,0 +1,2587 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 4 + +[[package]] +name = "adler2" +version = "2.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "320119579fcad9c21884f5c4861d16174d0e06250625266f50fe6898340abefa" + +[[package]] +name = "aho-corasick" +version = "1.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c982642fa9e8606056828ee9a8505737230110bb1099153c79efe865c59d12ba" +dependencies = [ + "memchr", +] + +[[package]] +name = "aligned" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ee4508988c62edf04abd8d92897fca0c2995d907ce1dfeaf369dac3716a40685" +dependencies = [ + "as-slice", +] + +[[package]] +name = "aligned-vec" +version = "0.6.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dc890384c8602f339876ded803c97ad529f3842aba97f6392b3dba0dd171769b" +dependencies = [ + "equator", +] + +[[package]] +name = "anyhow" +version = "1.0.104" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "330a5ed07fa54e4702c9d6c4174f74427fc0ef6e214bbd677ae50a5099946470" + +[[package]] +name = "appendlist" +version = "1.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e149dc73cd30538307e7ffa2acd3d2221148eaeed4871f246657b1c3eaa1cbd2" + +[[package]] +name = "approx" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f2a05fd1bd10b2527e20a2cd32d8873d115b8b39fe219ee25f42a8aca6ba278" +dependencies = [ + "num-traits", +] + +[[package]] +name = "arbitrary" +version = "1.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c3d036a3c4ab069c7b410a2ce876bd74808d2d0888a82667669f8e783a898bf1" + +[[package]] +name = "arg_enum_proc_macro" +version = "0.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0ae92a5119aa49cdbcf6b9f893fe4e1d98b04ccbf82ee0584ad948a44a734dea" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "arrayvec" +version = "0.7.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d3fb67a6e08acf24fdeccbac2cb6ac4305825bd1f117462e0e6f2f193345ad56" + +[[package]] +name = "as-slice" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "516b6b4f0e40d50dcda9365d53964ec74560ad4284da2e7fc97122cd83174516" +dependencies = [ + "stable_deref_trait", +] + +[[package]] +name = "ascii" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d92bec98840b8f03a5ff5413de5293bfcd8bf96467cf5452609f939ec6f5de16" + +[[package]] +name = "atomic_float" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "628d228f918ac3b82fe590352cc719d30664a0c13ca3a60266fe02c7132d480a" + +[[package]] +name = "autocfg" +version = "1.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f2032f911046de80f0a198e0901378627c33f59ea0ac00e363d481118bd70a53" + +[[package]] +name = "av-scenechange" +version = "0.14.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0f321d77c20e19b92c39e7471cf986812cbb46659d2af674adc4331ef3f18394" +dependencies = [ + "aligned", + "anyhow", + "arg_enum_proc_macro", + "arrayvec", + "log", + "num-rational", + "num-traits", + "pastey", + "rayon", + "thiserror 2.0.19", + "v_frame", + "y4m", +] + +[[package]] +name = "av1-grain" +version = "0.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8cfddb07216410377231960af4fcab838eaa12e013417781b78bd95ee22077f8" +dependencies = [ + "anyhow", + "arrayvec", + "log", + "nom 8.0.0", + "num-rational", + "v_frame", +] + +[[package]] +name = "avif-serialize" +version = "0.8.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e7178fe5f7d460b13895ebb9dcb28a3a6216d2df2574a0806cb51b555d297f38" +dependencies = [ + "arrayvec", +] + +[[package]] +name = "base64" +version = "0.22.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6" + +[[package]] +name = "bindgen" +version = "0.72.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "993776b509cfb49c750f11b8f07a46fa23e0a1386ffc01fb1e7d343efc387895" +dependencies = [ + "bitflags", + "cexpr", + "clang-sys", + "itertools 0.13.0", + "log", + "prettyplease", + "proc-macro2", + "quote", + "regex", + "rustc-hash", + "shlex 1.3.0", + "syn 2.0.119", +] + +[[package]] +name = "bit_field" +version = "0.10.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e4b40c7323adcfc0a41c4b88143ed58346ff65a288fc144329c5c45e05d70c6" + +[[package]] +name = "bitflags" +version = "2.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b588b76d00fde79687d7646a9b5bdf3cc0f655e0bbd080335a95d7e96f3587da" + +[[package]] +name = "bitstream-io" +version = "4.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7eff00be299a18769011411c9def0d827e8f2d7bf0c3dbf53633147a8867fd1f" +dependencies = [ + "no_std_io2", +] + +[[package]] +name = "block-buffer" +version = "0.10.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3078c7629b62d3f0439517fa394996acacc5cbc91c5a20d8c658e77abd503a71" +dependencies = [ + "generic-array", +] + +[[package]] +name = "built" +version = "0.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c0e531d93d39c34eef561e929e8a7f86d77a5af08aac4f6d6e39976c51858e9" + +[[package]] +name = "bumpalo" +version = "3.20.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72f5acc6cb2ba439de613abc23857ec3d78374d8ed5ac84e9d11336e87da8649" + +[[package]] +name = "bytemuck" +version = "1.25.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "95832e849adfb21180ccb6826a99da14e5d266ae5c2e668e1602cf234f153797" +dependencies = [ + "bytemuck_derive", +] + +[[package]] +name = "bytemuck_derive" +version = "1.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f65693059b6b9c588b9f62fed1cedbf0a8b805631457ea162d68f0de186f3de5" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "byteorder-lite" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f1fe948ff07f4bd06c30984e69f5b4899c516a3ef74f34df92a2df2ab535495" + +[[package]] +name = "calloop" +version = "0.14.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4dbf9978365bac10f54d1d4b04f7ce4427e51f71d61f2fe15e3fed5166474df7" +dependencies = [ + "bitflags", + "polling", + "rustix 1.1.4", + "slab", + "tracing", +] + +[[package]] +name = "cc" +version = "1.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5add81bb678e6cb321aff7fa0dc7689ad82b112dbc032cea19f91d6b8e3582b9" +dependencies = [ + "find-msvc-tools", + "jobserver", + "libc", + "shlex 2.0.1", +] + +[[package]] +name = "cexpr" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6fac387a98bb7c37292057cffc56d62ecb629900026402633ae9160df93a8766" +dependencies = [ + "nom 7.1.3", +] + +[[package]] +name = "cfg-expr" +version = "0.20.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fb693542bcafa528e198be0ebd9d3632ca5b7c93dbe7237460e199910835997c" +dependencies = [ + "smallvec", + "target-lexicon", +] + +[[package]] +name = "cfg-if" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9330f8b2ff13f34540b44e946ef35111825727b38d33286ef986142615121801" + +[[package]] +name = "cgmath" +version = "0.18.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1a98d30140e3296250832bbaaff83b27dcd6fa3cc70fb6f1f3e5c9c0023b5317" +dependencies = [ + "approx", + "num-traits", +] + +[[package]] +name = "chunked_transfer" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6e4de3bc4ea267985becf712dc6d9eed8b04c953b3fcfb339ebc87acd9804901" + +[[package]] +name = "clang-sys" +version = "1.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "157a8ba7b480713b56f4c09fd13fc3e0a22a5dfab8097ba61cbc5feef950788a" +dependencies = [ + "glob", + "libc", + "libloading 0.8.9", +] + +[[package]] +name = "cmake" +version = "0.1.58" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c0f78a02292a74a88ac736019ab962ece0bc380e3f977bf72e376c5d78ff0678" +dependencies = [ + "cc", +] + +[[package]] +name = "color_quant" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3d7b894f5411737b7867f4827955924d7c254fc9f4d91a6aad6b097804b1018b" + +[[package]] +name = "concurrent-queue" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4ca0197aee26d1ae37445ee532fefce43251d24cc7c166799f4d46817f1d3973" +dependencies = [ + "crossbeam-utils", +] + +[[package]] +name = "cpufeatures" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "59ed5838eebb26a2bb2e58f6d5b5316989ae9d08bab10e0e6d103e656d1b0280" +dependencies = [ + "libc", +] + +[[package]] +name = "crc32fast" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9481c1c90cbf2ac953f07c8d4a58aa3945c425b7185c9154d67a65e4230da511" +dependencies = [ + "cfg-if", +] + +[[package]] +name = "crossbeam-channel" +version = "0.5.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d85363c37faeca707aef026efa9f3b34d077bce547e48f770770625c6013679e" +dependencies = [ + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-deque" +version = "0.8.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5181e0de7b61eb03a81e347d6dd8797bae9da5146707b51077e2d71a54ec0ceb" +dependencies = [ + "crossbeam-epoch", + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-epoch" +version = "0.9.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2d6914041f254d6e9176c01941b21115dcfb7089e55135a35411081bd106ef3f" +dependencies = [ + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-utils" +version = "0.8.22" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "61803da095bee82a81bb1a452ecc25d3b2f1416d1897eb86430c6159ef717c17" + +[[package]] +name = "crunchy" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "460fbee9c2c2f33933d720630a6a0bac33ba7053db5344fac858d4b8952d77d5" + +[[package]] +name = "crypto-common" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "78c8292055d1c1df0cce5d180393dc8cce0abec0a7102adb6c7b1eef6016d60a" +dependencies = [ + "generic-array", + "typenum", +] + +[[package]] +name = "cursor-icon" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f27ae1dd37df86211c42e150270f82743308803d90a6f6e6651cd730d5e1732f" + +[[package]] +name = "digest" +version = "0.10.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292" +dependencies = [ + "block-buffer", + "crypto-common", +] + +[[package]] +name = "dlib" +version = "0.5.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ab8ecd87370524b461f8557c119c405552c396ed91fc0a8eec68679eab26f94a" +dependencies = [ + "libloading 0.8.9", +] + +[[package]] +name = "downcast-rs" +version = "1.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "75b325c5dbd37f80359721ad39aca5a29fb04c89279657cffdda8736d0c0b9d2" + +[[package]] +name = "drm" +version = "0.14.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "80bc8c5c6c2941f70a55c15f8d9f00f9710ebda3ffda98075f996a0e6c92756f" +dependencies = [ + "bitflags", + "bytemuck", + "drm-ffi", + "drm-fourcc", + "libc", + "rustix 0.38.44", +] + +[[package]] +name = "drm-ffi" +version = "0.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "51a91c9b32ac4e8105dec255e849e0d66e27d7c34d184364fb93e469db08f690" +dependencies = [ + "drm-sys", + "rustix 1.1.4", +] + +[[package]] +name = "drm-fourcc" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0aafbcdb8afc29c1a7ee5fbe53b5d62f4565b35a042a662ca9fecd0b54dae6f4" + +[[package]] +name = "drm-sys" +version = "0.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ecc8e1361066d91f5ffccff060a3c3be9c3ecde15be2959c1937595f7a82a9f8" +dependencies = [ + "libc", + "linux-raw-sys 0.9.4", +] + +[[package]] +name = "either" +version = "1.17.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9e5e8f6c15a24b9a3ee5efec809ccd006d3b30e8b3bb63c39af737c7f87daa1d" + +[[package]] +name = "equator" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4711b213838dfee0117e3be6ac926007d7f433d7bbe33595975d4190cb07e6fc" +dependencies = [ + "equator-macro", +] + +[[package]] +name = "equator-macro" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "44f23cf4b44bfce11a86ace86f8a73ffdec849c9fd00a386a53d278bd9e81fb3" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "equivalent" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "877a4ace8713b0bcf2a4e7eec82529c029f1d0619886d18145fea96c3ffe5c0f" + +[[package]] +name = "errno" +version = "0.3.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb" +dependencies = [ + "libc", + "windows-sys 0.61.2", +] + +[[package]] +name = "exr" +version = "1.74.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "711fe42c9964295e01ee3fba3f9fe0e1d24b98886950d68efe81b1c76e21adf3" +dependencies = [ + "bit_field", + "half", + "lebe", + "miniz_oxide", + "num-complex", + "pulp", + "rayon-core", + "smallvec", + "zune-inflate", +] + +[[package]] +name = "fastrand" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "da7c62ceae207dd37ea5b845da6a0696c799f85e97da1ab5b7910be3c1c80223" + +[[package]] +name = "fax" +version = "0.2.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "caf1079563223d5d59d83c85886a56e586cfd5c1a26292e971a0fa266531ac5a" + +[[package]] +name = "fdeflate" +version = "0.3.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e6853b52649d4ac5c0bd02320cddc5ba956bdb407c4b75a2c6b75bf51500f8c" +dependencies = [ + "simd-adler32", +] + +[[package]] +name = "ffmpeg-sys-next" +version = "8.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a314bc0e022a33a99567ed4bd2576bd58ffd8fcff7891c29194cfecc26a62547" +dependencies = [ + "bindgen", + "cc", + "libc", + "num_cpus", + "pkg-config", + "vcpkg", +] + +[[package]] +name = "find-msvc-tools" +version = "0.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5baebc0774151f905a1a2cc41989300b1e6fbb29aff0ceffa1064fdd3088d582" + +[[package]] +name = "flate2" +version = "1.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "843fba2746e448b37e26a819579957415c8cef339bf08564fe8b7ddbd959573c" +dependencies = [ + "crc32fast", + "miniz_oxide", +] + +[[package]] +name = "gbm" +version = "0.18.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ce852e998d3ca5e4a97014fb31c940dc5ef344ec7d364984525fd11e8a547e6a" +dependencies = [ + "bitflags", + "drm", + "drm-fourcc", + "gbm-sys", + "libc", + "wayland-backend", + "wayland-server", +] + +[[package]] +name = "gbm-sys" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c13a5f2acc785d8fb6bf6b7ab6bfb0ef5dad4f4d97e8e70bb8e470722312f76f" +dependencies = [ + "libc", +] + +[[package]] +name = "gcd" +version = "2.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d758ba1b47b00caf47f24925c0074ecb20d6dfcffe7f6d53395c0465674841a" + +[[package]] +name = "generic-array" +version = "0.14.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85649ca51fd72272d7821adaf274ad91c288277713d9c18820d8499a7ff69e9a" +dependencies = [ + "typenum", + "version_check", +] + +[[package]] +name = "gethostname" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1bd49230192a3797a9a4d6abe9b3eed6f7fa4c8a8a4947977c6f80025f92cbd8" +dependencies = [ + "rustix 1.1.4", + "windows-link", +] + +[[package]] +name = "getrandom" +version = "0.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "899def5c37c4fd7b2664648c28120ecec138e4d395b459e5ca34f9cce2dd77fd" +dependencies = [ + "cfg-if", + "libc", + "r-efi 5.3.0", + "wasip2", +] + +[[package]] +name = "getrandom" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "300e883d756b2e4ec94e02791f39b04b522276138852cfc41d9fb7e904106099" +dependencies = [ + "cfg-if", + "libc", + "r-efi 6.0.0", +] + +[[package]] +name = "gif" +version = "0.14.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ee8cfcc411d9adbbaba82fb72661cc1bcca13e8bba98b364e62b2dba8f960159" +dependencies = [ + "color_quant", + "weezl", +] + +[[package]] +name = "gl_generator" +version = "0.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1a95dfc23a2b4a9a2f5ab41d194f8bfda3cabec42af4e39f08c339eb2a0c124d" +dependencies = [ + "khronos_api", + "log", + "xml-rs", +] + +[[package]] +name = "glob" +version = "0.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e4eba85ea1d0a966a983acd07deee566e67395d2d96b6fb39e62b5a833f1eb0b" + +[[package]] +name = "half" +version = "2.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6ea2d84b969582b4b1864a92dc5d27cd2b77b622a8d79306834f1be5ba20d84b" +dependencies = [ + "cfg-if", + "crunchy", + "zerocopy", +] + +[[package]] +name = "hashbrown" +version = "0.17.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed5909b6e89a2db4456e54cd5f673791d7eca6732202bbf2a9cc504fe2f9b84a" + +[[package]] +name = "heck" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea" + +[[package]] +name = "hermit-abi" +version = "0.3.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d231dfb89cfffdbc30e7fc41579ed6066ad03abda9e567ccafae602b97ec5024" + +[[package]] +name = "hermit-abi" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fc0fef456e4baa96da950455cd02c081ca953b141298e41db3fc7e36b1da849c" + +[[package]] +name = "httpdate" +version = "1.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "df3b46402a9d5adb4c86a0cf463f42e19994e3ee891101b1841f30a545cb49a9" + +[[package]] +name = "image" +version = "0.25.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6506c6c10786659413faa717ceebcb8f70731c0a60cbae39795fdf114519c1a" +dependencies = [ + "bytemuck", + "byteorder-lite", + "color_quant", + "exr", + "gif", + "image-webp", + "moxcms", + "num-traits", + "png", + "qoi", + "ravif", + "rayon", + "rgb", + "tiff", + "zune-core 0.5.1", + "zune-jpeg 0.5.15", +] + +[[package]] +name = "image-webp" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "525e9ff3e1a4be2fbea1fdf0e98686a6d98b4d8f937e1bf7402245af1909e8c3" +dependencies = [ + "byteorder-lite", + "quick-error", +] + +[[package]] +name = "imgref" +version = "1.12.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "89194689a993ab15268672e99e7b0e19da2da3268ac682e8f02d29d4d1434cd7" + +[[package]] +name = "indexmap" +version = "2.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d466e9454f08e4a911e14806c24e16fba1b4c121d1ea474396f396069cf949d9" +dependencies = [ + "equivalent", + "hashbrown", +] + +[[package]] +name = "input" +version = "0.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fbdc09524a91f9cacd26f16734ff63d7dc650daffadd2b6f84d17a285bd875a9" +dependencies = [ + "bitflags", + "input-sys", + "libc", + "udev", +] + +[[package]] +name = "input-sys" +version = "1.19.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "36eee07d8e02bd95bf52b2e642cf13d33701b94c6e4b04fbf1d1fb07e9cb19e7" + +[[package]] +name = "interpolate_name" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c34819042dc3d3971c46c2190835914dfbe0c3c13f61449b2997f4e9722dfa60" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "io-lifetimes" +version = "1.0.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eae7b9aee968036d54dce06cebaefd919e4472e753296daccd6d344e3e2df0c2" +dependencies = [ + "hermit-abi 0.3.9", + "libc", + "windows-sys 0.48.0", +] + +[[package]] +name = "itertools" +version = "0.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "413ee7dfc52ee1a4949ceeb7dbc8a33f2d6c088194d9f922fb8318faf1f01186" +dependencies = [ + "either", +] + +[[package]] +name = "itertools" +version = "0.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2b192c782037fadd9cfa75548310488aabdbf3d2da73885b31bd0abd03351285" +dependencies = [ + "either", +] + +[[package]] +name = "itoa" +version = "1.0.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f42a60cbdf9a97f5d2305f08a87dc4e09308d1276d28c869c684d7777685682" + +[[package]] +name = "jobserver" +version = "0.1.35" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1c00acbd29eabad4a2392fa0e921c874934dbbf4194312ad20f04a0ed67a3cb3" +dependencies = [ + "getrandom 0.4.3", + "libc", +] + +[[package]] +name = "khronos_api" +version = "3.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e2db585e1d738fc771bf08a151420d3ed193d9d895a36df7f6f8a9456b911ddc" + +[[package]] +name = "lebe" +version = "0.5.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7a79a3332a6609480d7d0c9eab957bca6b455b91bb84e66d19f5ff66294b85b8" + +[[package]] +name = "libc" +version = "0.2.189" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3eaf3ede3fee6db1a4c2ee091bf8a8b4dccdc6d17f656fb07896ee72867612f2" + +[[package]] +name = "libfuzzer-sys" +version = "0.4.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a9fd2f41a1cba099f79a0b6b6c35656cf7c03351a7bae8ff0f28f25270f929d2" +dependencies = [ + "arbitrary", + "cc", +] + +[[package]] +name = "libloading" +version = "0.8.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d7c4b02199fee7c5d21a5ae7d8cfa79a6ef5bb2fc834d6e9058e89c825efdc55" +dependencies = [ + "cfg-if", + "windows-link", +] + +[[package]] +name = "libloading" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "754ca22de805bb5744484a5b151a9e1a8e837d5dc232c2d7d8c2e3492edc8b60" +dependencies = [ + "cfg-if", + "windows-link", +] + +[[package]] +name = "libm" +version = "0.2.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6d2cec3eae94f9f509c767b45932f1ada8350c4bdb85af2fcab4a3c14807981" + +[[package]] +name = "libudev-sys" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3c8469b4a23b962c1396b9b451dda50ef5b283e8dd309d69033475fa9b334324" +dependencies = [ + "libc", + "pkg-config", +] + +[[package]] +name = "linux-raw-sys" +version = "0.4.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d26c52dbd32dccf2d10cac7725f8eae5296885fb5703b261f7d0a0739ec807ab" + +[[package]] +name = "linux-raw-sys" +version = "0.9.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cd945864f07fe9f5371a27ad7b52a172b4b499999f1d97574c9fa68373937e12" + +[[package]] +name = "linux-raw-sys" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32a66949e030da00e8c7d4434b251670a91556f4144941d37452769c25d58a53" + +[[package]] +name = "log" +version = "0.4.33" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0ceec5bc11778974d1bcb055b18002eba7f4b3518b6a0081b3af5f21666da9ad" + +[[package]] +name = "loop9" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0fae87c125b03c1d2c0150c90365d7d6bcc53fb73a9acaef207d2d065860f062" +dependencies = [ + "imgref", +] + +[[package]] +name = "maybe-rayon" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8ea1f30cedd69f0a2954655f7188c6a834246d2bcf1e315e2ac40c4b24dc9519" +dependencies = [ + "cfg-if", + "rayon", +] + +[[package]] +name = "memchr" +version = "2.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf8baf1c55e62ffcace7a9f06f4bd9cd3f0c4beb022d3b367256b91b87513d98" + +[[package]] +name = "memmap2" +version = "0.9.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d1219ed1b7f229ee7104d281dd01d6802fe28bb6e95d292942c4daacdeb798c0" +dependencies = [ + "libc", +] + +[[package]] +name = "memoffset" +version = "0.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "488016bfae457b036d996092f6cb448677611ce4449e970ceaf42695203f218a" +dependencies = [ + "autocfg", +] + +[[package]] +name = "minimal-lexical" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "68354c5c6bd36d73ff3feceb05efa59b6acb7626617f4962be322a825e61f79a" + +[[package]] +name = "miniz_oxide" +version = "0.8.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fa76a2c86f704bdb222d66965fb3d63269ce38518b83cb0575fca855ebb6316" +dependencies = [ + "adler2", + "simd-adler32", +] + +[[package]] +name = "moxcms" +version = "0.7.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ac9557c559cd6fc9867e122e20d2cbefc9ca29d80d027a8e39310920ed2f0a97" +dependencies = [ + "num-traits", + "pxfm", +] + +[[package]] +name = "nasm-rs" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "706bf8a5e8c8ddb99128c3291d31bd21f4bcde17f0f4c20ec678d85c74faa149" +dependencies = [ + "log", +] + +[[package]] +name = "new_debug_unreachable" +version = "1.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "650eef8c711430f1a879fdd01d4745a7deea475becfb90269c06775983bbf086" + +[[package]] +name = "no_std_io2" +version = "0.9.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "418abd1b6d34fbf6cae440dc874771b0525a604428704c76e48b29a5e67b8003" +dependencies = [ + "memchr", +] + +[[package]] +name = "nom" +version = "7.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d273983c5a657a70a3e8f2a01329822f3b8c8172b73826411a55751e404a0a4a" +dependencies = [ + "memchr", + "minimal-lexical", +] + +[[package]] +name = "nom" +version = "8.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "df9761775871bdef83bee530e60050f7e54b1105350d6884eb0fb4f46c2f9405" +dependencies = [ + "memchr", +] + +[[package]] +name = "noop_proc_macro" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0676bb32a98c1a483ce53e500a81ad9c3d5b3f7c920c28c24e9cb0980d0b5bc8" + +[[package]] +name = "num-bigint" +version = "0.4.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c89e69e7e0f03bea5ef08013795c25018e101932225a656383bd384495ecc367" +dependencies = [ + "num-integer", + "num-traits", +] + +[[package]] +name = "num-complex" +version = "0.4.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "73f88a1307638156682bada9d7604135552957b7818057dcef22705b4d509495" +dependencies = [ + "bytemuck", + "num-traits", +] + +[[package]] +name = "num-derive" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed3955f1a9c7c0c15e092f9c887db08b1fc683305fdf6eb6684f22555355e202" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "num-integer" +version = "0.1.46" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7969661fd2958a5cb096e56c8e1ad0444ac2bbcd0061bd28660485a44879858f" +dependencies = [ + "num-traits", +] + +[[package]] +name = "num-rational" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f83d14da390562dca69fc84082e73e548e1ad308d24accdedd2720017cb37824" +dependencies = [ + "num-bigint", + "num-integer", + "num-traits", +] + +[[package]] +name = "num-traits" +version = "0.2.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "071dfc062690e90b734c0b2273ce72ad0ffa95f0c74596bc250dcfd960262841" +dependencies = [ + "autocfg", +] + +[[package]] +name = "num_cpus" +version = "1.17.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "91df4bbde75afed763b708b7eee1e8e7651e02d97f6d5dd763e89367e957b23b" +dependencies = [ + "hermit-abi 0.5.2", + "libc", +] + +[[package]] +name = "nvcodec-sys" +version = "0.1.0" + +[[package]] +name = "once_cell" +version = "1.21.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" + +[[package]] +name = "openh264" +version = "0.9.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6b2b561d2103303e233779545da757ecd35bad82188d619a6d8901f3007e1ff" +dependencies = [ + "openh264-sys2", + "wide", +] + +[[package]] +name = "openh264-sys2" +version = "0.9.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "75a8867e48183bbd9147380227448c065fe456eb30b0ebc68929809c36c30985" +dependencies = [ + "cc", + "nasm-rs", + "walkdir", +] + +[[package]] +name = "paste" +version = "1.0.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "57c0d7b74b563b49d38dae00a0c37d4d6de9b432382b2892f0574ddcae73fd0a" + +[[package]] +name = "pastey" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "35fb2e5f958ec131621fdd531e9fc186ed768cbe395337403ae56c17a74c68ec" + +[[package]] +name = "pin-project-lite" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a89322df9ebe1c1578d689c92318e070967d1042b512afbe49518723f4e6d5cd" + +[[package]] +name = "pixelflux" +version = "2.0.0" +dependencies = [ + "base64", + "calloop", + "cfg-if", + "crossbeam-channel", + "ffmpeg-sys-next", + "gbm", + "image", + "libc", + "libloading 0.9.0", + "memmap2", + "nvcodec-sys", + "openh264", + "openh264-sys2", + "pyo3", + "rayon", + "serde", + "serde_json", + "smithay", + "tiny_http", + "turbojpeg", + "wayland-client", + "wayland-protocols", + "wayland-protocols-misc", + "wayland-protocols-wlr", + "wayland-server", + "x11rb", + "x264-sys", + "xcursor", + "xxhash-rust", + "yuv", +] + +[[package]] +name = "pixman" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cea217d496c19ac0a8e502b37078e1f683d16344adee9eb247a5d57c165e1edf" +dependencies = [ + "drm-fourcc", + "paste", + "pixman-sys", + "thiserror 1.0.69", +] + +[[package]] +name = "pixman-sys" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a1a0483e89e81d7915defe83c51f23f6800594d64f6f4a21253ce87fd8444ada" + +[[package]] +name = "pkg-config" +version = "0.3.33" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "19f132c84eca552bf34cab8ec81f1c1dcc229b811638f9d283dceabe58c5569e" + +[[package]] +name = "png" +version = "0.18.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "60769b8b31b2a9f263dae2776c37b1b28ae246943cf719eb6946a1db05128a61" +dependencies = [ + "bitflags", + "crc32fast", + "fdeflate", + "flate2", + "miniz_oxide", +] + +[[package]] +name = "polling" +version = "3.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5d0e4f59085d47d8241c88ead0f274e8a0cb551f3625263c05eb8dd897c34218" +dependencies = [ + "cfg-if", + "concurrent-queue", + "hermit-abi 0.5.2", + "pin-project-lite", + "rustix 1.1.4", + "windows-sys 0.61.2", +] + +[[package]] +name = "portable-atomic" +version = "1.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3d20d5497ef88037a52ff98267d066e7f11fcc5e99bbfbd58a42336193aacec3" + +[[package]] +name = "ppv-lite86" +version = "0.2.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85eae3c4ed2f50dcfe72643da4befc30deadb458a9b590d720cde2f2b1e97da9" +dependencies = [ + "zerocopy", +] + +[[package]] +name = "prettyplease" +version = "0.2.37" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "479ca8adacdd7ce8f1fb39ce9ecccbfe93a3f1344b3d0d97f20bc0196208f62b" +dependencies = [ + "proc-macro2", + "syn 2.0.119", +] + +[[package]] +name = "proc-macro2" +version = "1.0.107" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "985e7ec9bb745e6ce6535b544d84d6cd6f7ad8bd711c398938ae983b91a766d9" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "profiling" +version = "1.0.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3d595e54a326bc53c1c197b32d295e14b169e3cfeaa8dc82b529f947fba6bcf5" +dependencies = [ + "profiling-procmacros", +] + +[[package]] +name = "profiling-procmacros" +version = "1.0.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4488a4a36b9a4ba6b9334a32a39971f77c1436ec82c38707bce707699cc3bbcb" +dependencies = [ + "quote", + "syn 2.0.119", +] + +[[package]] +name = "pulp" +version = "0.22.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "046aa45b989642ec2e4717c8e72d677b13edd831a4d3b6cf37d9a3e54912496a" +dependencies = [ + "bytemuck", + "cfg-if", + "libm", + "num-complex", + "paste", + "pulp-wasm-simd-flag", + "raw-cpuid", + "reborrow", + "version_check", +] + +[[package]] +name = "pulp-wasm-simd-flag" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d8f70e07b9c3962945a74e59ca1c511bba65b6419468acc217c457d93f3c740" + +[[package]] +name = "pxfm" +version = "0.1.30" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d55d956fa96f5ec02be2e13af0e20391a5aa83d6a074e3ad368959d0fab299ea" + +[[package]] +name = "pyo3" +version = "0.29.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fc153f5fd745cc038b5eed86622125969f8a39834a57bc96beaaf2512b1da729" +dependencies = [ + "libc", + "once_cell", + "portable-atomic", + "pyo3-build-config", + "pyo3-ffi", + "pyo3-macros", +] + +[[package]] +name = "pyo3-build-config" +version = "0.29.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bb77d9aa6d647507b55c69ee714d266d84c526c78ff0bc6dd8757f58591e64d1" +dependencies = [ + "target-lexicon", +] + +[[package]] +name = "pyo3-ffi" +version = "0.29.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3160087aa5733bce7d9a729f8c55f85a2a53b74742a09613178eeebff0722253" +dependencies = [ + "libc", + "pyo3-build-config", +] + +[[package]] +name = "pyo3-macros" +version = "0.29.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "91f9d455db760a9a0b0ddeaac25f1390b8a36ba73dfbda9f127cac6fc340d4d5" +dependencies = [ + "proc-macro2", + "pyo3-macros-backend", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "pyo3-macros-backend" +version = "0.29.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e343bcec300ff262f5806a33a4e51b6d097a8a46435f512fcb83e95592581625" +dependencies = [ + "heck", + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "qoi" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7f6d64c71eb498fe9eae14ce4ec935c555749aef511cca85b5568910d6e48001" +dependencies = [ + "bytemuck", +] + +[[package]] +name = "quick-error" +version = "2.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a993555f31e5a609f617c12db6250dedcac1b0a85076912c436e6fc9b2c8e6a3" + +[[package]] +name = "quick-xml" +version = "0.41.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e660451e55124f798a69a5af3f49ccfbefbd41910eefd25caf2393e1f3473ec1" +dependencies = [ + "memchr", +] + +[[package]] +name = "quote" +version = "1.0.47" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fbf4db142a473a8d80c26bbf18454ed458bf8d26c8219c331daecfdbd079001" +dependencies = [ + "proc-macro2", +] + +[[package]] +name = "r-efi" +version = "5.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "69cdb34c158ceb288df11e18b4bd39de994f6657d83847bdffdbd7f346754b0f" + +[[package]] +name = "r-efi" +version = "6.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8dcc9c7d52a811697d2151c701e0d08956f92b0e24136cf4cf27b57a6a0d9bf" + +[[package]] +name = "rand" +version = "0.9.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b9ef1d0d795eb7d84685bca4f72f3649f064e6641543d3a8c415898726a57b41" +dependencies = [ + "rand_chacha", + "rand_core", +] + +[[package]] +name = "rand_chacha" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d3022b5f1df60f26e1ffddd6c66e8aa15de382ae63b3a0c1bfc0e4d3e3f325cb" +dependencies = [ + "ppv-lite86", + "rand_core", +] + +[[package]] +name = "rand_core" +version = "0.9.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "76afc826de14238e6e8c374ddcc1fa19e374fd8dd986b0d2af0d02377261d83c" +dependencies = [ + "getrandom 0.3.4", +] + +[[package]] +name = "rav1e" +version = "0.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "43b6dd56e85d9483277cde964fd1bdb0428de4fec5ebba7540995639a21cb32b" +dependencies = [ + "aligned-vec", + "arbitrary", + "arg_enum_proc_macro", + "arrayvec", + "av-scenechange", + "av1-grain", + "bitstream-io", + "built", + "cfg-if", + "interpolate_name", + "itertools 0.14.0", + "libc", + "libfuzzer-sys", + "log", + "maybe-rayon", + "new_debug_unreachable", + "noop_proc_macro", + "num-derive", + "num-traits", + "paste", + "profiling", + "rand", + "rand_chacha", + "simd_helpers", + "thiserror 2.0.19", + "v_frame", + "wasm-bindgen", +] + +[[package]] +name = "ravif" +version = "0.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ef69c1990ceef18a116855938e74793a5f7496ee907562bd0857b6ac734ab285" +dependencies = [ + "avif-serialize", + "imgref", + "loop9", + "quick-error", + "rav1e", + "rayon", + "rgb", +] + +[[package]] +name = "raw-cpuid" +version = "11.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "498cd0dc59d73224351ee52a95fee0f1a617a2eae0e7d9d720cc622c73a54186" +dependencies = [ + "bitflags", +] + +[[package]] +name = "rayon" +version = "1.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fb39b166781f92d482534ef4b4b1b2568f42613b53e5b6c160e24cfbfa30926d" +dependencies = [ + "either", + "rayon-core", +] + +[[package]] +name = "rayon-core" +version = "1.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "22e18b0f0062d30d4230b2e85ff77fdfe4326feb054b9783a3460d8435c8ab91" +dependencies = [ + "crossbeam-deque", + "crossbeam-utils", +] + +[[package]] +name = "reborrow" +version = "0.5.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "03251193000f4bd3b042892be858ee50e8b3719f2b08e5833ac4353724632430" + +[[package]] +name = "regex" +version = "1.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f020237b6c8eed93db2e2cb53c00c60a8e1bc73da7d073199a1180401450218d" +dependencies = [ + "aho-corasick", + "memchr", + "regex-automata", + "regex-syntax", +] + +[[package]] +name = "regex-automata" +version = "0.4.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ad8553b9b26413251cbf30e620595c7a41b3887f03da04579c0e6b0d6a06b4b2" +dependencies = [ + "aho-corasick", + "memchr", + "regex-syntax", +] + +[[package]] +name = "regex-syntax" +version = "0.8.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d6f6ff9a378485b298a5286656da665ba74413d36db0979633275d2e708145d4" + +[[package]] +name = "rgb" +version = "0.8.53" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "47b34b781b31e5d73e9fbc8689c70551fd1ade9a19e3e28cfec8580a79290cc4" + +[[package]] +name = "rustc-hash" +version = "2.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6b1e7f9a428571be2dc5bc0505c13fb6bf936822b894ec87abf8a08a4e51742d" + +[[package]] +name = "rustix" +version = "0.38.44" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fdb5bc1ae2baa591800df16c9ca78619bf65c0488b41b96ccec5d11220d8c154" +dependencies = [ + "bitflags", + "errno", + "libc", + "linux-raw-sys 0.4.15", + "windows-sys 0.59.0", +] + +[[package]] +name = "rustix" +version = "1.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6fe4565b9518b83ef4f91bb47ce29620ca828bd32cb7e408f0062e9930ba190" +dependencies = [ + "bitflags", + "errno", + "libc", + "linux-raw-sys 0.12.1", + "windows-sys 0.61.2", +] + +[[package]] +name = "rustversion" +version = "1.0.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf54715a573b99ac80df0bc206da022bcd442c974952c7b9720069370852e21f" + +[[package]] +name = "safe_arch" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3a52ec151f024d703f9fd65abb7cbe81e7cdb39f18917a3a37e3014470dc7c59" +dependencies = [ + "bytemuck", +] + +[[package]] +name = "same-file" +version = "1.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "93fc1dc3aaa9bfed95e02e6eadabb4baf7e3078b0bd1b4d7b6b0b68378900502" +dependencies = [ + "winapi-util", +] + +[[package]] +name = "scoped-tls" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e1cf6437eb19a8f4a6cc0f7dca544973b0b78843adbfeb3683d1a94a0024a294" + +[[package]] +name = "serde" +version = "1.0.229" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4148590afebada386688f18773da617792bf2ef03ffc1e4cbd2b1d45b023e0ba" +dependencies = [ + "serde_core", + "serde_derive", +] + +[[package]] +name = "serde_core" +version = "1.0.229" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "67dca2c9c51e58a4791a4b1ed58308b39c64224d349a935ab5039aa360942a48" +dependencies = [ + "serde_derive", +] + +[[package]] +name = "serde_derive" +version = "1.0.229" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e7a5d71263a5a7d47b41f6b3f06ba276f10cc18b0931f1799f710578e2309348" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.3", +] + +[[package]] +name = "serde_json" +version = "1.0.151" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c841b55ecdae098c80dcae9cf767f6f8a0c2cdb3416bbef72181df4d0fe73f14" +dependencies = [ + "itoa", + "memchr", + "serde", + "serde_core", + "zmij", +] + +[[package]] +name = "serde_spanned" +version = "1.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6662b5879511e06e8999a8a235d848113e942c9124f211511b16466ee2995f26" +dependencies = [ + "serde_core", +] + +[[package]] +name = "sha2" +version = "0.10.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a7507d819769d01a365ab707794a4084392c824f54a7a6a7862f8c3d0892b283" +dependencies = [ + "cfg-if", + "cpufeatures", + "digest", +] + +[[package]] +name = "shlex" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0fda2ff0d084019ba4d7c6f371c95d8fd75ce3524c3cb8fb653a3023f6323e64" + +[[package]] +name = "shlex" +version = "2.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8fadd59c855ef2080decdef8ff161eb6661b86933c9d82e5ba29dc602a55aba" + +[[package]] +name = "simd-adler32" +version = "0.3.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3a219298ac11a56ea9a6d2120044824d6f01aeb034955e7af7bc16858527deea" + +[[package]] +name = "simd_helpers" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "95890f873bec569a0362c235787f3aca6e1e887302ba4840839bcc6459c42da6" +dependencies = [ + "quote", +] + +[[package]] +name = "slab" +version = "0.4.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0c790de23124f9ab44544d7ac05d60440adc586479ce501c1d6d7da3cd8c9cf5" + +[[package]] +name = "smallvec" +version = "1.15.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8ed6a63f02c8539c91a8685a86f4099661ba3da017932f6ebbea6de3f0fa7c90" + +[[package]] +name = "smithay" +version = "0.7.0" +source = "git+https://github.com/Smithay/smithay?rev=ca932e042fa9ad150605c150a86275b85f9ad5b3#ca932e042fa9ad150605c150a86275b85f9ad5b3" +dependencies = [ + "appendlist", + "atomic_float", + "bitflags", + "calloop", + "cc", + "cgmath", + "cursor-icon", + "downcast-rs", + "drm", + "drm-ffi", + "drm-fourcc", + "errno", + "gbm", + "gl_generator", + "indexmap", + "input", + "libc", + "libloading 0.8.9", + "pixman", + "pkg-config", + "profiling", + "rand", + "rustix 1.1.4", + "sha2", + "smallvec", + "tempfile", + "thiserror 2.0.19", + "tracing", + "udev", + "wayland-backend", + "wayland-protocols", + "wayland-protocols-misc", + "wayland-protocols-wlr", + "wayland-server", + "wayland-sys", + "xkbcommon", +] + +[[package]] +name = "stable_deref_trait" +version = "1.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6ce2be8dc25455e1f91df71bfa12ad37d7af1092ae736f3a6cd0e37bc7810596" + +[[package]] +name = "syn" +version = "2.0.119" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "872831b642d1a07999a962a351ed35b955ea2cfc8f3862091e2a240a84f17297" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "syn" +version = "3.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "53e9bae58849f64dfa4f5d5ae372c8341f7305f82a3868709269343628b659a3" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "system-deps" +version = "7.0.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "396a35feb67335377e0251fcbc1092fc85c484bd4e3a7a54319399da127796e7" +dependencies = [ + "cfg-expr", + "heck", + "pkg-config", + "toml", + "version-compare", +] + +[[package]] +name = "target-lexicon" +version = "0.13.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "adb6935a6f5c20170eeceb1a3835a49e12e19d792f6dd344ccc76a985ca5a6ca" + +[[package]] +name = "tempfile" +version = "3.27.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32497e9a4c7b38532efcdebeef879707aa9f794296a4f0244f6f69e9bc8574bd" +dependencies = [ + "fastrand", + "getrandom 0.4.3", + "once_cell", + "rustix 1.1.4", + "windows-sys 0.61.2", +] + +[[package]] +name = "thiserror" +version = "1.0.69" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6aaf5339b578ea85b50e080feb250a3e8ae8cfcdff9a461c9ec2904bc923f52" +dependencies = [ + "thiserror-impl 1.0.69", +] + +[[package]] +name = "thiserror" +version = "2.0.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09a43598840e33d5b0331f38c5e30d13bb11c11210a4b58f0d9b18a5a5eefcd9" +dependencies = [ + "thiserror-impl 2.0.19", +] + +[[package]] +name = "thiserror-impl" +version = "1.0.69" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4fee6c4efc90059e10f81e6d42c60a18f76588c3d74cb83a0b242a2b6c7504c1" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "thiserror-impl" +version = "2.0.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "43cbfe0cf76104d42a574802844187e84a305e531ed54455f11fbde0f10541cd" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.3", +] + +[[package]] +name = "tiff" +version = "0.10.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "af9605de7fee8d9551863fd692cce7637f548dbd9db9180fcc07ccc6d26c336f" +dependencies = [ + "fax", + "flate2", + "half", + "quick-error", + "weezl", + "zune-jpeg 0.4.21", +] + +[[package]] +name = "tiny_http" +version = "0.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "389915df6413a2e74fb181895f933386023c71110878cd0825588928e64cdc82" +dependencies = [ + "ascii", + "chunked_transfer", + "httpdate", + "log", +] + +[[package]] +name = "toml" +version = "1.1.4+spec-1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3aace63f4bbcdfc2c965b059de67119c89c4017a70d633be6c104910f67056f5" +dependencies = [ + "indexmap", + "serde_core", + "serde_spanned", + "toml_datetime", + "toml_parser", + "toml_writer", + "winnow", +] + +[[package]] +name = "toml_datetime" +version = "1.1.1+spec-1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3165f65f62e28e0115a00b2ebdd37eb6f3b641855f9d636d3cd4103767159ad7" +dependencies = [ + "serde_core", +] + +[[package]] +name = "toml_parser" +version = "1.1.3+spec-1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d38ac1cf9b95face32296c0a3ede1fdc270627c9d9c02a7274dd6d960dc4d56" +dependencies = [ + "winnow", +] + +[[package]] +name = "toml_writer" +version = "1.1.2+spec-1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7d56353a2a665ad0f41a421187180aab746c8c325620617ad883a99a1cbe66d2" + +[[package]] +name = "tracing" +version = "0.1.44" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "63e71662fa4b2a2c3a26f570f037eb95bb1f85397f3cd8076caed2f026a6d100" +dependencies = [ + "log", + "pin-project-lite", + "tracing-attributes", + "tracing-core", +] + +[[package]] +name = "tracing-attributes" +version = "0.1.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7490cfa5ec963746568740651ac6781f701c9c5ea257c58e057f3ba8cf69e8da" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "tracing-core" +version = "0.1.36" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "db97caf9d906fbde555dd62fa95ddba9eecfd14cb388e4f491a66d74cd5fb79a" +dependencies = [ + "once_cell", +] + +[[package]] +name = "turbojpeg" +version = "1.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "81313fcd64aaa30ddfd59eaa122968136b2dd9cd41d9619b126cf71e67a01702" +dependencies = [ + "gcd", + "libc", + "thiserror 1.0.69", + "turbojpeg-sys", +] + +[[package]] +name = "turbojpeg-sys" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "49b5a974bc59382d35927e3a07eef2ba9a24fe7227cefe52ec44b508e1b90f86" +dependencies = [ + "anyhow", + "cmake", + "libc", + "pkg-config", +] + +[[package]] +name = "typenum" +version = "1.20.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6f5e870be6c3b371b77fe0ee0bafb859fa4964b4404c27de1d380043c4dda20" + +[[package]] +name = "udev" +version = "0.9.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "af4e37e9ea4401fc841ff54b9ddfc9be1079b1e89434c1a6a865dd68980f7e9f" +dependencies = [ + "io-lifetimes", + "libc", + "libudev-sys", + "pkg-config", +] + +[[package]] +name = "unicode-ident" +version = "1.0.24" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6e4313cd5fcd3dad5cafa179702e2b244f760991f45397d14d4ebf38247da75" + +[[package]] +name = "v_frame" +version = "0.3.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "666b7727c8875d6ab5db9533418d7c764233ac9c0cff1d469aec8fa127597be2" +dependencies = [ + "aligned-vec", + "num-traits", + "wasm-bindgen", +] + +[[package]] +name = "vcpkg" +version = "0.2.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "accd4ea62f7bb7a82fe23066fb0957d48ef677f6eeb8215f372f52e48bb32426" + +[[package]] +name = "version-compare" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "03c2856837ef78f57382f06b2b8563a2f512f7185d732608fd9176cb3b8edf0e" + +[[package]] +name = "version_check" +version = "0.9.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a" + +[[package]] +name = "walkdir" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "29790946404f91d9c5d06f9874efddea1dc06c5efe94541a7d6863108e3a5e4b" +dependencies = [ + "same-file", + "winapi-util", +] + +[[package]] +name = "wasip2" +version = "1.0.4+wasi-0.2.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b67efb37e106e55ce722a510d6b5f9c17f083e5fc79afc2badeb12cc313d9487" +dependencies = [ + "wit-bindgen", +] + +[[package]] +name = "wasm-bindgen" +version = "0.2.126" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4b067c0c11094aef6b7a801c1e34a26affafdf3d051dba08456b868789aaf9a4" +dependencies = [ + "cfg-if", + "once_cell", + "rustversion", + "wasm-bindgen-macro", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-macro" +version = "0.2.126" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "167ce5e579f6bcf889c4f7175a8a5a585de84e8ff93976ce393efa5f2837aab1" +dependencies = [ + "quote", + "wasm-bindgen-macro-support", +] + +[[package]] +name = "wasm-bindgen-macro-support" +version = "0.2.126" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f3997c7839262f4ef12cf90b818d6340c18e80f263f1a94bf157d0ec4420380e" +dependencies = [ + "bumpalo", + "proc-macro2", + "quote", + "syn 2.0.119", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-shared" +version = "0.2.126" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dc1b4cb0cc549fcf58d7dfc081778139b3d283a081644e833e84682ad71cea24" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "wayland-backend" +version = "0.3.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "016ccf01d1c58b6f8999612813e17c9b2390f7d70671428869913310f83f54b8" +dependencies = [ + "cc", + "downcast-rs", + "rustix 1.1.4", + "scoped-tls", + "smallvec", + "wayland-sys", +] + +[[package]] +name = "wayland-client" +version = "0.31.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e3c36a0f861ad76d0901f2800b46321410d9f73f2ea88aac0650d86c32688073" +dependencies = [ + "bitflags", + "rustix 1.1.4", + "wayland-backend", + "wayland-scanner", +] + +[[package]] +name = "wayland-protocols" +version = "0.32.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "23d0c813de3daa2ed6520af85a3bd49b0e722a3078506899aa9686fea58dc4b6" +dependencies = [ + "bitflags", + "wayland-backend", + "wayland-client", + "wayland-scanner", + "wayland-server", +] + +[[package]] +name = "wayland-protocols-misc" +version = "0.3.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6e9567599ef23e09b8dad6e429e5738d4509dfc46b3b21f32841a304d16b29c8" +dependencies = [ + "bitflags", + "wayland-backend", + "wayland-client", + "wayland-protocols", + "wayland-scanner", + "wayland-server", +] + +[[package]] +name = "wayland-protocols-wlr" +version = "0.3.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eb04e52f7836d7c7976c78ca0250d61e33873c34156a2a1fc9474828ec268234" +dependencies = [ + "bitflags", + "wayland-backend", + "wayland-client", + "wayland-protocols", + "wayland-scanner", + "wayland-server", +] + +[[package]] +name = "wayland-scanner" +version = "0.31.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "338e30461b3a2b67d70eb30a6d89f8e0c93a833e07d2ae89085cd070c4a00ac0" +dependencies = [ + "proc-macro2", + "quick-xml", + "quote", +] + +[[package]] +name = "wayland-server" +version = "0.31.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0dde9c29be0f723a573977de51ee455bf3dfa03652730a74f9dd3b337e374d75" +dependencies = [ + "bitflags", + "downcast-rs", + "rustix 1.1.4", + "wayland-backend", + "wayland-scanner", +] + +[[package]] +name = "wayland-sys" +version = "0.31.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d8eab23fefc9e41f8e841df4a9c707e8a8c4ed26e944ef69297184de2785e3be" +dependencies = [ + "dlib", + "libc", + "log", + "memoffset", + "once_cell", + "pkg-config", +] + +[[package]] +name = "weezl" +version = "0.1.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a28ac98ddc8b9274cb41bb4d9d4d5c425b6020c50c46f25559911905610b4a88" + +[[package]] +name = "wide" +version = "1.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "be99e8317aa9f08e7d16e13033ca43faab59ab582b1d0feab7b385424c42f8b1" +dependencies = [ + "bytemuck", + "safe_arch", +] + +[[package]] +name = "winapi-util" +version = "0.1.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c2a7b1c03c876122aa43f3020e6c3c3ee5c05081c9a00739faf7503aeba10d22" +dependencies = [ + "windows-sys 0.61.2", +] + +[[package]] +name = "windows-link" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f0805222e57f7521d6a62e36fa9163bc891acd422f971defe97d64e70d0a4fe5" + +[[package]] +name = "windows-sys" +version = "0.48.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "677d2418bec65e3338edb076e806bc1ec15693c5d0104683f2efe857f61056a9" +dependencies = [ + "windows-targets 0.48.5", +] + +[[package]] +name = "windows-sys" +version = "0.59.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e38bc4d79ed67fd075bcc251a1c39b32a1776bbe92e5bef1f0bf1f8c531853b" +dependencies = [ + "windows-targets 0.52.6", +] + +[[package]] +name = "windows-sys" +version = "0.61.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ae137229bcbd6cdf0f7b80a31df61766145077ddf49416a728b02cb3921ff3fc" +dependencies = [ + "windows-link", +] + +[[package]] +name = "windows-targets" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9a2fa6e2155d7247be68c096456083145c183cbbbc2764150dda45a87197940c" +dependencies = [ + "windows_aarch64_gnullvm 0.48.5", + "windows_aarch64_msvc 0.48.5", + "windows_i686_gnu 0.48.5", + "windows_i686_msvc 0.48.5", + "windows_x86_64_gnu 0.48.5", + "windows_x86_64_gnullvm 0.48.5", + "windows_x86_64_msvc 0.48.5", +] + +[[package]] +name = "windows-targets" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9b724f72796e036ab90c1021d4780d4d3d648aca59e491e6b98e725b84e99973" +dependencies = [ + "windows_aarch64_gnullvm 0.52.6", + "windows_aarch64_msvc 0.52.6", + "windows_i686_gnu 0.52.6", + "windows_i686_gnullvm", + "windows_i686_msvc 0.52.6", + "windows_x86_64_gnu 0.52.6", + "windows_x86_64_gnullvm 0.52.6", + "windows_x86_64_msvc 0.52.6", +] + +[[package]] +name = "windows_aarch64_gnullvm" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2b38e32f0abccf9987a4e3079dfb67dcd799fb61361e53e2882c3cbaf0d905d8" + +[[package]] +name = "windows_aarch64_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32a4622180e7a0ec044bb555404c800bc9fd9ec262ec147edd5989ccd0c02cd3" + +[[package]] +name = "windows_aarch64_msvc" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dc35310971f3b2dbbf3f0690a219f40e2d9afcf64f9ab7cc1be722937c26b4bc" + +[[package]] +name = "windows_aarch64_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09ec2a7bb152e2252b53fa7803150007879548bc709c039df7627cabbd05d469" + +[[package]] +name = "windows_i686_gnu" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a75915e7def60c94dcef72200b9a8e58e5091744960da64ec734a6c6e9b3743e" + +[[package]] +name = "windows_i686_gnu" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e9b5ad5ab802e97eb8e295ac6720e509ee4c243f69d781394014ebfe8bbfa0b" + +[[package]] +name = "windows_i686_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0eee52d38c090b3caa76c563b86c3a4bd71ef1a819287c19d586d7334ae8ed66" + +[[package]] +name = "windows_i686_msvc" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f55c233f70c4b27f66c523580f78f1004e8b5a8b659e05a4eb49d4166cca406" + +[[package]] +name = "windows_i686_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "240948bc05c5e7c6dabba28bf89d89ffce3e303022809e73deaefe4f6ec56c66" + +[[package]] +name = "windows_x86_64_gnu" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "53d40abd2583d23e4718fddf1ebec84dbff8381c07cae67ff7768bbf19c6718e" + +[[package]] +name = "windows_x86_64_gnu" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "147a5c80aabfbf0c7d901cb5895d1de30ef2907eb21fbbab29ca94c5b08b1a78" + +[[package]] +name = "windows_x86_64_gnullvm" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b7b52767868a23d5bab768e390dc5f5c55825b6d30b86c844ff2dc7414044cc" + +[[package]] +name = "windows_x86_64_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "24d5b23dc417412679681396f2b49f3de8c1473deb516bd34410872eff51ed0d" + +[[package]] +name = "windows_x86_64_msvc" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed94fce61571a4006852b7389a063ab983c02eb1bb37b47f8272ce92d06d9538" + +[[package]] +name = "windows_x86_64_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec" + +[[package]] +name = "winnow" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "23b97319f7b8343df12cc98938e5c3eb436064524c8d2b4e30a1d3a36eecdf81" + +[[package]] +name = "wit-bindgen" +version = "0.57.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ebf944e87a7c253233ad6766e082e3cd714b5d03812acc24c318f549614536e" + +[[package]] +name = "x11rb" +version = "0.13.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9993aa5be5a26815fe2c3eacfc1fde061fc1a1f094bf1ad2a18bf9c495dd7414" +dependencies = [ + "gethostname", + "rustix 1.1.4", + "x11rb-protocol", +] + +[[package]] +name = "x11rb-protocol" +version = "0.13.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ea6fc2961e4ef194dcbfe56bb845534d0dc8098940c7e5c012a258bfec6701bd" + +[[package]] +name = "x264-sys" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b694bc460acd21d48a05977f57025a93e2a4d4a10feffde9f0c66925d6e72ff4" +dependencies = [ + "bindgen", + "system-deps", +] + +[[package]] +name = "xcursor" +version = "0.3.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "163b33ed8786455e2fa5d72f554057ce3f3182425434f756cd39c99839d88e23" + +[[package]] +name = "xkbcommon" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a7a974f48060a14e95705c01f24ad9c3345022f4d97441b8a36beb7ed5c4a02d" +dependencies = [ + "libc", + "memmap2", + "xkeysym", +] + +[[package]] +name = "xkeysym" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b9cc00251562a284751c9973bace760d86c0276c471b4be569fe6b068ee97a56" + +[[package]] +name = "xml-rs" +version = "0.8.28" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3ae8337f8a065cfc972643663ea4279e04e7256de865aa66fe25cec5fb912d3f" + +[[package]] +name = "xxhash-rust" +version = "0.8.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "aee1b19627c7c60102ab80d3a9cbe18de90bfe03bfa6c3715447681f0e8c8af6" + +[[package]] +name = "y4m" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7a5a4b21e1a62b67a2970e6831bc091d7b87e119e7f9791aef9702e3bef04448" + +[[package]] +name = "yuv" +version = "0.8.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5d85a782d94ee43f078bcfd6fa82d4e6a5b2d1cfbbad168e4df5a9f7b39ef48c" +dependencies = [ + "num-traits", +] + +[[package]] +name = "zerocopy" +version = "0.8.55" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b5a105cd7b140f6eeec8acff2ea38135d3cab283ada58540f629fe51e46696eb" +dependencies = [ + "zerocopy-derive", +] + +[[package]] +name = "zerocopy-derive" +version = "0.8.55" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0fe976fb70c78cd64cccfe3a6fc142244e8a77b70959b30faf9d0ac37ee228eb" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "zmij" +version = "1.0.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "29666d0abbfad1e3dc4dcf6144730dd3a3ab225bbbdac83319345b1b44ccfc1b" + +[[package]] +name = "zune-core" +version = "0.4.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f423a2c17029964870cfaabb1f13dfab7d092a62a29a89264f4d36990ca414a" + +[[package]] +name = "zune-core" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cb8a0807f7c01457d0379ba880ba6322660448ddebc890ce29bb64da71fb40f9" + +[[package]] +name = "zune-inflate" +version = "0.2.54" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "73ab332fe2f6680068f3582b16a24f90ad7096d5d39b974d1c0aff0125116f02" +dependencies = [ + "simd-adler32", +] + +[[package]] +name = "zune-jpeg" +version = "0.4.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "29ce2c8a9384ad323cf564b67da86e21d3cfdff87908bc1223ed5c99bc792713" +dependencies = [ + "zune-core 0.4.12", +] + +[[package]] +name = "zune-jpeg" +version = "0.5.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "27bc9d5b815bc103f142aa054f561d9187d191692ec7c2d1e2b4737f8dbd7296" +dependencies = [ + "zune-core 0.5.1", +] From 3fd14b5e1c28dafa82df8309b73132d616c32c4c Mon Sep 17 00:00:00 2001 From: Seungmin Kim <8457324+ehfd@users.noreply.github.com> Date: Wed, 5 Aug 2026 14:22:00 +0900 Subject: [PATCH 16/21] fix: Various resolutions --- .github/workflows/docs.yml | 8 +- .github/workflows/pre-release.yml | 13 +- .github/workflows/release.yml | 17 ++- README.md | 2 +- pixelflux/src/encoders/nvenc.rs | 47 ++++--- pixelflux/src/encoders/overlay.rs | 9 +- pixelflux/src/encoders/software.rs | 96 +++++++++----- pixelflux/src/lib.rs | 70 ++++++++-- pixelflux/src/pipeline.rs | 7 +- pixelflux/src/wayland/cursor.rs | 203 ++++++++++++++--------------- pixelflux/src/wayland/frontend.rs | 8 +- pixelflux/src/wayland/wlclient.rs | 20 +-- pixelflux/src/x11/mod.rs | 75 ++++++++--- 13 files changed, 342 insertions(+), 233 deletions(-) diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml index 3952e5f..05321ac 100644 --- a/.github/workflows/docs.yml +++ b/.github/workflows/docs.yml @@ -10,7 +10,7 @@ jobs: permissions: contents: write steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v7 - name: Install Rust uses: dtolnay/rust-toolchain@stable @@ -29,11 +29,11 @@ jobs: run: | curl -L -o /tmp/miniforge.sh \ "https://github.com/conda-forge/miniforge/releases/latest/download/Miniforge3-$(uname)-$(uname -m).sh" - bash /tmp/miniforge.sh -b -p $HOME/miniforge3 - source $HOME/miniforge3/etc/profile.d/conda.sh + bash /tmp/miniforge.sh -b -p "$HOME/miniforge3" + source "$HOME/miniforge3/etc/profile.d/conda.sh" conda create -y -n pixelflux-docs -c conda-forge "ffmpeg=8.1" conda activate pixelflux-docs - echo "PKG_CONFIG_PATH=$CONDA_PREFIX/lib/pkgconfig:${PKG_CONFIG_PATH:-}" >> $GITHUB_ENV + echo "PKG_CONFIG_PATH=$CONDA_PREFIX/lib/pkgconfig:${PKG_CONFIG_PATH:-}" >> "$GITHUB_ENV" - name: Build docs run: cargo doc --no-deps diff --git a/.github/workflows/pre-release.yml b/.github/workflows/pre-release.yml index 99cd916..5eca9aa 100644 --- a/.github/workflows/pre-release.yml +++ b/.github/workflows/pre-release.yml @@ -4,6 +4,9 @@ on: push: branches: [master] +permissions: + contents: read + jobs: build: strategy: @@ -52,14 +55,14 @@ jobs: cp314-musllinux_aarch64 runs-on: ${{ matrix.runner }} steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v7 - name: Build wheels - uses: pypa/cibuildwheel@v4.1.0 + uses: pypa/cibuildwheel@v4.2.0 env: CIBW_BUILD: ${{ matrix.build }} - - uses: actions/upload-artifact@v4 + - uses: actions/upload-artifact@v7 with: name: wheels-${{ matrix.arch }}-${{ matrix.platform }} path: wheelhouse/*.whl @@ -70,7 +73,7 @@ jobs: permissions: contents: write steps: - - uses: actions/download-artifact@v4 + - uses: actions/download-artifact@v8 with: pattern: wheels-* path: wheelhouse @@ -81,7 +84,7 @@ jobs: run: echo "short=$(echo '${{ github.sha }}' | cut -c1-7)" >> "$GITHUB_OUTPUT" - name: Create pre-release - uses: softprops/action-gh-release@v2 + uses: softprops/action-gh-release@v3 with: tag_name: ${{ steps.sha.outputs.short }} name: ${{ steps.sha.outputs.short }} diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index cd4b564..3bef997 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -5,11 +5,14 @@ on: tags: - '[0-9]+.[0-9]+.[0-9]*' +permissions: + contents: read + jobs: check: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v7 - name: Verify release notes exist run: | @@ -67,14 +70,14 @@ jobs: cp314-musllinux_aarch64 runs-on: ${{ matrix.runner }} steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v7 - name: Build wheels - uses: pypa/cibuildwheel@v4.1.0 + uses: pypa/cibuildwheel@v4.2.0 env: CIBW_BUILD: ${{ matrix.build }} - - uses: actions/upload-artifact@v4 + - uses: actions/upload-artifact@v7 with: name: wheels-${{ matrix.arch }}-${{ matrix.platform }} path: wheelhouse/*.whl @@ -85,9 +88,9 @@ jobs: permissions: contents: write steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v7 - - uses: actions/download-artifact@v4 + - uses: actions/download-artifact@v8 with: pattern: wheels-* path: wheelhouse @@ -104,7 +107,7 @@ jobs: } >> "$GITHUB_OUTPUT" - name: Create release - uses: softprops/action-gh-release@v2 + uses: softprops/action-gh-release@v3 with: name: ${{ github.ref_name }} body: ${{ steps.notes.outputs.content }} diff --git a/README.md b/README.md index 8657206..edb202e 100644 --- a/README.md +++ b/README.md @@ -177,7 +177,7 @@ settings.damage_block_duration = 30 # Frames a stripe stays "damaged" # --- Watermarking --- # Must be a bytes object. The path to your PNG image. settings.watermark_path = b"/path/to/your/watermark.png" -settings.cursor_size_cap = 0 # Cap out-of-band hardware-cursor PNGs to this longest edge (<= 0 = uncapped) +settings.cursor_size_cap = 32 # Cap out-of-band hardware-cursor PNGs to this longest edge (<= 0 = uncapped) # 0:None, 1:TopLeft, 2:TopRight, 3:BottomLeft, 4:BottomRight, 5:Middle, 6:Animated settings.watermark_location_enum = 4 ``` diff --git a/pixelflux/src/encoders/nvenc.rs b/pixelflux/src/encoders/nvenc.rs index faddfb0..2e630a6 100644 --- a/pixelflux/src/encoders/nvenc.rs +++ b/pixelflux/src/encoders/nvenc.rs @@ -521,20 +521,20 @@ impl Drop for NvencEncoder { } } -/// The lowest H.264 level (nvcodec-sys numeric encoding: 52/60/61/62 for 5.2/6.0/6.1/6.2) -/// whose Annex-A limits fit this resolution and frame rate, floored at 5.2. -/// -/// A frame's cost is measured two ways against Annex-A Table A-1: **MaxFS**, the frame size in -/// macroblocks, and **MaxMBPS**, the macroblock rate (frame size × fps). The candidate table is -/// ascending and begins at 5.2 — there is no 5.1 entry — so every resolution up to 4K resolves -/// deterministically to High@5.2. Only above 4K does the level step up to 6.0 / 6.1 / 6.2; a flat -/// 5.2 would fail NVENC init there (5.2's MaxFS of 36864 MBs is ≈ 4096×2304). Picking the lowest -/// fitting level makes the SPS advertise the smallest level a decoder must support, and because the -/// level is fixed from frame 1 (with the profile held at High) the stream never bumps its level -/// mid-GOP — which would force some hardware decoders to re-initialize. Above 6.2's limits there is -/// no higher level, so it returns 6.2 as a best effort. use super::min_h264_level; +/// The level an NVENC session advertises for this geometry, floored at 5.2 (`NV_ENC_LEVEL` +/// shares level_idc's numbering: 52/60/61/62 for 5.2/6.0/6.1/6.2). +/// +/// `reconfigure_resolution` resizes a live session in place, so the level has to cover every +/// geometry that session can still reach — a level bump mid-GOP forces some hardware decoders to +/// re-initialize. 5.2's MaxFS of 36864 macroblocks (≈ 4096×2304) spans everything up to 4K, so +/// pinning that floor makes the whole range resolve to High@5.2; only beyond 4K does the shared +/// ladder step up to 6.0 / 6.1 / 6.2. +fn nvenc_h264_level(width: u32, height: u32, fps: u32) -> u32 { + min_h264_level(width, height, fps).max(52) +} + impl NvencEncoder { /// Resolve EGL at runtime rather than link against it, so one binary boots even on hosts /// without EGL — it is needed only by the zero-copy dmabuf path — and reach the @@ -732,10 +732,11 @@ impl NvencEncoder { /// High-4:4:4 profile; CBR (two-pass quarter-resolution for tighter per-frame rate adherence, /// VBV sizing, optional min/max QP clamps) or ConstQP; infinite GOP (`gopLength` / `idrPeriod` /// = `0xFFFFFFFF`); `zeroReorderDelay` plus a bitstream-restriction VUI (`max_num_reorder_frames=0`) - /// so no-reorder decoders don't buffer; an explicit Annex-A level from `min_h264_level` pinned - /// from frame 1 so the level never bumps mid-stream; BT.709 VUI colorimetry; chroma 4:2:0 or - /// 4:4:4 with the matching full-range flag; repeated SPS/PPS; CABAC; no AUD; strict GOP - /// target; and lookahead disabled for real-time latency. + /// so no-reorder decoders don't buffer; an explicit Annex-A level from `nvenc_h264_level` + /// pinned from frame 1 so the level never bumps mid-stream; SMPTE170M VUI colorimetry, + /// matching the fixed BT.601 matrix the hardware ARGB CSC applies; chroma 4:2:0 or 4:4:4 + /// with the matching full-range flag; repeated SPS/PPS; CABAC; no AUD; strict GOP target; + /// and lookahead disabled for real-time latency. /// 6. **Initialize with resize headroom**: `maxEncodeWidth` / `maxEncodeHeight` are raised to at /// least 4096×2304 (the 5.2 ceiling) so `reconfigure_resolution` can grow in place; this costs /// ~290 MiB of device memory, so a failed init retries at the exact size (in-place resize then @@ -927,7 +928,7 @@ impl NvencEncoder { config.rcParams.set_zeroReorderDelay(1); config.encodeCodecConfig.h264Config.h264VUIParameters.bitstreamRestrictionFlag = 1; config.encodeCodecConfig.h264Config.level = - min_h264_level(width, height, settings.target_fps as u32); + nvenc_h264_level(width, height, settings.target_fps as u32); config.encodeCodecConfig.h264Config.idrPeriod = 0xFFFFFFFF; config.encodeCodecConfig.h264Config.h264VUIParameters.videoSignalTypePresentFlag = 1; config.encodeCodecConfig.h264Config.h264VUIParameters.videoFormat = @@ -1175,7 +1176,7 @@ impl NvencEncoder { } self.encode_config.encodeCodecConfig.h264Config.level = - min_h264_level(new_w, new_h, settings.target_fps as u32); + nvenc_h264_level(new_w, new_h, settings.target_fps as u32); if is_cbr { let bps = (settings.video_bitrate_kbps.max(0) as u32).saturating_mul(1000); self.encode_config.rcParams.averageBitRate = bps; @@ -1365,7 +1366,7 @@ impl NvencEncoder { if self.init_params.frameRateNum != fps { self.init_params.frameRateNum = fps; self.init_params.frameRateDen = 1; - self.encode_config.encodeCodecConfig.h264Config.level = min_h264_level( + self.encode_config.encodeCodecConfig.h264Config.level = nvenc_h264_level( self.init_params.encodeWidth, self.init_params.encodeHeight, fps, @@ -1616,11 +1617,9 @@ impl NvencEncoder { } /// Encode a host ARGB frame by uploading it straight into the ARGB input surface, with no - /// CPU-side colour conversion. NVENC performs its hardware ARGB->YUV conversion with the - /// BT.601 matrix while the session's VUI is written BT.709 (a slight core-visual shift vs - /// the BT.709 CPU path): the VUI is reinterpreted by decoders but cannot be made to match - /// the nvenc hardware conversion — a 601->709 CPU prepass is intentionally skipped here to - /// keep this path zero-copy. + /// CPU-side colour conversion. NVENC's hardware ARGB→YUV conversion is fixed at the BT.601 + /// matrix, which is why the session VUI declares SMPTE170M; a CPU 601→709 prepass would cost + /// this path its zero-copy property. /// /// The packed rows are copied host→device into the registered ARGB surface and NVENC's hardware /// CSC produces the YUV. Bytes must be in NVENC ARGB order (`B,G,R,A` in memory) — the host BGRA diff --git a/pixelflux/src/encoders/overlay.rs b/pixelflux/src/encoders/overlay.rs index e0be09c..df0a86d 100644 --- a/pixelflux/src/encoders/overlay.rs +++ b/pixelflux/src/encoders/overlay.rs @@ -115,6 +115,9 @@ pub(crate) fn blend_pixel(dst: &mut [u8], r: u8, g: u8, b: u8, a: u8) { /// premultiplied by format definition): dst = src + dst*(1-a). The straight-alpha /// `blend_pixel` on premultiplied input multiplies alpha in twice and darkens /// every translucent pixel (visible dark fringes on the composited cursor). +/// +/// The sum saturates: some toolkits ship cursors whose colour exceeds its alpha, and on those +/// a wrapping cast would turn an over-bright pixel into a dark one. pub(crate) fn blend_pixel_premultiplied(dst: &mut [u8], r: u8, g: u8, b: u8, a: u8) { if a == 255 { dst[0] = b; @@ -122,9 +125,9 @@ pub(crate) fn blend_pixel_premultiplied(dst: &mut [u8], r: u8, g: u8, b: u8, a: dst[2] = r; } else if a > 0 { let ia = 255 - a as u32; - dst[0] = (b as u32 + dst[0] as u32 * ia / 255) as u8; - dst[1] = (g as u32 + dst[1] as u32 * ia / 255) as u8; - dst[2] = (r as u32 + dst[2] as u32 * ia / 255) as u8; + dst[0] = (b as u32 + dst[0] as u32 * ia / 255).min(255) as u8; + dst[1] = (g as u32 + dst[1] as u32 * ia / 255).min(255) as u8; + dst[2] = (r as u32 + dst[2] as u32 * ia / 255).min(255) as u8; } } diff --git a/pixelflux/src/encoders/software.rs b/pixelflux/src/encoders/software.rs index 550b65b..36db942 100644 --- a/pixelflux/src/encoders/software.rs +++ b/pixelflux/src/encoders/software.rs @@ -688,7 +688,6 @@ pub struct EncodedStripe { /// conversion band each, since the parallelism there already comes from encoding the stripes /// concurrently. #[allow(clippy::too_many_arguments)] -#[cfg_attr(not(feature = "gpl"), allow(unused_assignments, unused_mut, unused_variables))] pub fn encode_cpu( stripes: &mut Vec, raw_pixels: &[u8], @@ -846,6 +845,9 @@ pub fn encode_cpu( let mut send_this_stripe = false; let mut quality_or_crf = if output_mode == 0 { jpeg_q } else { video_crf }; + // Only the x264 stripe encoder consumes this; without `gpl` there is no + // consumer, so the flag and its sites go away with it. + #[cfg(feature = "gpl")] let mut force_idr = false; let is_dirty = if !hash_damage { stripe_is_dirty[i] @@ -892,7 +894,10 @@ pub fn encode_cpu( send_this_stripe = true; stripe_state.paint_over_sent = true; quality_or_crf = video_po_crf; - force_idr = true; + #[cfg(feature = "gpl")] + { + force_idr = true; + } stripe_state.h264_burst_frames_remaining = video_burst - 1; } } @@ -901,7 +906,10 @@ pub fn encode_cpu( if force_idr_all { send_this_stripe = true; if output_mode == 1 { - force_idr = true; + #[cfg(feature = "gpl")] + { + force_idr = true; + } if stripe_state.h264_burst_frames_remaining <= 0 && video_burst > 0 { stripe_state.paint_over_sent = true; stripe_state.h264_burst_frames_remaining = video_burst; @@ -1280,8 +1288,9 @@ mod qp_bound_sweep { //! Invariants under test: the CBR QP clamp reaches libx264/OpenH264 (a max clamp must //! raise worst-case fidelity on rate-starved text at the cost of bitrate overshoot; //! a min clamp must cut spend on over-budgeted content) and defaults (0) leave the - //! encoders' own behavior untouched. The printed table is the measurement behind the - //! shipped guidance for video_min_qp/video_max_qp. + //! encoders' own behavior untouched. Each encoder is swept separately so the OpenH264 + //! sweep still runs without the `gpl` feature, where it is the only software H.264 path. + #[cfg(feature = "gpl")] use super::H264EncoderWrapper; use crate::encoders::oh264::Openh264Encoder; use crate::RustCaptureSettings; @@ -1328,6 +1337,7 @@ mod qp_bound_sweep { /// Encode `FRAMES` scrolling-text luma frames through the x264 stripe encoder at the /// given rate-control settings (constant grey chroma), returning each frame's raw bitstream. + #[cfg(feature = "gpl")] fn encode_x264(cbr: bool, kbps: i32, crf: i32, min_qp: i32, max_qp: i32) -> Vec> { let mut enc = H264EncoderWrapper::new( W as i32, H as i32, crf, false, 60.0, 4, cbr, kbps, 50, min_qp, max_qp, @@ -1429,53 +1439,67 @@ mod qp_bound_sweep { / 1000.0 } - /// Diagnostic that the CBR QP clamp is actually plumbed through to both x264 and - /// OpenH264, printing a bitrate/PSNR table on scrolling text and asserting the effect. + /// Diagnostic that the CBR QP clamp is actually plumbed through to x264, printing a + /// bitrate/PSNR table on scrolling text and asserting the effect. /// /// Encodes worst-case scrolling text at 2 Mbps CBR across a sweep of `max_qp` values (plus a /// separate `min_qp` sweep on an over-provisioned 12 Mbps budget), measuring luma PSNR against a - /// per-encoder near-lossless CRF/QP-12 reference so colour-conversion differences cancel out. It - /// asserts that capping `max_qp` at 30 on rate-starved content lifts fidelity by more than - /// 0.5 dB over the unclamped run on both encoders — proving the clamp reaches the encoder rather - /// than being silently dropped (paid for in bitrate overshoot). + /// near-lossless CRF-12 reference from the same encoder so colour-conversion differences cancel + /// out. Capping `max_qp` at 30 on rate-starved content must lift fidelity by more than 0.5 dB + /// over the unclamped run — proving the clamp reaches the encoder rather than being silently + /// dropped (paid for in bitrate overshoot). + #[cfg(feature = "gpl")] #[test] - fn cbr_qp_bound_sweep_diagnostic() { - let ref_x264 = decode_luma(&encode_x264(false, 0, 12, 0, 0)); - let ref_oh264 = decode_luma(&encode_oh264(false, 0, 12, 0, 0)); + fn cbr_qp_bound_sweep_x264() { + let reference = decode_luma(&encode_x264(false, 0, 12, 0, 0)); - println!("scrolling-text 720p60 @ 2 Mbps CBR (PSNR vs own CRF/QP-12 decode):"); + println!("scrolling-text 720p60 @ 2 Mbps CBR, x264 (PSNR vs own CRF-12 decode):"); let mut rows = Vec::new(); for &max_qp in &[0i32, 45, 40, 35, 30] { let x = encode_x264(true, 2000, 25, 0, max_qp); - let o = encode_oh264(true, 2000, 25, 0, max_qp); - let px = mean_psnr(&decode_luma(&x), &ref_x264); - let po = mean_psnr(&decode_luma(&o), &ref_oh264); - println!( - " max_qp {:>2}: x264 {:>8.1} kbps / {:>5.2} dB | oh264 {:>8.1} kbps / {:>5.2} dB", - max_qp, kbps(&x), px, kbps(&o), po - ); - rows.push((max_qp, kbps(&x), px, kbps(&o), po)); + let psnr = mean_psnr(&decode_luma(&x), &reference); + println!(" max_qp {:>2}: {:>8.1} kbps / {:>5.2} dB", max_qp, kbps(&x), psnr); + rows.push((max_qp, psnr)); } - println!("scrolling-text 720p60 @ 12 Mbps CBR, min-QP sweep:"); + println!("scrolling-text 720p60 @ 12 Mbps CBR, x264 min-QP sweep:"); for &min_qp in &[0i32, 10, 15] { let x = encode_x264(true, 12000, 25, min_qp, 0); - let px = mean_psnr(&decode_luma(&x), &ref_x264); - println!(" min_qp {:>2}: x264 {:>8.1} kbps / {:>5.2} dB", min_qp, kbps(&x), px); + let psnr = mean_psnr(&decode_luma(&x), &reference); + println!(" min_qp {:>2}: {:>8.1} kbps / {:>5.2} dB", min_qp, kbps(&x), psnr); } - let base = &rows[0]; - let capped = rows.last().unwrap(); + let base = rows[0].1; + let capped = rows.last().unwrap().1; assert!( - capped.2 > base.2 + 0.5, - "x264 max-QP clamp had no effect: {:.2} vs {:.2} dB", - capped.2, - base.2 + capped > base + 0.5, + "x264 max-QP clamp had no effect: {capped:.2} vs {base:.2} dB" ); + } + + /// The OpenH264 counterpart of [`cbr_qp_bound_sweep_x264`], and the only software H.264 + /// sweep that runs in a build without the `gpl` feature. + /// + /// Same scrolling-text workload and 2 Mbps CBR `max_qp` sweep, with luma PSNR measured against + /// this encoder's own near-lossless QP-12 reference. Capping `max_qp` at 30 must lift fidelity + /// by more than 0.5 dB over the unclamped run. + #[test] + fn cbr_qp_bound_sweep_openh264() { + let reference = decode_luma(&encode_oh264(false, 0, 12, 0, 0)); + + println!("scrolling-text 720p60 @ 2 Mbps CBR, oh264 (PSNR vs own QP-12 decode):"); + let mut rows = Vec::new(); + for &max_qp in &[0i32, 45, 40, 35, 30] { + let o = encode_oh264(true, 2000, 25, 0, max_qp); + let psnr = mean_psnr(&decode_luma(&o), &reference); + println!(" max_qp {:>2}: {:>8.1} kbps / {:>5.2} dB", max_qp, kbps(&o), psnr); + rows.push((max_qp, psnr)); + } + + let base = rows[0].1; + let capped = rows.last().unwrap().1; assert!( - capped.4 > base.4 + 0.5, - "oh264 max-QP clamp had no effect: {:.2} vs {:.2} dB", - capped.4, - base.4 + capped > base + 0.5, + "oh264 max-QP clamp had no effect: {capped:.2} vs {base:.2} dB" ); } } diff --git a/pixelflux/src/lib.rs b/pixelflux/src/lib.rs index 5ac3683..69739bd 100644 --- a/pixelflux/src/lib.rs +++ b/pixelflux/src/lib.rs @@ -139,14 +139,23 @@ pub mod encoders { pub mod software; /// VA-API hardware H.264 encoder for Intel / AMD GPUs via FFmpeg. pub mod vaapi; - /// Minimum H.264 level for a stream of these dimensions at this frame rate, on the - /// 5.2-6.2 conformance ladder every hardware encoder here shares. NVENC's startup - /// clamp historically also covers all smaller sizes; keeping ONE ladder avoids two - /// encoders disagreeing on the same geometry. + /// Lowest H.264 level whose Annex-A Table A-1 limits admit a `width` x `height` stream at + /// `fps`, as level_idc (41 = 4.1, 52 = 5.2, 62 = 6.2). + /// + /// A frame is charged two ways: **MaxFS**, its size in macroblocks, and **MaxMBPS**, that + /// size times the frame rate. Advertising the lowest fitting level asks the least of a + /// decoder, so clients that gate hardware decode on the level accept the widest range of + /// streams. One shared ladder keeps two encoders from disagreeing on the same geometry; + /// it starts at 4.1 and a caller needing more raises the floor itself. Above 6.2's limits + /// there is no higher level, so it returns 62 as a best effort. pub(crate) fn min_h264_level(width: u32, height: u32, fps: u32) -> u32 { let mbs = (width as u64).div_ceil(16) * (height as u64).div_ceil(16); let mbps = mbs * fps.max(1) as u64; - const LEVELS: [(u32, u64, u64); 4] = [ + const LEVELS: [(u32, u64, u64); 8] = [ + (41, 8192, 245760), + (42, 8704, 522240), + (50, 22080, 589824), + (51, 36864, 983040), (52, 36864, 2073600), (60, 139264, 4177920), (61, 139264, 8355840), @@ -1118,6 +1127,10 @@ struct WlEncodeConfig { /// compatible NVENC session is handed over from the previous encode thread it is reconfigured in /// place (milliseconds) rather than rebuilt, which would stall the stream. The EGL display is /// always null here: readback mode never imports dmabufs. +/// +/// Without the `gpl` feature there is no striped x264: an `output_mode == 1` request that reaches +/// the software fallback is served by the BSD-licensed OpenH264 full-frame encoder instead, so +/// `(None, None)` then means the striped JPEG path alone. fn build_readback_encoders( settings: &RustCaptureSettings, try_gpu: bool, @@ -1216,9 +1229,10 @@ fn build_readback_encoders( /// 2. **Dispatch by encoder**: a hardware session runs `decide_hw_fullframe`, then colorspace- /// converts only the frames actually being encoded (RGBA vs BGRA source chosen by the renderer) /// into the reused NV12/YUV444 buffer and calls `encode_raw`; OpenH264 encodes host pixels -/// directly; otherwise `encode_cpu` runs the striped x264/JPEG path with compositor damage. The -/// software H.264 path keeps an infinite GOP, forcing an IDR only on an explicit request or the -/// configured interval, and an explicit request also forces a full JPEG resend for joiners. +/// directly; otherwise `encode_cpu` runs the striped x264/JPEG path with compositor damage +/// (JPEG alone in a build without the `gpl` feature). The software H.264 path keeps an infinite +/// GOP, forcing an IDR only on an explicit request or the configured interval, and an explicit +/// request also forces a full JPEG resend for joiners. /// 3. **Recycle then deliver**: the capture buffer is recycled BEFORE delivery so a slow consumer /// never pins one, then the stripes go to the delivery thread through a single-slot `send` whose /// blocking is the backpressure that overlaps delivery with the next render + encode. @@ -1303,6 +1317,15 @@ fn wayland_encode_loop(pool: &WlFramePool, cfg: WlEncodeConfig) -> Option YuvStandardMatrix::Bt601, + GpuEncoder::Vaapi(_) => YuvStandardMatrix::Bt709, + }; if settings.video_fullcolor { let y_size = (w * h) as usize; let (y_plane, rest) = nv12_buffer.split_at_mut(y_size); @@ -1318,9 +1341,9 @@ fn wayland_encode_loop(pool: &WlFramePool, cfg: WlEncodeConfig) -> Option Option { cap.video_encoder = Some(enc); cap.pending_force_idr = true; + cap.hw_rebuilt = true; eprintln!("[Wayland] zero-copy HW encoder rebuilt after repeated encode errors."); } None => { eprintln!("[Wayland] zero-copy HW encoder unrecoverable; demoting to readback encode."); cap.video_encoder = None; + cap.hw_rebuilt = false; // Mirror the startup intent: readback still // tries the GPU unless the operator opted out. let s = &cap.settings; @@ -3522,7 +3561,10 @@ fn run_wayland_thread( ..RustCaptureSettings::default() }, cursor_callback_set: false, - cursor_tx: wayland::cursor::spawn_cursor_worker(cursor_size, 32), + cursor_tx: wayland::cursor::spawn_cursor_worker( + cursor_size, + RustCaptureSettings::default().cursor_size_cap, + ), clipboard_callback: None, pending_clipboard_read: None, current_selection_mime: None, diff --git a/pixelflux/src/pipeline.rs b/pixelflux/src/pipeline.rs index 4c4960f..b682fbc 100644 --- a/pixelflux/src/pipeline.rs +++ b/pixelflux/src/pipeline.rs @@ -211,8 +211,11 @@ impl X11Pipeline { /// /// 1. **OpenH264** — when `use_openh264` is set (explicit opt-in, full-frame software). /// 2. **NVENC** — on an NVIDIA driver (or no detectable GPU, since the attempt is cheap). - /// 3. **VA-API** — on any other GPU driver (except 4:4:4 full-color, which falls to x264). - /// 4. **Software** (`X11Encoder::None`) — on any hardware init failure or explicit software. + /// 3. **VA-API** — on any other GPU driver (except 4:4:4 full-color, which falls back to the + /// software path). + /// 4. **Software** — `software_h264_encoder`: `X11Encoder::None`, meaning the striped x264 + /// inside `encode_cpu`, in a GPL build; the full-frame OpenH264 encoder when the `gpl` + /// feature is disabled. pub fn new(settings: RustCaptureSettings) -> Self { let hw = if settings.output_mode == 1 && settings.use_openh264 { println!("[x11] OpenH264 software encoder selected."); diff --git a/pixelflux/src/wayland/cursor.rs b/pixelflux/src/wayland/cursor.rs index 15525e7..d8959b7 100644 --- a/pixelflux/src/wayland/cursor.rs +++ b/pixelflux/src/wayland/cursor.rs @@ -8,7 +8,7 @@ //! suitable for the Python cursor callback. Uses `xcursor` to load the system cursor theme //! and render the appropriate frame at the current scale. -use image::{ImageBuffer, Rgba}; +use image::{ImageBuffer, Rgba, RgbaImage}; use pyo3::prelude::*; use pyo3::types::PyBytes; use std::collections::HashMap; @@ -29,8 +29,8 @@ pub enum CursorJob { SetCallback(Py), /// Reload the worker's theme handle at a new pixel size; later `Named` jobs render at it. SetSize(i32), - /// Cap the longest delivered cursor edge; larger images are downscaled like the X11 - /// XFixes monitor does (parity: `CaptureSettings.cursor_size_cap` was X11-only). + /// Cap the longest delivered cursor edge in pixels; larger sprites are downscaled with + /// the hotspot. `<= 0` delivers them uncapped. SetSizeCap(i32), Named { name: &'static str }, Hide, @@ -65,7 +65,7 @@ pub fn spawn_cursor_worker(cursor_size: i32, size_cap: i32) -> std::sync::mpsc:: let _ = std::thread::Builder::new().name("wl-cursor".into()).spawn(move || { let mut helper = Cursor::load(cursor_size); let mut cap = size_cap; - let mut cache: HashMap> = HashMap::new(); + let mut cache: HashMap<(u64, i32), CappedSprite> = HashMap::new(); let mut callback: Option> = None; while let Ok(job) = rx.recv() { let (msg_type, data, hot_x, hot_y): (&str, Vec, i32, i32) = match job { @@ -81,11 +81,14 @@ pub fn spawn_cursor_worker(cursor_size: i32, size_cap: i32) -> std::sync::mpsc:: cap = c; continue; } - CursorJob::Named { name } => match helper.get_png_data(name) { - Some((png, x, y)) => { - let (png, x, y) = cap_cursor_png(png, x as i32, y as i32, cap); - ("png", png, x, y) - } + CursorJob::Named { name } => match helper.get_sprite(name) { + Some((img, x, y)) => match cap_and_encode(img, cap) { + Some(sprite) => { + let (sx, sy) = scaled_hotspot(&sprite, x as i32, y as i32); + ("png", sprite.png, sx, sy) + } + None => ("error", Vec::new(), 0, 0), + }, None => ("error", Vec::new(), 0, 0), }, CursorJob::Hide => ("hide", Vec::new(), 0, 0), @@ -105,7 +108,7 @@ pub fn spawn_cursor_worker(cursor_size: i32, size_cap: i32) -> std::sync::mpsc:: cap, hot_x, hot_y, - || encode_shm_cursor(width, height, stride, offset, opaque, &bytes), + || decode_shm_cursor(width, height, stride, offset, opaque, &bytes), ), CursorJob::Gles { hash, width, height, bytes, hot_x, hot_y } => capped_job( &mut cache, @@ -113,7 +116,7 @@ pub fn spawn_cursor_worker(cursor_size: i32, size_cap: i32) -> std::sync::mpsc:: cap, hot_x, hot_y, - || encode_gles_cursor(width, height, &bytes), + || decode_gles_cursor(width, height, &bytes), ), }; // A sprite whose pixels could not be read yields empty data; suppressing it @@ -136,96 +139,100 @@ pub fn spawn_cursor_worker(cursor_size: i32, size_cap: i32) -> std::sync::mpsc:: tx } -/// Content-hash PNG cache lookup with bounded arbitrary eviction; an evicted sprite simply -/// re-encodes on its next appearance. -fn cached_png( - cache: &mut HashMap>, - hash: u64, - encode: impl FnOnce() -> Vec, -) -> Vec { - if let Some(png) = cache.get(&hash) { - return png.clone(); - } - let png = encode(); - if !png.is_empty() { - cache.insert(hash, png.clone()); - if cache.len() > 100 { - if let Some(&evict) = cache.keys().next() { - cache.remove(&evict); - } - } - } - png +/// A cursor sprite encoded at its delivered size: the straight-alpha PNG, the factor the +/// source was scaled by, and the payload's dimensions. Hotspots are per-job rather than a +/// property of the pixels, so they are scaled against `scale` instead of being stored here. +struct CappedSprite { + png: Vec, + scale: f64, + width: u32, + height: u32, } -/// Downscale a cursor PNG so its longest edge is at most `cap` (<= 0 = uncapped), scaling -/// the hotspot with it. Mirrors the X11 XFixes monitor's `cursor_size_cap` handling so a -/// Wayland session ships cursor payloads no larger than an X11 one. -fn cap_cursor_png(png: Vec, hot_x: i32, hot_y: i32, cap: i32) -> (Vec, i32, i32) { - if cap <= 0 || png.is_empty() { - return (png, hot_x, hot_y); - } - let img = match image::load_from_memory(&png) { - Ok(i) => i.to_rgba8(), - Err(_) => return (png, hot_x, hot_y), - }; +/// Sprites retained before arbitrary eviction; an evicted sprite re-encodes on next appearance. +const SPRITE_CACHE_MAX: usize = 100; + +/// Apply the size cap to a **premultiplied** sprite and encode it as a straight-alpha PNG. +/// +/// `cap <= 0`, or a sprite already within it, is encoded at its source size. Resizing happens +/// while the pixels are still premultiplied — filtering straight alpha pulls the (zero) colour +/// of transparent texels into the edges and leaves a dark fringe, and Lanczos3 overshoot has no +/// headroom to land in — which is the order the X11 XFixes path uses. +fn cap_and_encode(mut img: RgbaImage, cap: i32) -> Option { let (w, h) = (img.width(), img.height()); - let longest = w.max(h); - if longest <= cap as u32 || longest == 0 { - return (png, hot_x, hot_y); + if w == 0 || h == 0 { + return None; } - let scale = cap as f64 / longest as f64; - let nw = ((w as f64 * scale).round() as u32).max(1); - let nh = ((h as f64 * scale).round() as u32).max(1); - let resized = image::imageops::resize( - &img, - nw, - nh, - image::imageops::FilterType::Lanczos3, - ); - let mut out = Vec::new(); - if resized - .write_to(&mut IoCursor::new(&mut out), image::ImageFormat::Png) - .is_err() - { - return (png, hot_x, hot_y); + let longest = w.max(h); + let mut scale = 1.0; + if cap > 0 && longest > cap as u32 { + scale = cap as f64 / longest as f64; + let nw = ((w as f64 * scale).round() as u32).max(1); + let nh = ((h as f64 * scale).round() as u32).max(1); + img = image::imageops::resize(&img, nw, nh, image::imageops::FilterType::Lanczos3); } + crate::unpremultiply_rgba(&mut img); + let mut png = Vec::new(); + img.write_to(&mut IoCursor::new(&mut png), image::ImageFormat::Png) + .ok()?; + Some(CappedSprite { png, scale, width: img.width(), height: img.height() }) +} + +/// Scale a source hotspot onto a delivered sprite. Consumers treat the hotspot as an offset +/// INTO the bitmap, so the rounded coordinate is clamped to the payload rather than allowed to +/// land one pixel past its right or bottom edge. +fn scaled_hotspot(sprite: &CappedSprite, hot_x: i32, hot_y: i32) -> (i32, i32) { ( - out, - (hot_x as f64 * scale).round() as i32, - (hot_y as f64 * scale).round() as i32, + ((hot_x as f64 * sprite.scale).round() as i32).clamp(0, sprite.width as i32 - 1), + ((hot_y as f64 * sprite.scale).round() as i32).clamp(0, sprite.height as i32 - 1), ) } -/// Cache wrapper for sprite jobs: encode (or reuse) the full-size PNG, then apply the -/// size cap. `cap_cursor_png` is idempotent for already-capped images, so cache hits -/// only pay a cheap dimension check. +/// Cache wrapper for sprite jobs, keyed by `(sprite hash, cap)` so a repeat of a sprite already +/// delivered under the same cap is a plain clone of its PNG; a miss decodes the sprite, caps it +/// and encodes it once. Keying on the cap keeps a live `SetSizeCap` from serving stale sizes. fn capped_job( - cache: &mut HashMap>, + cache: &mut HashMap<(u64, i32), CappedSprite>, hash: u64, cap: i32, hot_x: i32, hot_y: i32, - encode: impl FnOnce() -> Vec, + decode: impl FnOnce() -> Option, ) -> (&'static str, Vec, i32, i32) { - let png = cached_png(cache, hash, encode); - let (png, sx, sy) = cap_cursor_png(png, hot_x, hot_y, cap); - ("png", png, sx, sy) + use std::collections::hash_map::Entry; + let key = (hash, cap); + if cache.len() >= SPRITE_CACHE_MAX && !cache.contains_key(&key) { + if let Some(&evict) = cache.keys().next() { + cache.remove(&evict); + } + } + let sprite = match cache.entry(key) { + Entry::Occupied(e) => e.into_mut(), + // An unreadable sprite yields an empty payload, which the worker suppresses so the + // consumer keeps the cursor it already has. + Entry::Vacant(e) => match decode().and_then(|img| cap_and_encode(img, cap)) { + Some(sprite) => e.insert(sprite), + None => return ("png", Vec::new(), 0, 0), + }, + }; + let (sx, sy) = scaled_hotspot(sprite, hot_x, hot_y); + ("png", sprite.png.clone(), sx, sy) } -/// Convert a wl_shm BGRA/XRGB sprite sub-image to a straight-alpha PNG. Stride/offset are -/// clamped non-negative with checked arithmetic so a garbage descriptor skips pixels instead of +/// Read a wl_shm BGRA/XRGB sprite sub-image into a premultiplied RGBA image, the form +/// `cap_and_encode` must resize before it unpremultiplies. Stride/offset are clamped +/// non-negative with checked arithmetic so a garbage descriptor skips pixels instead of /// panicking; sprites larger than 128x128 are ignored (never a hardware cursor). -fn encode_shm_cursor( +fn decode_shm_cursor( width: i32, height: i32, stride: i32, offset: i32, opaque: bool, raw_bytes: &[u8], -) -> Vec { +) -> Option { if width <= 0 || height <= 0 || width > 128 || height > 128 || raw_bytes.is_empty() { - return Vec::new(); + return None; } let mut img_buf = ImageBuffer::, Vec>::new(width as u32, height as u32); let stride_usize = stride.max(0) as usize; @@ -250,20 +257,14 @@ fn encode_shm_cursor( } } } - crate::unpremultiply_rgba(&mut img_buf); - let mut bytes = Vec::new(); - if img_buf.write_to(&mut IoCursor::new(&mut bytes), image::ImageFormat::Png).is_ok() { - bytes - } else { - Vec::new() - } + Some(img_buf) } -/// Convert a dmabuf sprite's RGBA readback (stride recovered from the mapping length) to a -/// straight-alpha PNG. -fn encode_gles_cursor(width: i32, height: i32, raw_bytes: &[u8]) -> Vec { +/// Read a dmabuf sprite's RGBA readback (stride recovered from the mapping length) into a +/// premultiplied RGBA image. +fn decode_gles_cursor(width: i32, height: i32, raw_bytes: &[u8]) -> Option { if width <= 0 || height <= 0 || width > 128 || height > 128 || raw_bytes.is_empty() { - return Vec::new(); + return None; } let stride = super::frontend::rgba_readback_stride( raw_bytes.len(), @@ -288,13 +289,7 @@ fn encode_gles_cursor(width: i32, height: i32, raw_bytes: &[u8]) -> Vec { } } } - crate::unpremultiply_rgba(&mut img_buf); - let mut bytes = Vec::new(); - if img_buf.write_to(&mut IoCursor::new(&mut bytes), image::ImageFormat::Png).is_ok() { - bytes - } else { - Vec::new() - } + Some(img_buf) } /// The loaded XCursor theme, held for the whole capture so cursor lookups stay cheap. @@ -354,27 +349,21 @@ impl Cursor { Some(frame(time.as_millis() as u32, size, &icons)) } - /// A named cursor icon as PNG bytes plus its hotspot (x, y), for web clients. - /// Xcursor stores premultiplied color; the PNG carries straight alpha. The - /// compositing paths (`get_image*`) keep the premultiplied pixels blending needs. - pub fn get_png_data(&self, name: &str) -> Option<(Vec, u32, u32)> { + /// A named cursor icon as a premultiplied RGBA image plus its hotspot (x, y), ready for + /// `cap_and_encode` to size and convert to the straight alpha web clients want. Xcursor + /// stores premultiplied colour, which is also what the compositing paths (`get_image*`) + /// need for blending, so it is carried through unconverted. + pub fn get_sprite(&self, name: &str) -> Option<(RgbaImage, u32, u32)> { let icons = load_icon(&self.theme, name).ok()?; let image_data = nearest_images(self.size, &icons).next()?; - let mut img_buf: ImageBuffer, Vec> = ImageBuffer::from_raw( + let img_buf: RgbaImage = ImageBuffer::from_raw( image_data.width, image_data.height, image_data.pixels_rgba.clone(), )?; - crate::unpremultiply_rgba(&mut img_buf); - - let mut bytes: Vec = Vec::new(); - let mut cursor = IoCursor::new(&mut bytes); - img_buf - .write_to(&mut cursor, image::ImageFormat::Png) - .ok()?; - Some((bytes, image_data.xhot, image_data.yhot)) + Some((img_buf, image_data.xhot, image_data.yhot)) } } diff --git a/pixelflux/src/wayland/frontend.rs b/pixelflux/src/wayland/frontend.rs index f446e42..5d5e1cd 100644 --- a/pixelflux/src/wayland/frontend.rs +++ b/pixelflux/src/wayland/frontend.rs @@ -226,9 +226,13 @@ pub struct WlCapture { /// Last tick this capture actually rendered; paces per-display fps under the one /// shared render timer (which fires at the fastest active capture's rate). pub last_tick: Option, - /// Consecutive zero-copy encode failures: past the recovery threshold the session - /// is rebuilt once, then the capture demotes to the readback path. + /// Consecutive zero-copy encode failures; reset by any frame that encodes cleanly. + /// Reaching `HW_ERROR_RECOVERY_THRESHOLD` triggers recovery. pub hw_error_streak: u32, + /// Whether the zero-copy session has already been rebuilt without a clean frame since. + /// Recovery rebuilds once and demotes to readback on the next streak, so a session that + /// constructs but never encodes cannot loop rebuilding forever. + pub hw_rebuilt: bool, } impl WlCapture { diff --git a/pixelflux/src/wayland/wlclient.rs b/pixelflux/src/wayland/wlclient.rs index 31a5fd6..6a3850e 100644 --- a/pixelflux/src/wayland/wlclient.rs +++ b/pixelflux/src/wayland/wlclient.rs @@ -196,11 +196,16 @@ pub(crate) fn read_fd_to_end(fd: &OwnedFd, idle: Duration) -> Result, St read_fd_to_end_capped(fd, idle, READ_CAP_MAX) } -/// Same as `read_fd_to_end` but bounded: a hostile or stuck source can otherwise -/// stream unbounded bytes into memory (the compositor-internal read caps at the -/// same 64 MiB the selkies clipboard protocol allows). +/// Ceiling on an unbounded clipboard source, matching what the selkies clipboard protocol +/// allows: a hostile or stuck peer could otherwise stream bytes into memory forever. const READ_CAP_MAX: usize = 64 * 1024 * 1024; +/// Same as `read_fd_to_end` but refuses a source larger than `cap`. +/// +/// `cap` is inclusive: a payload of exactly `cap` bytes is accepted. The limit is therefore +/// tested against the size this chunk *would* produce, before appending — a test on the +/// already-accumulated length is reached again before EOF can be observed, so it would reject +/// a payload that exactly fills the cap. pub(crate) fn read_fd_to_end_capped( fd: &OwnedFd, idle: Duration, @@ -209,9 +214,6 @@ pub(crate) fn read_fd_to_end_capped( let mut out = Vec::new(); let mut chunk = [0u8; 65536]; loop { - if out.len() >= cap { - return Err(format!("clipboard source exceeds the {cap} byte cap")); - } if !wait_readable(fd.as_raw_fd(), idle)? { return Err("clipboard source stalled".into()); } @@ -228,8 +230,10 @@ pub(crate) fn read_fd_to_end_capped( if n == 0 { return Ok(out); } - let take = (cap - out.len()).min(n as usize); - out.extend_from_slice(&chunk[..take]); + if out.len() + n as usize > cap { + return Err(format!("clipboard source exceeds the {cap} byte cap")); + } + out.extend_from_slice(&chunk[..n as usize]); } } diff --git a/pixelflux/src/x11/mod.rs b/pixelflux/src/x11/mod.rs index da701ff..459a06a 100644 --- a/pixelflux/src/x11/mod.rs +++ b/pixelflux/src/x11/mod.rs @@ -195,6 +195,29 @@ impl ShmSurface { } } +/// Reject a server whose root visual is not 32 bits per pixel. +/// +/// Every `ShmSurface` is sized at `width * 4` bytes per row and the capture is read as packed +/// BGRA, so a root of any other depth would be grabbed as garbage rather than failing. Checked +/// on the initial connection and again on every reconnect, since a restarted server can come +/// back with a different visual. +fn require_32bpp(conn: &RustConnection, screen: usize) -> Result<(), String> { + let root_depth = conn.setup().roots[screen].root_depth; + let bpp = conn + .setup() + .pixmap_formats + .iter() + .find(|f| f.depth == root_depth) + .map(|f| f.bits_per_pixel) + .unwrap_or(32); + if bpp != 32 { + return Err(format!( + "unsupported root depth {root_depth} ({bpp} bpp); only 32-bpp BGRA is supported" + )); + } + Ok(()) +} + /// Clamp a capture offset to the live root so a grab always has at least one root pixel in /// range: a past-end offset fails every `shm_get_image` with BadMatch and would burn out the /// failure budget with no chance of recovery. The upper clamp also keeps protocol-range roots @@ -251,6 +274,8 @@ fn try_rebuild_channel( cap_y: &mut i16, cap_w: &mut u16, cap_h: &mut u16, + root_w: &mut u16, + root_h: &mut u16, pool_n: usize, pool: &FramePool, surfaces: &mut Vec, @@ -258,6 +283,7 @@ fn try_rebuild_channel( ) -> Result<(), String> { let (new_conn, new_screen) = x11rb::connect(None).map_err(|e| format!("X11 reconnect failed: {e}"))?; + require_32bpp(&new_conn, new_screen)?; new_conn .shm_query_version() .map_err(|e| format!("shm_query_version: {e}"))? @@ -299,6 +325,8 @@ fn try_rebuild_channel( *cap_y = clamp_offset(rsettings.capture_y, geo.height); *cap_w = fw; *cap_h = fh; + *root_w = geo.width; + *root_h = geo.height; *surfaces = fresh; pool.bump_generation(); Ok(()) @@ -774,20 +802,7 @@ where let (mut conn, screen_num) = x11rb::connect(None).map_err(|e| format!("X11 connect failed: {e}"))?; let mut root = conn.setup().roots[screen_num].root; - let root_depth = conn.setup().roots[screen_num].root_depth; - - let bpp = conn - .setup() - .pixmap_formats - .iter() - .find(|f| f.depth == root_depth) - .map(|f| f.bits_per_pixel) - .unwrap_or(32); - if bpp != 32 { - return Err(format!( - "unsupported root depth {root_depth} ({bpp} bpp); only 32-bpp BGRA is supported" - )); - } + require_32bpp(&conn, screen_num)?; conn.shm_query_version() .map_err(|e| format!("shm_query_version: {e}"))? @@ -808,6 +823,9 @@ where let (mut cap_w, mut cap_h) = resolve_dims(geo.width, geo.height, &rsettings); let mut cap_x = clamp_offset(rsettings.capture_x, geo.width); let mut cap_y = clamp_offset(rsettings.capture_y, geo.height); + // Last root size seen. A live pan has to clamp against something even when the geometry + // round-trip fails, or the requested origin would be dropped instead of applied. + let (mut root_w, mut root_h) = (geo.width, geo.height); const POOL_N: usize = 3; let mut surfaces: Vec = Vec::with_capacity(POOL_N); @@ -828,13 +846,17 @@ where let enc_pool = pool.clone(); let enc_controls = controls.clone(); let enc_settings = settings.clone(); + let encode_panicked = Arc::new(AtomicBool::new(false)); + let guard_panicked = encode_panicked.clone(); let encode_thread = thread::spawn(move || { crate::boost_thread_priority(-10); let _ = encode_tid_tx.send(thread::current().id()); let mut on_frame = on_frame; // An encoder panic must never wedge the capture: without the pool shutdown // the capture thread would block in publish forever while is_capturing still - // reports true on this thread's handle. + // reports true on this thread's handle. Shutting the pool unblocks it through + // the same path a clean stop takes, so the panic is also recorded — otherwise + // the capture would return Ok and Python could not tell the two apart. let guard_pool = enc_pool.clone(); let guard_controls = enc_controls.clone(); let result = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| { @@ -842,6 +864,7 @@ where })); if result.is_err() { eprintln!("[pixelflux x11] encode thread panicked; shutting the pool to fail the capture"); + guard_panicked.store(true, Ordering::Release); guard_pool.shutdown(); guard_controls.stop.store(true, Ordering::Relaxed); } @@ -882,9 +905,13 @@ where // settings) would glue resolve_dims back to the ROOT size, undoing the // re-target on the next geometry poll. w/h <= 0 keep following the root. rsettings.auto_adjust_screen_capture_size = nw <= 0 || nh <= 0; + cap_x = clamp_offset(nx, root_w); + cap_y = clamp_offset(ny, root_h); if let Some(g) = conn.get_geometry(root).ok().and_then(|c| c.reply().ok()) { - cap_x = clamp_offset(nx, g.width); - cap_y = clamp_offset(ny, g.height); + root_w = g.width; + root_h = g.height; + cap_x = clamp_offset(nx, root_w); + cap_y = clamp_offset(ny, root_h); let (fw, fh) = resolve_dims(g.width, g.height, &rsettings); if fw != cap_w || fh != cap_h { if !pool.drain_for_resize(POOL_N, &controls.stop) { @@ -916,8 +943,10 @@ where if let Some(g) = conn.get_geometry(root).ok().and_then(|c| c.reply().ok()) { // A root shrink can strand even a previously-valid offset past // the new edge; re-clamp it against the live root. - cap_x = clamp_offset(rsettings.capture_x, g.width); - cap_y = clamp_offset(rsettings.capture_y, g.height); + root_w = g.width; + root_h = g.height; + cap_x = clamp_offset(rsettings.capture_x, root_w); + cap_y = clamp_offset(rsettings.capture_y, root_h); let (nw, nh) = resolve_dims(g.width, g.height, &rsettings); if nw != cap_w || nh != cap_h { if !pool.drain_for_resize(POOL_N, &controls.stop) { @@ -976,6 +1005,7 @@ where match try_rebuild_channel( &mut conn, &mut root, &rsettings, &mut cap_x, &mut cap_y, &mut cap_w, &mut cap_h, + &mut root_w, &mut root_h, POOL_N, &pool, &mut surfaces, &controls.stop, ) { Ok(()) => { recovered = true; break; } @@ -1068,7 +1098,12 @@ where for s in surfaces.iter_mut() { s.destroy(&conn); } - result + match result { + // A real capture error is the more specific diagnosis, so it outranks the panic flag. + Err(e) => Err(e), + Ok(()) if encode_panicked.load(Ordering::Acquire) => Err("encode thread panicked".into()), + Ok(()) => Ok(()), + } } #[cfg(test)] From e62c816c674e50047c15da9ae531295384fb8da4 Mon Sep 17 00:00:00 2001 From: Seungmin Kim <8457324+ehfd@users.noreply.github.com> Date: Wed, 5 Aug 2026 19:15:58 +0900 Subject: [PATCH 17/21] fix: More CI/CD --- .github/dependabot.yml | 34 +++++++++++++++++ pixelflux/src/wayland/frontend.rs | 61 ++++++++++++++++++++++++------- pyproject.toml | 2 +- 3 files changed, 82 insertions(+), 15 deletions(-) create mode 100644 .github/dependabot.yml diff --git a/.github/dependabot.yml b/.github/dependabot.yml new file mode 100644 index 0000000..f387727 --- /dev/null +++ b/.github/dependabot.yml @@ -0,0 +1,34 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at https://mozilla.org/MPL/2.0/. + +# One grouped weekly pull request per ecosystem, so a crate bump does not arrive +# as a dozen separate reviews. +version: 2 +updates: + - package-ecosystem: github-actions + directory: / + schedule: + interval: weekly + groups: + actions: + patterns: ["*"] + + - package-ecosystem: cargo + directories: ["/pixelflux", "/pixelflux/nvcodec-sys"] + schedule: + interval: weekly + groups: + crates: + patterns: ["*"] + ignore: + # Pinned to the version that produced the committed NVENC bindings + - dependency-name: bindgen + + - package-ecosystem: devcontainers + directory: /.devcontainer + schedule: + interval: weekly + groups: + devcontainer-features: + patterns: ["*"] diff --git a/pixelflux/src/wayland/frontend.rs b/pixelflux/src/wayland/frontend.rs index 5d5e1cd..51c438e 100644 --- a/pixelflux/src/wayland/frontend.rs +++ b/pixelflux/src/wayland/frontend.rs @@ -138,7 +138,7 @@ use crate::encoders::vaapi::VaapiEncoder; use crate::nvenc::NvencEncoder; use crate::{RustCaptureSettings, StripeState}; -use std::sync::atomic::{AtomicU32, Ordering}; +use std::sync::atomic::{AtomicBool, AtomicU32, Ordering}; static SERIAL_COUNTER: AtomicU32 = AtomicU32::new(1); @@ -296,10 +296,16 @@ impl OutputNode { static NEXT_WINDOW_ID: AtomicU32 = AtomicU32::new(1); /// Per-window bookkeeping carried in the window's user-data map: a stable numeric id the -/// Python side addresses the window by, and the display id of the output it is placed on. +/// Python side addresses the window by, the display id of the output it is placed on, and +/// whether that output has been chosen yet. pub struct WindowMeta { pub id: u32, pub output: AtomicU32, + /// Set once the first commit has picked the window's output. The choice cannot key off + /// xdg's `initial_configure_sent`: a client that negotiates xdg-decoration (every + /// wlroots-based nested compositor, GTK and Qt) is answered with a configure before it + /// ever commits, which would leave every window on the pointer's output. + pub placed: AtomicBool, } /// The window's meta, inserted at `new_toplevel`; windows created before that (none in @@ -677,17 +683,15 @@ impl CompositorHandler for AppState { let window = self.pending_windows.remove(idx); let toplevel = window.toplevel().unwrap(); - let initial_configure_sent = with_states(surface, |states| { - states - .data_map - .get::() - .unwrap() - .lock() - .unwrap() - .initial_configure_sent - }); + // Whether this window still needs an output. Deliberately not xdg's + // `initial_configure_sent`: the decoration handshake answers with a configure + // before the client's first commit, which would skip the choice below for + // every client that negotiates decorations. + let needs_placement = window_meta(&window) + .map(|meta| !meta.placed.swap(true, Ordering::Relaxed)) + .unwrap_or(false); - if !initial_configure_sent { + if needs_placement { // A new toplevel opens fullscreened on the output the pointer is on // (primary when indeterminate); the choice is pinned on the window's // meta so the acked commit maps to the same output. @@ -1987,6 +1991,7 @@ impl XdgShellHandler for AppState { window.user_data().insert_if_missing_threadsafe(|| WindowMeta { id: NEXT_WINDOW_ID.fetch_add(1, Ordering::Relaxed), output: AtomicU32::new(target_id), + placed: AtomicBool::new(false), }); self.pending_windows.push(window); let (title, app_id) = with_states(surface.wl_surface(), |states| { @@ -2037,12 +2042,40 @@ impl XdgShellHandler for AppState { let _ = surface.send_repositioned(token); } /// Client fullscreen request: always granted at the compositor's forced-fullscreen - /// geometry (the CURRENT logical size), so the toggle gets a definite answer. + /// geometry (the CURRENT logical size), so the toggle gets a definite answer. A request + /// naming an output moves the window there, which is how a nested compositor puts each + /// of its screens on a monitor of its choosing. fn fullscreen_request( &mut self, surface: ToplevelSurface, - _output: Option, + output: Option, ) { + if let Some(id) = output + .as_ref() + .and_then(|wlo| self.output_nodes.iter().find(|n| n.output.owns(wlo)).map(|n| n.id)) + { + let mapped = self + .space + .elements() + .find(|w| w.toplevel().map(|tl| *tl == surface).unwrap_or(false)) + .cloned(); + if let Some(window) = mapped { + if self.place_window_on_output(&window, id) { + return; + } + } else if let Some(meta) = self + .pending_windows + .iter() + .find(|w| w.toplevel().map(|tl| *tl == surface).unwrap_or(false)) + .and_then(window_meta) + { + // Still waiting for its first buffer: record the output instead of mapping + // it, so nothing renders or hit-tests before the client has drawn. The + // configure below then carries that output's geometry. + meta.output.store(id, Ordering::Relaxed); + meta.placed.store(true, Ordering::Relaxed); + } + } self.send_forced_fullscreen_configure(&surface); } /// Client unfullscreen request: the forced-fullscreen policy stands, but the client diff --git a/pyproject.toml b/pyproject.toml index 72feece..06d8756 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -7,7 +7,7 @@ archs = [ "x86_64", "aarch64" ] manylinux-x86_64-image = "manylinux_2_28" manylinux-aarch64-image = "manylinux_2_28" build = "cp39-* cp310-* cp311-* cp312-* cp313-* cp314-*" -skip = "*t-* pp*" +skip = "*t-*" [tool.cibuildwheel.environment] LD_LIBRARY_PATH = "/usr/local/lib:/usr/local/lib64" From 283586acdfc91afe69f6b0d0d2346fadabfb03a7 Mon Sep 17 00:00:00 2001 From: Seungmin Kim <8457324+ehfd@users.noreply.github.com> Date: Wed, 5 Aug 2026 19:40:59 +0900 Subject: [PATCH 18/21] fix: Dependabot --- .github/dependabot.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/dependabot.yml b/.github/dependabot.yml index f387727..c2a038d 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -25,8 +25,10 @@ updates: # Pinned to the version that produced the committed NVENC bindings - dependency-name: bindgen + # The devcontainers updater discovers `.devcontainer/**/devcontainer.json` + # itself; naming that directory makes it report the file as not found - package-ecosystem: devcontainers - directory: /.devcontainer + directory: / schedule: interval: weekly groups: From 67cc605c6d04002ba89f21a6734d714dcc14a07d Mon Sep 17 00:00:00 2001 From: Seungmin Kim <8457324+ehfd@users.noreply.github.com> Date: Wed, 5 Aug 2026 20:18:29 +0900 Subject: [PATCH 19/21] fix: Git flapping --- pyproject.toml | 24 +++++++++++++++++++----- 1 file changed, 19 insertions(+), 5 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 06d8756..02c2e8a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -16,7 +16,21 @@ PATH = "$HOME/.cargo/bin:$PATH" [tool.cibuildwheel.linux] before-all = """ set -euxo pipefail - + + # A transient clone failure fails the whole wheel build, so every source is + # retried; FFmpeg is additionally mirrored on GitHub at the identical tag. + clone_source() { + ref="$1"; dir="$2"; shift 2 + for url in "$@"; do + for attempt in 1 2 3; do + rm -rf "${dir}" + git clone --branch "${ref}" --depth 1 "${url}" "${dir}" && return 0 + sleep $((attempt * 5)) + done + done + return 1 + } + # --- MANYLINUX (RPM-based) --- # x264 is built from source for x264-sys (manylinux has no usable package); # ffmpeg n8.1 is built plain LGPL (no --enable-libx264/--enable-gpl: only @@ -48,17 +62,17 @@ before-all = """ cmake \ gcc-c++ && \ \ - curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y --profile minimal && \ + curl --proto '=https' --tlsv1.2 -sSf --retry 5 --retry-delay 3 --retry-connrefused https://sh.rustup.rs | sh -s -- -y --profile minimal && \ \ (cd /tmp && \ - git clone --branch stable --depth 1 https://code.videolan.org/videolan/x264.git && \ + clone_source stable x264 https://code.videolan.org/videolan/x264.git && \ cd x264 && \ ./configure --prefix=/usr/local --enable-shared --disable-cli && \ make -j$(nproc) && \ make install) && \ \ (cd /tmp && \ - git clone --branch n8.1 --depth 1 https://git.ffmpeg.org/ffmpeg.git && \ + clone_source n8.1 ffmpeg https://git.ffmpeg.org/ffmpeg.git https://github.com/FFmpeg/FFmpeg.git && \ cd ffmpeg && \ ./configure \ --prefix=/usr/local \ @@ -97,7 +111,7 @@ before-all = """ musl-dev \ ffmpeg-dev && \ \ - curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y --profile minimal + curl --proto '=https' --tlsv1.2 -sSf --retry 5 --retry-delay 3 --retry-connrefused https://sh.rustup.rs | sh -s -- -y --profile minimal else echo "Unsupported package manager" exit 1 From c762e6334ffcabaf009348b62e3de023f146fa3d Mon Sep 17 00:00:00 2001 From: Seungmin Kim <8457324+ehfd@users.noreply.github.com> Date: Thu, 6 Aug 2026 03:51:32 +0900 Subject: [PATCH 20/21] fix: Wayland multi-display --- pixelflux/src/lib.rs | 61 ++++++++++++----- pixelflux/src/wayland/frontend.rs | 106 +++++++++++++++++++++--------- 2 files changed, 118 insertions(+), 49 deletions(-) diff --git a/pixelflux/src/lib.rs b/pixelflux/src/lib.rs index 69739bd..a683d6a 100644 --- a/pixelflux/src/lib.rs +++ b/pixelflux/src/lib.rs @@ -642,7 +642,7 @@ pub enum ThreadCommand { /// Move the window with the given id onto output `output_id` (fullscreened there). MoveWindowToOutput { window_id: u32, output_id: u32, reply: std::sync::mpsc::Sender }, /// Reply with every mapped window as `(window_id, title, app_id, output_id)`. - ListWindows { reply: std::sync::mpsc::Sender> }, + ListWindows { reply: std::sync::mpsc::Sender> }, SetCursorCallback(Py), SetClipboardCallback(Py), /// Server-side clipboard offer: the compositor owns the selection and serves `data` as @@ -3205,9 +3205,10 @@ fn create_output_on( overlay_state: OverlayState::default(), capture: None, }); - // A nested session opens one host toplevel per screen and stacks the extras - // on an existing output until a display exists for them: hand the newest - // stacked window to the new output. + // A nested session opens one host toplevel per screen and the extras wait + // parked until a display exists for them: hand the newest waiting window to + // the new output. Windows stacked on an output rather than parked — anything + // placed before this compositor started parking them — remain candidates. let mut counts: Vec<(u32, usize)> = Vec::new(); for w in state.space.elements() { let oid = wayland::frontend::window_output_id(w); @@ -3216,19 +3217,29 @@ fn create_output_on( None => counts.push((oid, 1)), } } - let adopt = state - .space - .elements() - .filter(|w| { + let newest = |pred: &dyn Fn(&smithay::desktop::Window) -> bool| { + state + .space + .elements() + .filter(|w| pred(w)) + .max_by_key(|w| wayland::frontend::window_meta(w).map(|m| m.id).unwrap_or(0)) + .cloned() + }; + let adopt = newest(&|w| { + wayland::frontend::window_meta(w) + .map(|m| m.parked.load(std::sync::atomic::Ordering::Relaxed)) + .unwrap_or(false) + }) + .or_else(|| { + newest(&|w| { let oid = wayland::frontend::window_output_id(w); counts.iter().any(|(o, c)| *o == oid && *c >= 2) }) - .max_by_key(|w| wayland::frontend::window_meta(w).map(|m| m.id).unwrap_or(0)) - .cloned(); + }); if let Some(window) = adopt { state.place_window_on_output(&window, id); println!( - "[Wayland] Output {id}: adopted stacked window {}.", + "[Wayland] Output {id}: adopted waiting window {}.", wayland::frontend::window_meta(&window).map(|m| m.id).unwrap_or(0) ); } @@ -3311,7 +3322,13 @@ fn destroy_output_on(state: &mut AppState, id: u32) -> bool { .cloned() .collect(); for window in &windows { - state.place_window_on_output(window, 0); + // The primary keeps whichever screen it already shows: a nested session's + // second screen parks again rather than covering the first. + if state.would_cover_screen(window, 0) { + state.park_window(window, 0); + } else { + state.place_window_on_output(window, 0); + } } for w in &state.pending_windows { if let Some(meta) = wayland::frontend::window_meta(w) { @@ -3732,7 +3749,13 @@ fn run_wayland_thread( }) }) .unwrap_or_default(); - list.push((meta.id, title, app_id, meta.output.load(Ordering::Relaxed))); + list.push(( + meta.id, + title, + app_id, + meta.output.load(Ordering::Relaxed), + meta.parked.load(Ordering::Relaxed), + )); } let _ = reply.send(list); } @@ -4504,8 +4527,10 @@ impl WaylandBackend { .unwrap_or(false)) } - /// Every mapped window as `(window_id, title, app_id, output_id)`. - fn list_windows(&self, py: Python<'_>) -> PyResult> { + /// Every mapped window as `(window_id, title, app_id, output_id, waiting)`. A waiting + /// window is tagged for that output but mapped clear of every one of them, holding its + /// size until an output exists for it — a nested session's spare screens. + fn list_windows(&self, py: Python<'_>) -> PyResult> { let (reply_tx, reply_rx) = std::sync::mpsc::channel(); self.send(ThreadCommand::ListWindows { reply: reply_tx }) .map_err(|e| PyErr::new::(format!("Failed to list windows: {}", e)))?; @@ -5763,9 +5788,9 @@ impl ScreenCapture { be.bind(py).borrow().move_window_to_output(py, window_id, output_id) }) } - /// Every mapped window as `(window_id, title, app_id, output_id)`; empty when no - /// backend runs. - fn list_windows(&self, py: Python<'_>) -> PyResult> { + /// Every mapped window as `(window_id, title, app_id, output_id, waiting)`; empty when + /// no backend runs. + fn list_windows(&self, py: Python<'_>) -> PyResult> { wayland_backend_running(py) .map_or(Ok(Vec::new()), |be| be.bind(py).borrow().list_windows(py)) } diff --git a/pixelflux/src/wayland/frontend.rs b/pixelflux/src/wayland/frontend.rs index 51c438e..7636079 100644 --- a/pixelflux/src/wayland/frontend.rs +++ b/pixelflux/src/wayland/frontend.rs @@ -306,8 +306,16 @@ pub struct WindowMeta { /// wlroots-based nested compositor, GTK and Qt) is answered with a configure before it /// ever commits, which would leave every window on the pointer's output. pub placed: AtomicBool, + /// Set while the window is mapped outside every output, tagged to the output it will + /// take once one exists. See `park_window`. + pub parked: AtomicBool, } +/// Where a parked window is mapped: clear of every output, so it is composited into none of +/// them. Layout offsets are non-negative — the union layout Selkies computes re-anchors at +/// the origin — so far negative coordinates can never collide with a real output. +pub const PARKED_POS: (i32, i32) = (-(1 << 20), -(1 << 20)); + /// The window's meta, inserted at `new_toplevel`; windows created before that (none in /// practice) read as id 0 / primary output. pub fn window_meta(window: &Window) -> Option<&WindowMeta> { @@ -698,34 +706,26 @@ impl CompositorHandler for AppState { let mut target_id = self.pointer_display(); // A second fullscreen surface from a client that already owns one on // the target output is screen-like (a nested compositor opens one - // host toplevel per screen): place it on an empty output when one - // exists instead of stacking it. - if let Some(client) = toplevel.wl_surface().client() { - let same_client = |w: &Window| { - w.wl_surface() - .and_then(|s| s.client()) - .map_or(false, |c| c.id() == client.id()) - }; - let crowded = self - .space - .elements() - .chain(self.pending_windows.iter()) - .any(|w| window_output_id(w) == target_id && same_client(w)); - if crowded { - let empty = self.output_nodes.iter().map(|n| n.id).find(|oid| { - !self - .space - .elements() - .chain(self.pending_windows.iter()) - .any(|w| window_output_id(w) == *oid) - }); - if let Some(oid) = empty { - target_id = oid; - } + // host toplevel per screen): give it an empty output when one exists, + // and park it otherwise rather than let it cover the screen already + // there. `create_output` hands it the output it is waiting for. + let mut park = false; + if self.would_cover_screen(&window, target_id) { + let empty = self.output_nodes.iter().map(|n| n.id).find(|oid| { + !self + .space + .elements() + .chain(self.pending_windows.iter()) + .any(|w| window_output_id(w) == *oid) + }); + match empty { + Some(oid) => target_id = oid, + None => park = true, } } if let Some(meta) = window_meta(&window) { meta.output.store(target_id, Ordering::Relaxed); + meta.parked.store(park, Ordering::Relaxed); } let (logical_width, logical_height) = if let Some(size) = self.logical_size_of(target_id) { @@ -762,15 +762,24 @@ impl CompositorHandler for AppState { self.send_forced_fullscreen_configure(&tl); } else { let target_id = window_output_id(&window); + let parked = window_meta(&window) + .map(|meta| meta.parked.load(Ordering::Relaxed)) + .unwrap_or(false); let node_idx = self.node_idx_for_id(target_id).unwrap_or(0); let (target_output, pos) = { let node = &self.output_nodes[node_idx]; (node.output.clone(), node.pos) }; - self.space.map_element(window.clone(), pos, true); + // A parked window enters no output and takes no focus: it is waiting for + // an output of its own, and holds its size until one arrives. + self.space.map_element( + window.clone(), + if parked { PARKED_POS } else { pos }, + !parked, + ); window.on_commit(); - { + if !parked { target_output.enter(surface); let scale = target_output.current_scale().fractional_scale(); with_states(surface, |states| { @@ -793,12 +802,12 @@ impl CompositorHandler for AppState { toplevel.send_configure(); } } - } - let serial = next_serial(); - let target = FocusTarget::Window(window.clone()); - if let Some(keyboard) = self.seat.get_keyboard() { - keyboard.set_focus(self, Some(target.clone()), serial); + let serial = next_serial(); + let target = FocusTarget::Window(window.clone()); + if let Some(keyboard) = self.seat.get_keyboard() { + keyboard.set_focus(self, Some(target.clone()), serial); + } } } } @@ -896,6 +905,39 @@ impl AppState { .unwrap_or(0) } + /// Whether `window` would land on top of a screen another window from the same client + /// already occupies. A nested session opens one host toplevel per screen, and a second + /// one covering the first would replace what the display shows rather than add to it. + pub(crate) fn would_cover_screen(&self, window: &Window, id: u32) -> bool { + let Some(client) = window.wl_surface().and_then(|s| s.client()) else { return false }; + self.space + .elements() + .chain(self.pending_windows.iter()) + .filter(|w| !std::ptr::eq(*w, window) && w.wl_surface() != window.wl_surface()) + .any(|w| { + window_output_id(w) == id + && !window_meta(w).map(|m| m.parked.load(Ordering::Relaxed)).unwrap_or(false) + && w.wl_surface() + .and_then(|s| s.client()) + .map_or(false, |c| c.id() == client.id()) + }) + } + + /// Map `window` clear of every output while keeping it tagged for `id`, so it holds its + /// size and its place in the window list without being composited anywhere. A nested + /// session's spare screens wait here until `create_output` gives them one. + pub(crate) fn park_window(&mut self, window: &Window, id: u32) { + let old_id = window_output_id(window); + if let Some(meta) = window_meta(window) { + meta.output.store(id, Ordering::Relaxed); + meta.parked.store(true, Ordering::Relaxed); + } + if let (Some(surface), Some(idx)) = (window.wl_surface(), self.node_idx_for_id(old_id)) { + self.output_nodes[idx].output.leave(&surface); + } + self.space.map_element(window.clone(), PARKED_POS, false); + } + /// Place `window` on output `id`: retag its meta, remap it at the output's layout /// origin, move output enter/leave, push the output's fractional scale, and send the /// forced-fullscreen configure at that output's logical size. @@ -911,6 +953,7 @@ impl AppState { .map(|i| self.output_nodes[i].output.clone()); if let Some(meta) = window_meta(window) { meta.output.store(id, Ordering::Relaxed); + meta.parked.store(false, Ordering::Relaxed); } self.space.map_element(window.clone(), pos, true); if let Some(surface) = window.wl_surface() { @@ -1992,6 +2035,7 @@ impl XdgShellHandler for AppState { id: NEXT_WINDOW_ID.fetch_add(1, Ordering::Relaxed), output: AtomicU32::new(target_id), placed: AtomicBool::new(false), + parked: AtomicBool::new(false), }); self.pending_windows.push(window); let (title, app_id) = with_states(surface.wl_surface(), |states| { From 57d48d135eac170d6d44b6ab7166cb275f24cdf1 Mon Sep 17 00:00:00 2001 From: Seungmin Kim <8457324+ehfd@users.noreply.github.com> Date: Fri, 7 Aug 2026 16:08:52 +0900 Subject: [PATCH 21/21] fix: Linting and regression fixes --- pixelflux/src/encoders/nvenc.rs | 56 ++++++-- pixelflux/src/encoders/oh264.rs | 8 +- pixelflux/src/lib.rs | 225 ++++++++++++++++++++++-------- pixelflux/src/recorder/mp4.rs | 15 +- pixelflux/src/recording_sink.rs | 56 ++++---- pixelflux/src/wayland/dcclient.rs | 13 +- pixelflux/src/wayland/frontend.rs | 22 ++- pixelflux/src/wayland/host.rs | 56 ++------ pixelflux/src/wayland/keymap.rs | 52 ++++++- pixelflux/src/x11/computer_use.rs | 2 +- pixelflux/src/x11/cursor.rs | 3 +- pixelflux/src/x11/mod.rs | 14 +- 12 files changed, 341 insertions(+), 181 deletions(-) diff --git a/pixelflux/src/encoders/nvenc.rs b/pixelflux/src/encoders/nvenc.rs index 2e630a6..f171e94 100644 --- a/pixelflux/src/encoders/nvenc.rs +++ b/pixelflux/src/encoders/nvenc.rs @@ -733,10 +733,10 @@ impl NvencEncoder { /// VBV sizing, optional min/max QP clamps) or ConstQP; infinite GOP (`gopLength` / `idrPeriod` /// = `0xFFFFFFFF`); `zeroReorderDelay` plus a bitstream-restriction VUI (`max_num_reorder_frames=0`) /// so no-reorder decoders don't buffer; an explicit Annex-A level from `nvenc_h264_level` - /// pinned from frame 1 so the level never bumps mid-stream; SMPTE170M VUI colorimetry, - /// matching the fixed BT.601 matrix the hardware ARGB CSC applies; chroma 4:2:0 or 4:4:4 - /// with the matching full-range flag; repeated SPS/PPS; CABAC; no AUD; strict GOP target; - /// and lookahead disabled for real-time latency. + /// pinned from frame 1 so the level never bumps mid-stream; BT.709 VUI primaries and + /// transfer for the sRGB source, with an SMPTE170M matrix and limited range to match the + /// hardware ARGB CSC; chroma 4:2:0 or 4:4:4; repeated SPS/PPS; CABAC; no AUD; strict GOP + /// target; and lookahead disabled for real-time latency. /// 6. **Initialize with resize headroom**: `maxEncodeWidth` / `maxEncodeHeight` are raised to at /// least 4096×2304 (the 5.2 ceiling) so `reconfigure_resolution` can grow in place; this costs /// ~290 MiB of device memory, so a failed init retries at the exact size (in-place resize then @@ -934,15 +934,22 @@ impl NvencEncoder { config.encodeCodecConfig.h264Config.h264VUIParameters.videoFormat = NV_ENC_VUI_VIDEO_FORMAT::NV_ENC_VUI_VIDEO_FORMAT_UNSPECIFIED; config.encodeCodecConfig.h264Config.h264VUIParameters.colourDescriptionPresentFlag = 1; + // Primaries and transfer describe the source, which is sRGB desktop pixels — + // sRGB shares BT.709's primaries and transfer function. Only the matrix follows + // the encoder: NVENC's ARGB hardware CSC is fixed at BT.601, so a client that + // inverts BT.709 shifts saturated colour badly. config.encodeCodecConfig.h264Config.h264VUIParameters.colourPrimaries = - NV_ENC_VUI_COLOR_PRIMARIES::NV_ENC_VUI_COLOR_PRIMARIES_SMPTE170M; + NV_ENC_VUI_COLOR_PRIMARIES::NV_ENC_VUI_COLOR_PRIMARIES_BT709; config.encodeCodecConfig.h264Config.h264VUIParameters.transferCharacteristics = - NV_ENC_VUI_TRANSFER_CHARACTERISTIC::NV_ENC_VUI_TRANSFER_CHARACTERISTIC_SMPTE170M; + NV_ENC_VUI_TRANSFER_CHARACTERISTIC::NV_ENC_VUI_TRANSFER_CHARACTERISTIC_BT709; config.encodeCodecConfig.h264Config.h264VUIParameters.colourMatrix = NV_ENC_VUI_MATRIX_COEFFS::NV_ENC_VUI_MATRIX_COEFFS_SMPTE170M; config.encodeCodecConfig.h264Config.chromaFormatIDC = if is_444 { 3 } else { 1 }; - config.encodeCodecConfig.h264Config.h264VUIParameters.videoFullRangeFlag = - if is_444 { 1 } else { 0 }; + // That same hardware CSC emits limited range in every chroma format. The + // host-planar path could encode full range, but a capture moves between the + // two at runtime when zero-copy demotes to readback, and the declared range + // has to hold across that, so every NVENC session declares limited. + config.encodeCodecConfig.h264Config.h264VUIParameters.videoFullRangeFlag = 0; config.encodeCodecConfig.h264Config.set_repeatSPSPPS(1); config.encodeCodecConfig.h264Config.entropyCodingMode = NV_ENC_H264_ENTROPY_CODING_MODE::NV_ENC_H264_ENTROPY_CODING_MODE_CABAC; @@ -1617,8 +1624,8 @@ impl NvencEncoder { } /// Encode a host ARGB frame by uploading it straight into the ARGB input surface, with no - /// CPU-side colour conversion. NVENC's hardware ARGB→YUV conversion is fixed at the BT.601 - /// matrix, which is why the session VUI declares SMPTE170M; a CPU 601→709 prepass would cost + /// CPU-side colour conversion. NVENC's hardware ARGB→YUV conversion is fixed at BT.601 + /// limited range, which is what the session VUI declares; a CPU prepass to BT.709 would cost /// this path its zero-copy property. /// /// The packed rows are copied host→device into the registered ARGB surface and NVENC's hardware @@ -1949,7 +1956,8 @@ mod gpu_tests { /// first post-resize frame is an IDR (steady frames are P), shrink to 480p, exercise the /// rejection cases (beyond headroom, chroma flip, RC-mode flip) and confirm the session still /// encodes afterward, and optionally dump a decodable stream. Ignored by default; run with - /// `cargo test gpu_ -- --ignored --nocapture`. + /// `cargo test gpu_ -- --ignored --nocapture --test-threads=1` — building NVENC sessions + /// concurrently races in the driver and intermittently faults, so the GPU set runs serially. #[test] #[ignore] fn gpu_resolution_reconfigure_roundtrip() { @@ -2080,6 +2088,32 @@ mod gpu_tests { /// On a real GPU, a session that starts taller than the default 2304 headroom (portrait /// 4K: 2160×4096, within NVENC's 4096 H.264 cap) takes its own size as the `maxEncode` ceiling /// and encodes at that resolution. Ignored by default. + /// The VUI has to describe what the session actually emits. NVENC converts its ARGB + /// input with a fixed BT.601 matrix at limited range in both chroma formats, so the + /// matrix and range follow the hardware; the primaries and transfer follow the source, + /// which is sRGB desktop pixels and therefore BT.709. A client that inverts the wrong + /// matrix, or expands a limited-range frame as full-range, shifts colour visibly. + #[test] + #[ignore] + fn gpu_vui_describes_the_hardware_csc() { + for fullcolor in [false, true] { + let mut s = settings(1280, 720, 60.0); + s.video_fullcolor = fullcolor; + let enc = NvencEncoder::new(&s, ptr::null()).expect("NVENC init"); + // encodeCodecConfig is a union; a successful init leaves the H.264 arm live. + let h264 = unsafe { &enc.encode_config.encodeCodecConfig.h264Config }; + let vui = &h264.h264VUIParameters; + assert_eq!(vui.colourMatrix as u32, + NV_ENC_VUI_MATRIX_COEFFS::NV_ENC_VUI_MATRIX_COEFFS_SMPTE170M as u32); + assert_eq!(vui.colourPrimaries as u32, + NV_ENC_VUI_COLOR_PRIMARIES::NV_ENC_VUI_COLOR_PRIMARIES_BT709 as u32); + assert_eq!(vui.transferCharacteristics as u32, + NV_ENC_VUI_TRANSFER_CHARACTERISTIC::NV_ENC_VUI_TRANSFER_CHARACTERISTIC_BT709 as u32); + assert_eq!(vui.videoFullRangeFlag, 0, "fullcolor={fullcolor}"); + assert_eq!(h264.chromaFormatIDC, if fullcolor { 3 } else { 1 }); + } + } + #[test] #[ignore] fn gpu_init_above_default_headroom() { diff --git a/pixelflux/src/encoders/oh264.rs b/pixelflux/src/encoders/oh264.rs index a766d5e..1923daa 100644 --- a/pixelflux/src/encoders/oh264.rs +++ b/pixelflux/src/encoders/oh264.rs @@ -134,10 +134,10 @@ impl Openh264Encoder { .skip_frames(true) .adaptive_quantization(false) .background_detection(false) - // Scene-change detection would inject IDRs the pipeline never asked for, - // breaking the infinite-GOP / on-demand-keyframe contract (x264 parity: - // `i_scenecut_threshold = 0`). - .scene_change_detect(false) + // Scene-change detection stays on: OpenH264 forces it back on under + // ScreenContentRealTime and warns when asked for anything else, so the + // infinite-GOP contract cannot be tightened from here the way x264's + // `i_scenecut_threshold = 0` allows. .debug(settings.debug_logging) .intra_frame_period(IntraFramePeriod::from_num_frames(INFINITE_INTRA_PERIOD)) .num_threads(threads); diff --git a/pixelflux/src/lib.rs b/pixelflux/src/lib.rs index a683d6a..48d1fc6 100644 --- a/pixelflux/src/lib.rs +++ b/pixelflux/src/lib.rs @@ -511,8 +511,10 @@ pub(crate) fn extract_settings(settings: &Bound<'_, PyAny>) -> PyResult i32 { if v <= 0 { 0 } else { v.min(MAX_CAPTURE_DIM) } }; @@ -604,6 +606,14 @@ pub(crate) fn extract_settings(settings: &Bound<'_, PyAny>) -> PyResult }, /// Reply with every live output as `(id, x, y, width, height, scale, capturing)`. - ListOutputs { reply: std::sync::mpsc::Sender> }, + ListOutputs { reply: std::sync::mpsc::Sender> }, /// Move the window with the given id onto output `output_id` (fullscreened there). MoveWindowToOutput { window_id: u32, output_id: u32, reply: std::sync::mpsc::Sender }, /// Reply with every mapped window as `(window_id, title, app_id, output_id)`. - ListWindows { reply: std::sync::mpsc::Sender> }, + ListWindows { reply: std::sync::mpsc::Sender> }, SetCursorCallback(Py), SetClipboardCallback(Py), /// Server-side clipboard offer: the compositor owns the selection and serves `data` as /// `mime` (plus text aliases) to pasting clients. SetClipboard { mime: String, data: Vec }, KeyboardKey { scancode: u32, state: u32 }, + /// A whole ordered run of key events in one message. Typing a paste one event at a + /// time costs a channel send and a calloop wake per event, which competes with the + /// render loop on the same thread; the caller still decides the sequence. + KeyboardKeys { events: Vec<(u32, u32)> }, /// Set the seat's BASE keymap from a full XKB_KEYMAP_FORMAT_TEXT_V1 string. The /// compositor's keymap policy rebuilds on top: overlay binds are re-spliced onto the new /// base (same keycodes) and the combined keymap is applied in one swap. @@ -666,11 +680,18 @@ pub enum ThreadCommand { }, /// Resolve keysyms to `(keycode, level)` against the seat keymap, overlay-binding every /// keysym the base cannot produce — ONE keymap swap for the whole batch, and a keycode that - /// is currently pressed is never recycled. `(0, 0)` marks an unbindable keysym. + /// is currently pressed is never recycled. `(0, 0)` marks an unbindable keysym. Serves + /// computer-use only; selkies resolves its own keysyms and injects plain keycodes. BindKeysyms { keysyms: Vec, reply: std::sync::mpsc::Sender>, }, + /// Bind explicit `(keycode, keysym)` pairs onto the CURRENT base keymap and deliver it, + /// in one swap. The caller decides the assignment; this only assembles and applies, so the + /// base is neither re-sent nor recompiled and its reverse map is not rebuilt. Ordered with + /// key events on the one command channel, so the keys that need the new binds cannot + /// overtake it — no reply is awaited, and the caller's loop is never blocked on the swap. + SetKeymapOverlay { binds: Vec<(u32, u32)> }, /// Debug/verification readback: currently pressed xkb keycodes plus the modifier state /// bitmask (1 ctrl, 2 shift, 4 alt, 8 logo, 16 caps, 32 num, 64 altgr, 128 level5). GetKeyboardState { @@ -1326,6 +1347,10 @@ fn wayland_encode_loop(pool: &WlFramePool, cfg: WlEncodeConfig) -> Option YuvStandardMatrix::Bt601, GpuEncoder::Vaapi(_) => YuvStandardMatrix::Bt709, }; + // Limited range suits both: NVENC's ARGB CSC emits limited whatever the + // chroma format, so its session declares limited, and VA-API's own + // convert targets tv range as well. + let range = YuvRange::Limited; if settings.video_fullcolor { let y_size = (w * h) as usize; let (y_plane, rest) = nv12_buffer.split_at_mut(y_size); @@ -1341,9 +1366,9 @@ fn wayland_encode_loop(pool: &WlFramePool, cfg: WlEncodeConfig) -> Option Option settings.video_fullcolor, }; let cs_str = if is_444 { "CS_IN:I444" } else { "CS_IN:I420" }; - let range_str = if is_444 { "FR" } else { "LR" }; + // Only the software encoder carries 4:4:4 at full range; NVENC's hardware CSC is + // limited-range whatever the chroma format. + let range_str = if is_444 && video_encoder.is_none() { "FR" } else { "LR" }; let frame_str = if video_encoder.is_some() || openh264 || settings.video_fullframe { "FF" } else { @@ -1592,10 +1619,10 @@ fn log_stream_settings( } }; - let range_str = if is_actually_444 { - "I444 (Full Range)" - } else { - "I420 (Limited Range)" + let range_str = match (is_actually_444, video_encoder.is_none()) { + (true, true) => "I444 (Full Range)", + (true, false) => "I444 (Limited Range)", + _ => "I420 (Limited Range)", }; log_msg.push_str(&format!(" | Colorspace: {}", range_str)); } @@ -2630,7 +2657,7 @@ fn render_node_tick( }, Err(e) => eprintln!("Render error: {:?}", e) } - if let Some(c) = cap.as_deref_mut() { + if let Some(c) = cap { if !damage_rects.is_empty() { c.content_gen += 1; } @@ -2826,7 +2853,7 @@ fn render_node_tick( }, Err(e) => eprintln!("Render error: {:?}", e) } - if let Some(c) = cap.as_deref_mut() { + if let Some(c) = cap { c.render_seq += 1; if render_success { if let Some((id, _)) = pool_slot { @@ -2855,8 +2882,7 @@ fn render_node_tick( let prior = cap .encode_join .take() - .map(|j| j.join().ok().flatten()) - .flatten(); + .and_then(|j| j.join().ok().flatten()); if let Some(pool) = cap.encode_pool.take() { pool.shutdown(); } @@ -3083,8 +3109,12 @@ fn rects_overlap(a: (i32, i32, i32, i32), b: (i32, i32, i32, i32)) -> bool { && ay < by + bh && by < ay + ah } +/// An overlapping output as `(id, flavor, rect)`, where flavor names which rectangle +/// pair matched and rect is `(x, y, width, height)`. +type OutputOverlap = (u32, &'static str, (i32, i32, i32, i32)); + /// The first live output (excluding `skip_id`) whose rectangle overlaps a candidate -/// placement, as `(id, flavor, rect)`. Both rectangle flavors are checked — logical +/// placement. Both rectangle flavors are checked — logical /// (Space layout, scale-divided) and physical (mode pixels at the same origin) — because /// input injection and cursor compositing key off the physical rects while window layout /// keys off the logical ones, and neither may overlap. @@ -3093,7 +3123,7 @@ fn find_output_overlap( skip_id: Option, logical: (i32, i32, i32, i32), physical: (i32, i32, i32, i32), -) -> Option<(u32, &'static str, (i32, i32, i32, i32))> { +) -> Option { for n in nodes { if Some(n.id) == skip_id { continue; @@ -3347,6 +3377,19 @@ fn destroy_output_on(state: &mut AppState, id: u32) -> bool { ); true } +/// Startup inputs for the compositor thread: its two calloop channels, the sender it hands +/// to computer-use, the initial geometry, and the GPU / cursor policy. +struct WaylandThreadConfig { + command_rx: smithay::reexports::calloop::channel::Channel, + wake_rx: smithay::reexports::calloop::channel::Channel<()>, + command_tx: smithay::reexports::calloop::channel::Sender, + initial_width: i32, + initial_height: i32, + explicit_dri_node: String, + auto_gpu_selected: bool, + cursor_size: i32, +} + /// The main execution loop of the Wayland backend. /// @@ -3390,16 +3433,17 @@ fn destroy_output_on(state: &mut AppState, id: u32) -> bool { /// delivery thread — it feeds the delivery sender and must be gone first — and the retained /// callbacks are dropped and gated by a process-shutdown flag so nothing fires into a finalizing /// interpreter. -fn run_wayland_thread( - command_rx: smithay::reexports::calloop::channel::Channel, - wake_rx: smithay::reexports::calloop::channel::Channel<()>, - command_tx: smithay::reexports::calloop::channel::Sender, - initial_width: i32, - initial_height: i32, - explicit_dri_node: String, - auto_gpu_selected: bool, - cursor_size: i32, -) { +fn run_wayland_thread(cfg: WaylandThreadConfig) { + let WaylandThreadConfig { + command_rx, + wake_rx, + command_tx, + initial_width, + initial_height, + explicit_dri_node, + auto_gpu_selected, + cursor_size, + } = cfg; let width: i32 = if initial_width > 0 { initial_width } else { 1024 }; let height: i32 = if initial_height > 0 { initial_height } else { 768 }; @@ -3807,6 +3851,31 @@ fn run_wayland_thread( state.send_cursor_image(&CursorImageStatus::Named(Default::default())); } } + ThreadCommand::KeyboardKeys { events } => { + for (scancode, key_state_val) in events { + if let Some(host) = state.host.as_ref() { + host.key(scancode, key_state_val > 0); + continue; + } + let key_state = if key_state_val > 0 { + KeyState::Pressed + } else { + KeyState::Released + }; + let serial = next_serial(); + let time = wayland_time(); + if let Some(keyboard) = state.seat.get_keyboard() { + keyboard.input( + state, + Keycode::new(scancode), + key_state, + serial, + time, + |_, _, _| FilterResult::<()>::Forward, + ); + } + } + } ThreadCommand::KeyboardKey { scancode, state: key_state_val } => { if let Some(host) = state.host.as_ref() { host.key(scancode, key_state_val > 0); @@ -3822,13 +3891,12 @@ fn run_wayland_thread( } } ThreadCommand::SetKeymapString(text) => { - // Validate before mutating the policy so a bad string leaves the seat - // keymap untouched. - if crate::wayland::keymap::compile_keymap(&text).is_none() { - eprintln!("[Wayland] set_keymap_string: keymap failed to compile; keeping current keymap."); - } else { - state.keymap_policy.rebuild_base(text); + // rebuild_base rejects a string that will not compile without touching + // the policy, so the seat keymap survives a bad one either way. + if state.keymap_policy.rebuild_base(text) { state.apply_keymap_policy(); + } else { + eprintln!("[Wayland] set_keymap_string: keymap failed to compile; keeping current keymap."); } } ThreadCommand::SetXkbLayout { rules, model, layout, variant, options, reply } => { @@ -3847,6 +3915,14 @@ fn run_wayland_thread( ThreadCommand::BindKeysyms { keysyms, reply } => { let _ = reply.send(state.bind_keysyms(&keysyms)); } + ThreadCommand::SetKeymapOverlay { binds } => { + match state.keymap_policy.overlay_text_for(&binds) { + Some(text) => state.apply_keymap_text(text), + None => eprintln!( + "[Wayland] set_keymap_overlay: no base keymap to splice onto." + ), + } + } ThreadCommand::GetKeyboardState { reply } => { let (pressed, mods) = state .seat @@ -4426,7 +4502,16 @@ impl WaylandBackend { let cu_tx = tx.clone(); thread::spawn(move || { crate::boost_thread_priority(-15); - run_wayland_thread(rx, wake_rx, cu_tx, width, height, dri_node, auto_gpu_selected, cursor_size); + run_wayland_thread(WaylandThreadConfig { + command_rx: rx, + wake_rx, + command_tx: cu_tx, + initial_width: width, + initial_height: height, + explicit_dri_node: dri_node, + auto_gpu_selected, + cursor_size, + }); }); WaylandBackend { tx, wake_tx } } @@ -4461,6 +4546,8 @@ impl WaylandBackend { /// live output, or the GPU render target cannot be allocated. Capture reconfigures /// (`start_capture` on an existing output) resize without this validation, so a /// multi-step relayout must stay overlap-free at every step by caller ordering. + // The parameter list is the Python signature; grouping it would change the ABI. + #[allow(clippy::too_many_arguments)] #[pyo3(signature = (id, width, height, x = 0, y = 0, scale = 1.0))] fn create_output( &self, @@ -4507,7 +4594,7 @@ impl WaylandBackend { /// Every live output as `(id, x, y, width, height, scale, capturing)` — width/height in /// physical pixels, `(x, y)` the layout offset. - fn list_outputs(&self, py: Python<'_>) -> PyResult> { + fn list_outputs(&self, py: Python<'_>) -> PyResult> { let (reply_tx, reply_rx) = std::sync::mpsc::channel(); self.send(ThreadCommand::ListOutputs { reply: reply_tx }) .map_err(|e| PyErr::new::(format!("Failed to list outputs: {}", e)))?; @@ -4530,7 +4617,7 @@ impl WaylandBackend { /// Every mapped window as `(window_id, title, app_id, output_id, waiting)`. A waiting /// window is tagged for that output but mapped clear of every one of them, holding its /// size until an output exists for it — a nested session's spare screens. - fn list_windows(&self, py: Python<'_>) -> PyResult> { + fn list_windows(&self, py: Python<'_>) -> PyResult> { let (reply_tx, reply_rx) = std::sync::mpsc::channel(); self.send(ThreadCommand::ListWindows { reply: reply_tx }) .map_err(|e| PyErr::new::(format!("Failed to list windows: {}", e)))?; @@ -4577,9 +4664,32 @@ impl WaylandBackend { Ok(()) } + /// Inject an ordered run of `(keycode, state)` events as one message, so a paste + /// costs one channel send and one wake instead of one per event. + fn inject_keys(&self, events: Vec<(u32, u32)>) -> PyResult<()> { + if events.is_empty() { + return Ok(()); + } + self.send(ThreadCommand::KeyboardKeys { events }) + .map_err(|e| PyErr::new::(format!("Failed to inject keys: {}", e)))?; + Ok(()) + } + /// Swap the seat keyboard's xkb keymap (XKB_KEYMAP_FORMAT_TEXT_V1 text). The caller /// owns keysym-to-keycode policy: define keycodes here, then press them via /// `inject_key`. Ordered with key events on the one compositor channel. + /// Bind explicit `(keycode, keysym)` pairs onto the current base keymap in ONE swap. + /// + /// The caller owns the assignment — which keysym goes to which keycode, and when to + /// recycle one — because that tracks layouts and user reports. This end only assembles + /// the xkb text and delivers it, reusing the installed base rather than recompiling a + /// re-supplied one. False when no base keymap is installed yet. + fn set_keymap_overlay(&self, binds: Vec<(u32, u32)>) -> PyResult<()> { + self.send(ThreadCommand::SetKeymapOverlay { binds }) + .map_err(|e| PyErr::new::(format!("Failed to set overlay: {}", e)))?; + Ok(()) + } + fn set_keymap_string(&self, text: String) -> PyResult<()> { self.send(ThreadCommand::SetKeymapString(text)) .map_err(|e| PyErr::new::(format!("Failed to set keymap: {}", e)))?; @@ -4674,19 +4784,6 @@ impl WaylandBackend { .unwrap_or(false)) } - /// Resolve `keysyms` to `(keycode, level)` pairs against the seat keymap, overlay-binding - /// every keysym the base cannot produce. Binding N new keysyms costs ONE keymap swap, and a - /// keycode currently held down is never rebound. `(0, 0)` marks an unbindable keysym. - fn bind_keysyms(&self, py: Python<'_>, keysyms: Vec) -> PyResult> { - let n = keysyms.len(); - let (reply_tx, reply_rx) = std::sync::mpsc::channel::>(); - self.send(ThreadCommand::BindKeysyms { keysyms, reply: reply_tx }) - .map_err(|e| PyErr::new::(format!("Failed to bind keysyms: {}", e)))?; - Ok(py - .detach(move || reply_rx.recv_timeout(Duration::from_secs(2))) - .unwrap_or_else(|_| vec![(0, 0); n])) - } - /// Debug/verification readback of the seat keyboard: `(pressed_keycodes, modifier_mask)` /// with mask bits 1 ctrl, 2 shift, 4 alt, 8 logo, 16 caps, 32 num, 64 altgr, 128 level5. fn get_keyboard_state(&self, py: Python<'_>) -> PyResult<(Vec, u32)> { @@ -5554,9 +5651,20 @@ impl ScreenCapture { fn inject_key(&self, py: Python<'_>, scancode: u32, state: u32) -> PyResult<()> { wayland_backend_running(py).map_or(Ok(()), |be| be.bind(py).borrow().inject_key(scancode, state)) } + /// Inject an ordered run of `(keycode, state)` events in one message. + fn inject_keys(&self, py: Python<'_>, events: Vec<(u32, u32)>) -> PyResult<()> { + wayland_backend_running(py).map_or(Ok(()), |be| be.bind(py).borrow().inject_keys(events)) + } fn set_keymap_string(&self, py: Python<'_>, text: String) -> PyResult<()> { wayland_backend_running(py).map_or(Ok(()), |be| be.bind(py).borrow().set_keymap_string(text)) } + /// Bind explicit `(keycode, keysym)` pairs onto the current base keymap in one swap; + /// false when no backend runs or no base keymap is installed. See the backend method + /// for the split of responsibility. + fn set_keymap_overlay(&self, py: Python<'_>, binds: Vec<(u32, u32)>) -> PyResult<()> { + wayland_backend_running(py) + .map_or(Ok(()), |be| be.bind(py).borrow().set_keymap_overlay(binds)) + } /// Compositor apps run under (a nested labwc/kwin session pixelflux captures), /// the target for Computer-Use text injection; selkies resolves it and hands it /// over. Empty clears it. Stored process-wide since the CU server is per-process. @@ -5723,13 +5831,6 @@ impl ScreenCapture { be.bind(py).borrow().set_xkb_layout(py, layout, variant, options, model, rules) }) } - /// Resolve keysyms to `(keycode, level)` with batched overlay binding (one keymap swap - /// per call, held keycodes never rebound); all `(0, 0)` when no backend runs. - fn bind_keysyms(&self, py: Python<'_>, keysyms: Vec) -> PyResult> { - let n = keysyms.len(); - wayland_backend_running(py) - .map_or(Ok(vec![(0, 0); n]), |be| be.bind(py).borrow().bind_keysyms(py, keysyms)) - } /// Seat keyboard readback: `(pressed_keycodes, modifier_mask)`; empty when no backend runs. fn get_keyboard_state(&self, py: Python<'_>) -> PyResult<(Vec, u32)> { wayland_backend_running(py) @@ -5744,6 +5845,8 @@ impl ScreenCapture { } /// Create an additional Wayland output (see `WaylandBackend.create_output`); false when /// no backend runs. + // The parameter list is the Python signature; grouping it would change the ABI. + #[allow(clippy::too_many_arguments)] #[pyo3(signature = (id, width, height, x = 0, y = 0, scale = 1.0))] fn create_output( &self, @@ -5778,7 +5881,7 @@ impl ScreenCapture { } /// Every live Wayland output as `(id, x, y, width, height, scale, capturing)`; empty /// when no backend runs. - fn list_outputs(&self, py: Python<'_>) -> PyResult> { + fn list_outputs(&self, py: Python<'_>) -> PyResult> { wayland_backend_running(py) .map_or(Ok(Vec::new()), |be| be.bind(py).borrow().list_outputs(py)) } @@ -5790,7 +5893,7 @@ impl ScreenCapture { } /// Every mapped window as `(window_id, title, app_id, output_id, waiting)`; empty when /// no backend runs. - fn list_windows(&self, py: Python<'_>) -> PyResult> { + fn list_windows(&self, py: Python<'_>) -> PyResult> { wayland_backend_running(py) .map_or(Ok(Vec::new()), |be| be.bind(py).borrow().list_windows(py)) } diff --git a/pixelflux/src/recorder/mp4.rs b/pixelflux/src/recorder/mp4.rs index 5e96c13..f1a2297 100644 --- a/pixelflux/src/recorder/mp4.rs +++ b/pixelflux/src/recorder/mp4.rs @@ -563,13 +563,14 @@ impl H264SampleBuilder { let pps = self.pps.as_deref()?; let (width, height) = parse_sps_dimensions(sps)?; - let mut avcc_p = Vec::new(); - avcc_p.push(1); // configurationVersion - avcc_p.push(sps[1]); // AVCProfileIndication - avcc_p.push(sps[2]); // profile_compatibility - avcc_p.push(sps[3]); // AVCLevelIndication - avcc_p.push(0xff); // lengthSizeMinusOne = 3 - avcc_p.push(0xe1); // numOfSequenceParameterSets = 1 + let mut avcc_p = vec![ + 1, // configurationVersion + sps[1], // AVCProfileIndication + sps[2], // profile_compatibility + sps[3], // AVCLevelIndication + 0xff, // lengthSizeMinusOne = 3 + 0xe1, // numOfSequenceParameterSets = 1 + ]; avcc_p.extend_from_slice(&(sps.len() as u16).to_be_bytes()); avcc_p.extend_from_slice(sps); avcc_p.push(1); // numOfPictureParameterSets diff --git a/pixelflux/src/recording_sink.rs b/pixelflux/src/recording_sink.rs index ac96c25..b4eb426 100644 --- a/pixelflux/src/recording_sink.rs +++ b/pixelflux/src/recording_sink.rs @@ -234,6 +234,34 @@ impl Drop for RecordingSink { } } +/// Write one whole frame to a recorder's socket, resuming across the soft timeouts a slow reader +/// induces so a partial Annex-B NAL is never left behind. Aborts if `stop` is set (the client was +/// dropped by [`RecordingSink::write_encoded_frame`]) or a hard error occurs. +fn write_all_frame(stream: &mut W, buf: &[u8], stop: &AtomicBool) -> std::io::Result<()> { + let mut written = 0usize; + while written < buf.len() { + if stop.load(Ordering::Relaxed) { + return Err(std::io::Error::other("writer stopped (client dropped)")); + } + match stream.write(&buf[written..]) { + Ok(0) => { + return Err(std::io::Error::new( + ErrorKind::WriteZero, + "failed to write whole frame", + )); + } + Ok(n) => written += n, + Err(ref e) if e.kind() == ErrorKind::TimedOut => {} + Err(ref e) if e.kind() == ErrorKind::WouldBlock => { + thread::sleep(Duration::from_millis(5)); + } + Err(ref e) if e.kind() == ErrorKind::Interrupted => {} + Err(e) => return Err(e), + } + } + Ok(()) +} + #[cfg(test)] mod cost_tests { //! The sink's isolation contract, measured: feeding a frame must cost nothing when no @@ -314,31 +342,3 @@ mod cost_tests { ); } } - -/// Write one whole frame to a recorder's socket, resuming across the soft timeouts a slow reader -/// induces so a partial Annex-B NAL is never left behind. Aborts if `stop` is set (the client was -/// dropped by [`RecordingSink::write_encoded_frame`]) or a hard error occurs. -fn write_all_frame(stream: &mut W, buf: &[u8], stop: &AtomicBool) -> std::io::Result<()> { - let mut written = 0usize; - while written < buf.len() { - if stop.load(Ordering::Relaxed) { - return Err(std::io::Error::other("writer stopped (client dropped)")); - } - match stream.write(&buf[written..]) { - Ok(0) => { - return Err(std::io::Error::new( - ErrorKind::WriteZero, - "failed to write whole frame", - )); - } - Ok(n) => written += n, - Err(ref e) if e.kind() == ErrorKind::TimedOut => {} - Err(ref e) if e.kind() == ErrorKind::WouldBlock => { - thread::sleep(Duration::from_millis(5)); - } - Err(ref e) if e.kind() == ErrorKind::Interrupted => {} - Err(e) => return Err(e), - } - } - Ok(()) -} diff --git a/pixelflux/src/wayland/dcclient.rs b/pixelflux/src/wayland/dcclient.rs index 286b139..bc442ec 100644 --- a/pixelflux/src/wayland/dcclient.rs +++ b/pixelflux/src/wayland/dcclient.rs @@ -212,7 +212,7 @@ pub(crate) fn read(socket_path: &str, mime: &str) -> Result>, Str device.destroy(); return Ok(None); }; - let offered = state.offer_mimes.get(&offer.id()).map_or(false, |m| m.iter().any(|x| x == mime)); + let offered = state.offer_mimes.get(&offer.id()).is_some_and(|m| m.iter().any(|x| x == mime)); if !offered { device.destroy(); return Ok(None); @@ -272,14 +272,11 @@ fn serve_selection(socket_path: &str, entries: Vec<(String, Vec)>) -> Result // Sends are served inside dispatch; block until the compositor has // something (with a poll so a dead compositor can't pin the thread). queue.flush().map_err(|e| format!("flush: {e}"))?; - match conn.prepare_read() { - Some(guard) => { - use std::os::fd::AsRawFd; - if wait_readable(guard.connection_fd().as_raw_fd(), STOP_POLL)? { - guard.read().map_err(|e| format!("read: {e}"))?; - } + if let Some(guard) = conn.prepare_read() { + use std::os::fd::AsRawFd; + if wait_readable(guard.connection_fd().as_raw_fd(), STOP_POLL)? { + guard.read().map_err(|e| format!("read: {e}"))?; } - None => {} } queue.dispatch_pending(&mut state).map_err(|e| format!("dispatch: {e}"))?; } diff --git a/pixelflux/src/wayland/frontend.rs b/pixelflux/src/wayland/frontend.rs index 7636079..512179b 100644 --- a/pixelflux/src/wayland/frontend.rs +++ b/pixelflux/src/wayland/frontend.rs @@ -327,6 +327,10 @@ pub fn window_output_id(window: &Window) -> u32 { window_meta(window).map(|m| m.output.load(Ordering::Relaxed)).unwrap_or(0) } +/// A queued computer-use screenshot as `(display id, reply)`; the reply carries the +/// encoded image or the reason it could not be produced. +pub type ScreenshotRequest = (u32, std::sync::mpsc::Sender, String>>); + /// Central context threaded through every Smithay handler; owns the Wayland globals, the /// GBM/EGL (or pixman) renderer state, and the capture/encode pipeline state. /// @@ -446,9 +450,9 @@ pub struct AppState { pub pointer_constraints_state: PointerConstraintsState, pub render_node_path: String, pub auto_gpu_selected: bool, - /// Computer-use screenshot request `(display id, reply)`; served from that output's - /// next render (the id was validated live when the request was queued). - pub pending_screenshot: Option<(u32, std::sync::mpsc::Sender, String>>)>, + /// Computer-use screenshot request; served from that output's next render (the id + /// was validated live when the request was queued). + pub pending_screenshot: Option, /// The command channel, drained in place (wakeups arrive on a separate ping channel) so /// the render tick can apply every queued command BEFORE starting a long render/encode — /// queued input is never starved behind the tick it arrived during. @@ -919,7 +923,7 @@ impl AppState { && !window_meta(w).map(|m| m.parked.load(Ordering::Relaxed)).unwrap_or(false) && w.wl_surface() .and_then(|s| s.client()) - .map_or(false, |c| c.id() == client.id()) + .is_some_and(|c| c.id() == client.id()) }) } @@ -1080,6 +1084,7 @@ impl AppState { /// the whole pool would collide; ship the raw pool bytes plus descriptor to the worker. /// - **dmabuf** (`NotManaged`): bind it to the GLES renderer, copy the framebuffer to /// `Abgr8888`, map it back (calloop-affine), and ship the raw RGBA readback. + /// /// A sprite whose buffer could not be read is dropped by the worker, preserving the /// consumer's last cursor instead of blanking it (only "hide" carries empty data). pub(crate) fn send_cursor_image(&mut self, image: &CursorImageStatus) { @@ -1218,6 +1223,12 @@ impl AppState { /// clients only when the content actually changed (smithay dedupes by content hash). pub(crate) fn apply_keymap_policy(&mut self) { let text = self.keymap_policy.keymap_text(); + self.apply_keymap_text(text); + } + + /// Deliver `text` as the seat keymap (and the host virtual keyboard's, in + /// host-capture mode). Empty text is ignored. + pub(crate) fn apply_keymap_text(&mut self, text: String) { if text.is_empty() { return; } @@ -1236,7 +1247,8 @@ impl AppState { /// Resolve `keysyms` to `(keycode, level)` pairs, overlay-binding whatever the base /// cannot produce — at most ONE keymap swap for the whole batch, and never rebinding a - /// keycode that is currently held down. + /// keycode that is currently held down. Serves computer-use; the interactive input path + /// resolves keysyms in selkies and injects plain keycodes. pub(crate) fn bind_keysyms(&mut self, keysyms: &[u32]) -> Vec<(u32, u32)> { let pressed: std::collections::HashSet = self .seat diff --git a/pixelflux/src/wayland/host.rs b/pixelflux/src/wayland/host.rs index 1b3e11f..5247699 100644 --- a/pixelflux/src/wayland/host.rs +++ b/pixelflux/src/wayland/host.rs @@ -165,6 +165,7 @@ enum CtrlMsg { // Control connection: seat + virtual input devices + wlr-output-management. // --------------------------------------------------------------------------- +#[derive(Default)] struct CtrlState { seat: Option, vk_mgr: Option, @@ -182,24 +183,6 @@ struct CtrlState { sync_done: bool, } -impl Default for CtrlState { - fn default() -> Self { - Self { - seat: None, - vk_mgr: None, - vptr_mgr: None, - has_screencopy: false, - outputs: Vec::new(), - order: Vec::new(), - output_mgr: None, - heads: Vec::new(), - om_serial: None, - cfg_result: None, - cfg_cancelled: false, - sync_done: false, - } - } -} /// Natural sort key for an output name: text prefix plus trailing number, so /// HEADLESS-2 orders before HEADLESS-10. Unnamed outputs keep registry order @@ -357,6 +340,7 @@ impl Dispatch for CtrlState { // Capture connections: one per host output (screencopy + buffer allocation). // --------------------------------------------------------------------------- +#[derive(Default)] struct CaptureState { shm: Option, dmabuf: Option, @@ -372,23 +356,6 @@ struct CaptureState { sync_done: bool, } -impl Default for CaptureState { - fn default() -> Self { - Self { - shm: None, - dmabuf: None, - screencopy: None, - outputs: Vec::new(), - announce_dmabuf: None, - announce_shm: None, - buffer_done: false, - damage: Vec::new(), - ready: false, - failed: false, - sync_done: false, - } - } -} impl CaptureState { fn reset_frame(&mut self) { @@ -480,10 +447,10 @@ impl Dispatch for CaptureState { _: &QueueHandle, ) { match event { - zwlr_screencopy_frame_v1::Event::Buffer { format, width, height, stride } => { - if let WEnum::Value(f) = format { - state.announce_shm = Some((f as u32, width as i32, height as i32, stride as i32)); - } + zwlr_screencopy_frame_v1::Event::Buffer { + format: WEnum::Value(f), width, height, stride, + } => { + state.announce_shm = Some((f as u32, width as i32, height as i32, stride as i32)); } zwlr_screencopy_frame_v1::Event::LinuxDmabuf { format, width, height } => { state.announce_dmabuf = Some((format, width as i32, height as i32)); @@ -749,14 +716,9 @@ impl HostSession { pub fn try_take_frame(&self, display_id: u32) -> Option { let handle = self.outputs.get(self.output_index_for(display_id)?)?; let mut newest: Option = None; - loop { - match handle.frames.try_recv() { - Ok(frame) => { - if let Some(stale) = newest.replace(frame) { - handle.send(ToHost::Release { gen: stale.gen, slot: stale.slot }); - } - } - Err(TryRecvError::Empty) | Err(TryRecvError::Disconnected) => break, + while let Ok(frame) = handle.frames.try_recv() { + if let Some(stale) = newest.replace(frame) { + handle.send(ToHost::Release { gen: stale.gen, slot: stale.slot }); } } newest diff --git a/pixelflux/src/wayland/keymap.rs b/pixelflux/src/wayland/keymap.rs index fe3185e..0233284 100644 --- a/pixelflux/src/wayland/keymap.rs +++ b/pixelflux/src/wayland/keymap.rs @@ -132,9 +132,15 @@ impl KeymapPolicy { /// Replace the base keymap text and rebuild the reverse map. Overlay assignments are /// retained (same keycodes), so keycodes already handed out stay valid across the swap. - pub fn rebuild_base(&mut self, base_text: String) { + /// Returns whether `base_text` compiled. A string that does not is rejected without + /// touching any state, so the caller needs no separate validation pass — compiling a + /// keymap is the expensive part of installing one and doing it twice is pure cost. + pub fn rebuild_base(&mut self, base_text: String) -> bool { + let Some(keymap) = compile_keymap(&base_text) else { + return false; + }; self.base_map.clear(); - if let Some(keymap) = compile_keymap(&base_text) { + { let lo = keymap.min_keycode().raw(); let hi = keymap.max_keycode().raw(); // Lower levels win across ALL keys, so a keysym reachable unshifted never @@ -155,6 +161,7 @@ impl KeymapPolicy { } } self.base_text = base_text; + true } /// True once a base keymap has been installed. @@ -260,6 +267,47 @@ impl KeymapPolicy { /// The full seat keymap: the base text with every occupied overlay slot spliced into the /// `xkb_keycodes` and `xkb_symbols` sections (and `maximum` raised to cover them). With no /// overlays, the base text verbatim. + /// Splice explicit `(keycode, keysym)` binds onto the current base text. + /// + /// Which keysym lands on which keycode is the caller's decision — that part tracks + /// layouts and user reports and belongs where it can be changed quickly. Assembling + /// the xkb text is fixed grammar, so it is done here: no 30 KB string is built in the + /// caller, and the base keymap is reused rather than re-supplied and recompiled. + /// `None` when the base has no recognizable keycodes/symbols sections. + pub fn overlay_text_for(&self, binds: &[(u32, u32)]) -> Option { + if self.base_text.is_empty() { + return None; + } + if binds.is_empty() { + return Some(self.base_text.clone()); + } + let base = &self.base_text; + let max_at = base.find("maximum = ")?; + let num_at = max_at + "maximum = ".len(); + let num_len = base[num_at..].find(';')?; + let old_max: u32 = base[num_at..num_at + num_len].trim().parse().unwrap_or(255); + let need_max = binds.iter().map(|&(kc, _)| kc).max().unwrap_or(0); + let mut text = String::with_capacity(base.len() + binds.len() * 48); + text.push_str(&base[..num_at]); + text.push_str(&old_max.max(need_max).to_string()); + let rest = &base[num_at + num_len..]; + let kc_end = rest.find("};")?; + text.push_str(&rest[..kc_end]); + for &(kc, _) in binds { + text.push_str(&format!("\t = {kc};\n")); + } + let rest = &rest[kc_end..]; + let close_at = rest + .find("xkb_symbols") + .and_then(|sym_at| Self::section_close(rest, sym_at))?; + text.push_str(&rest[..close_at]); + for &(kc, sym) in binds { + text.push_str(&format!("\tkey {{ [ {sym:#x} ] }};\n")); + } + text.push_str(&rest[close_at..]); + Some(text) + } + pub fn keymap_text(&self) -> String { let occupied: Vec<(usize, u32)> = self .slots diff --git a/pixelflux/src/x11/computer_use.rs b/pixelflux/src/x11/computer_use.rs index 52f4d11..6e172bc 100644 --- a/pixelflux/src/x11/computer_use.rs +++ b/pixelflux/src/x11/computer_use.rs @@ -299,7 +299,7 @@ impl CuX11Backend { // integration mirrors a core single-group binding into the group-2 columns, // so the refetch shows `sym` at more levels than the bind wrote. let still_ours = - cur.iter().any(|&s| s == sym) && cur.iter().all(|&s| s == 0 || s == sym); + cur.contains(&sym) && cur.iter().all(|&s| s == 0 || s == sym); if still_ours { cur.fill(0); changed = true; diff --git a/pixelflux/src/x11/cursor.rs b/pixelflux/src/x11/cursor.rs index 970f01a..5f929a0 100644 --- a/pixelflux/src/x11/cursor.rs +++ b/pixelflux/src/x11/cursor.rs @@ -444,7 +444,8 @@ mod tests { #[test] fn out_of_bbox_hotspot_clamped() { let mut px = vec![0u32; 16]; - px[1 * 4 + 2] = 0xFF00_0000; + let (x, y) = (2usize, 1usize); + px[y * 4 + x] = 0xFF00_0000; // Visible pixel at (2,1) only. Hotspot (0,0): rebased (-2,-1) -> (0,0). let (t, _, hx, hy) = cursor_to_png(&reply(4, 4, 0, 0, px.clone()), 32); assert_eq!(t, "png"); diff --git a/pixelflux/src/x11/mod.rs b/pixelflux/src/x11/mod.rs index 459a06a..4a9a486 100644 --- a/pixelflux/src/x11/mod.rs +++ b/pixelflux/src/x11/mod.rs @@ -1299,12 +1299,14 @@ mod cursor_tests { #[test] fn resolve_dims_survives_past_end_offset() { - let mut s = RustCaptureSettings::default(); - s.capture_x = 100000; - s.capture_y = 100000; - s.width = 1920; - s.height = 1080; - s.output_mode = 1; + let s = RustCaptureSettings { + capture_x: 100000, + capture_y: 100000, + width: 1920, + height: 1080, + output_mode: 1, + ..Default::default() + }; let (w, h) = resolve_dims(1920, 1080, &s); assert!(w >= 2 && h >= 2, "a past-end offset now degrades to a small grab, not death"); assert_eq!(w % 2, 0);