diff --git a/.github/dependabot.yml b/.github/dependabot.yml new file mode 100644 index 0000000..c2a038d --- /dev/null +++ b/.github/dependabot.yml @@ -0,0 +1,36 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at https://mozilla.org/MPL/2.0/. + +# One grouped weekly pull request per ecosystem, so a crate bump does not arrive +# as a dozen separate reviews. +version: 2 +updates: + - package-ecosystem: github-actions + directory: / + schedule: + interval: weekly + groups: + actions: + patterns: ["*"] + + - package-ecosystem: cargo + directories: ["/pixelflux", "/pixelflux/nvcodec-sys"] + schedule: + interval: weekly + groups: + crates: + patterns: ["*"] + ignore: + # Pinned to the version that produced the committed NVENC bindings + - dependency-name: bindgen + + # The devcontainers updater discovers `.devcontainer/**/devcontainer.json` + # itself; naming that directory makes it report the file as not found + - package-ecosystem: devcontainers + directory: / + schedule: + interval: weekly + groups: + devcontainer-features: + patterns: ["*"] diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml index 3952e5f..05321ac 100644 --- a/.github/workflows/docs.yml +++ b/.github/workflows/docs.yml @@ -10,7 +10,7 @@ jobs: permissions: contents: write steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v7 - name: Install Rust uses: dtolnay/rust-toolchain@stable @@ -29,11 +29,11 @@ jobs: run: | curl -L -o /tmp/miniforge.sh \ "https://github.com/conda-forge/miniforge/releases/latest/download/Miniforge3-$(uname)-$(uname -m).sh" - bash /tmp/miniforge.sh -b -p $HOME/miniforge3 - source $HOME/miniforge3/etc/profile.d/conda.sh + bash /tmp/miniforge.sh -b -p "$HOME/miniforge3" + source "$HOME/miniforge3/etc/profile.d/conda.sh" conda create -y -n pixelflux-docs -c conda-forge "ffmpeg=8.1" conda activate pixelflux-docs - echo "PKG_CONFIG_PATH=$CONDA_PREFIX/lib/pkgconfig:${PKG_CONFIG_PATH:-}" >> $GITHUB_ENV + echo "PKG_CONFIG_PATH=$CONDA_PREFIX/lib/pkgconfig:${PKG_CONFIG_PATH:-}" >> "$GITHUB_ENV" - name: Build docs run: cargo doc --no-deps diff --git a/.github/workflows/pre-release.yml b/.github/workflows/pre-release.yml index 99cd916..5eca9aa 100644 --- a/.github/workflows/pre-release.yml +++ b/.github/workflows/pre-release.yml @@ -4,6 +4,9 @@ on: push: branches: [master] +permissions: + contents: read + jobs: build: strategy: @@ -52,14 +55,14 @@ jobs: cp314-musllinux_aarch64 runs-on: ${{ matrix.runner }} steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v7 - name: Build wheels - uses: pypa/cibuildwheel@v4.1.0 + uses: pypa/cibuildwheel@v4.2.0 env: CIBW_BUILD: ${{ matrix.build }} - - uses: actions/upload-artifact@v4 + - uses: actions/upload-artifact@v7 with: name: wheels-${{ matrix.arch }}-${{ matrix.platform }} path: wheelhouse/*.whl @@ -70,7 +73,7 @@ jobs: permissions: contents: write steps: - - uses: actions/download-artifact@v4 + - uses: actions/download-artifact@v8 with: pattern: wheels-* path: wheelhouse @@ -81,7 +84,7 @@ jobs: run: echo "short=$(echo '${{ github.sha }}' | cut -c1-7)" >> "$GITHUB_OUTPUT" - name: Create pre-release - uses: softprops/action-gh-release@v2 + uses: softprops/action-gh-release@v3 with: tag_name: ${{ steps.sha.outputs.short }} name: ${{ steps.sha.outputs.short }} diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index cd4b564..3bef997 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -5,11 +5,14 @@ on: tags: - '[0-9]+.[0-9]+.[0-9]*' +permissions: + contents: read + jobs: check: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v7 - name: Verify release notes exist run: | @@ -67,14 +70,14 @@ jobs: cp314-musllinux_aarch64 runs-on: ${{ matrix.runner }} steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v7 - name: Build wheels - uses: pypa/cibuildwheel@v4.1.0 + uses: pypa/cibuildwheel@v4.2.0 env: CIBW_BUILD: ${{ matrix.build }} - - uses: actions/upload-artifact@v4 + - uses: actions/upload-artifact@v7 with: name: wheels-${{ matrix.arch }}-${{ matrix.platform }} path: wheelhouse/*.whl @@ -85,9 +88,9 @@ jobs: permissions: contents: write steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v7 - - uses: actions/download-artifact@v4 + - uses: actions/download-artifact@v8 with: pattern: wheels-* path: wheelhouse @@ -104,7 +107,7 @@ jobs: } >> "$GITHUB_OUTPUT" - name: Create release - uses: softprops/action-gh-release@v2 + uses: softprops/action-gh-release@v3 with: name: ${{ github.ref_name }} body: ${{ steps.notes.outputs.content }} diff --git a/MANIFEST.in b/MANIFEST.in index a606bf7..0127132 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -5,6 +5,7 @@ recursive-include pixelflux/nvcodec-sys * recursive-exclude pixelflux *.so *.pyc prune pixelflux/target prune pixelflux/nvcodec-sys/target +include LICENSE include pyproject.toml include README.md include setup.py diff --git a/README.md b/README.md index 37f7f27..edb202e 100644 --- a/README.md +++ b/README.md @@ -8,11 +8,11 @@ This module provides a Python interface to a high-performance capture library supporting both **X11** and **Wayland** environments. It captures pixel data, detects changes, and encodes modified stripes into JPEG or H.264. -It supports CPU-based encoding (x264, JPEG) as well as hardware-accelerated H.264 encoding via NVIDIA's NVENC and VA-API for Intel/AMD GPUs. Both backends share a **zero-copy pipeline** that minimizes copies and latency end to end. +It supports CPU-based encoding (x264 JPEG, or the BSD-licensed OpenH264) as well as hardware-accelerated H.264 encoding via NVIDIA's NVENC and VA-API for Intel/AMD GPUs. **About "zero copy":** the Wayland GPU path is truly zero-copy (dmabuf frames flow GBM → encoder without touching system RAM). The X11 path copies **exactly once**: the X server renders each frame into a shared-memory surface (`XShmGetImage`); the encoder threads then read that mapped surface **in place** and pass the encoded bytes to Python through the buffer protocol without any further copies. ## Installation -pixelflux is a single self-contained **Rust** extension (no C/C++ sources) compiled during installation. Both the X11 and Wayland backends, all encoders, and the Python API live in it. +pixelflux is a single self-contained **Rust** extension compiled during installation. Both the X11 and Wayland backends, all encoders, and the Python API live in it. (`libjpeg-turbo` and `openh264` C sources are vendored and built by their Rust `-sys` crates, so `cmake` and `nasm` are required, but no system copies of those libraries are used.) ### 1. Prerequisites @@ -34,7 +34,6 @@ sudo apt-get install -y \ libavcodec-dev \ libavutil-dev \ libx264-dev \ - libturbojpeg0-dev \ libgbm-dev \ libdrm-dev \ libwayland-dev \ @@ -43,7 +42,13 @@ sudo apt-get install -y \ libva-dev ``` -> **Notes:** the FFmpeg bindings (`ffmpeg-sys-next` 8.1) work with any system **FFmpeg 6.0–8.1** (only `avcodec`/`avfilter` are used, for `h264_vaapi`); on distros shipping an older FFmpeg, install a newer build and point `PKG_CONFIG_PATH` at it. X11 capture uses pure-Rust XCB (no `libX11`/`libxcb`/`Xfixes` dev packages needed); colorspace conversion is pure-Rust and the NVENC/CUDA libraries are loaded at runtime (no compile-time NVIDIA packages). +> **Notes:** the FFmpeg bindings (`ffmpeg-sys-next` 8.1) work with any system **FFmpeg 6.0–8.1 built LGPL-only** (only `avcodec`/`avfilter` are used, for `h264_vaapi`; `--enable-gpl` is deliberately never passed to upstream FFmpeg builds); on distros shipping an older FFmpeg, install a newer build and point `PKG_CONFIG_PATH` at it. `libjpeg-turbo` is vendored and built statically by its crate — **no `libturbojpeg` system package is needed** (only `cmake` + `nasm`). X11 capture uses pure-Rust XCB; colorspace conversion is pure-Rust and the NVENC/CUDA libraries are loaded at runtime (no compile-time NVIDIA packages). +> +> **GPL component (`libx264`):** striped software H.264 uses the system `libx264` (GPL-2.0+), which is the only GPL-licensed dependency **of pixelflux itself**. It is enabled **by default**; to build without it, set `PIXELFLUX_ENABLE_GPL=0` (or `=false`) before `pip install`. You then lose the striped/4:4:4 software H.264 path (the default CPU H.264 path) and `libx264-dev` is no longer required; **kept** are all JPEG modes, the BSD-licensed OpenH264 full-frame software H.264 (`use_openh264 = True`), NVENC, and VA-API. A notice is printed at install time whether GPL components are enabled or not. +> +> **Caveat (transitively-linked x264):** the extension links the *system* FFmpeg (`libavcodec`/`libavfilter`) for VA-API, and many distro FFmpeg builds (e.g. Ubuntu/Debian's) are themselves compiled with `--enable-libx264`, so their `libavcodec` drags `libx264` in as a transitive shared-library dependency even when pixelflux was built GPL-free. pixelflux contains no x264 code in that case (verified: no `x264` symbols or `NEEDED` entries), for a deployment that must be x264-free end to end, use an FFmpeg built without `--enable-libx264` (the project's wheel CI builds FFmpeg n8.1 LGPL-only, and the project's AppImage bundles such a variant). +> +> **Official wheels are always GPL-enabled** (x264 enabled, LGPL-only FFmpeg); the `PIXELFLUX_ENABLE_GPL=0` path is for verified license-minimal source builds. The AppImage distribution bundles the LGPL-only FFmpeg variant so the optional GPL-free posture holds end-to-end. ### 2. Hardware Acceleration (Optional but Recommended) * **NVIDIA (NVENC):** The library detects the NVIDIA driver at runtime. No extra compile-time packages are needed. @@ -56,6 +61,8 @@ sudo apt-get install -y \ pip install pixelflux ``` +Prebuilt wheels are published on the GitHub Releases page (`manylinux_2_28` and `musllinux`, x86_64 and aarch64, CPython 3.9–3.14). PyPI serves them automatically on supported platforms; on other platforms pip builds from source with the prerequisites above. + **Option B: Install from local source** ```bash # From the root of the project repository @@ -113,6 +120,7 @@ settings.capture_y = 0 settings.capture_cursor = True settings.target_fps = 60.0 settings.scale = 1.0 # Fractional scaling (Wayland only) +settings.wayland_host_display = "" # Capture from an EXTERNAL wlroots compositor instead of the built-in one (host-capture mode) # --- Encoding Mode --- # 0 for JPEG, 1 for H.264 @@ -134,6 +142,8 @@ settings.video_paintover_burst_frames = 5 # Number of high-quality fram settings.video_fullcolor = False # Use I444/full color (High 4:4:4) instead of I420. Supported by software encoding and NVENC. settings.video_fullframe = True # Encode full frames (required for HW accel) instead of just changed stripes settings.video_streaming_mode = False # Bypass all VNC logic and work like a normal video encoder, higher constant CPU usage for fullscreen gaming/videos +settings.use_openh264 = False # Use Cisco OpenH264 (BSD-licensed) for software full-frame H.264 instead of x264 (GPL). Required for software H.264 when built with PIXELFLUX_ENABLE_GPL=0 +settings.keyframe_interval_s = 0.0 # Periodic keyframe interval in seconds (0 = keyframes only on demand/paint-over) settings.video_cbr_mode = False # Switches to CBR mode and ignores CRF value. Used in conjunction with video_bitrate_kbps. settings.video_bitrate_kbps = 4000 # Target bitrate for CBR mode. Required when video_cbr_mode is enabled. settings.video_vbv_multiplier = 1.5 # Optional CBR VBV size as a multiple of one frame's bit budget (0 = auto: 1.5, or 3 with periodic keyframes). @@ -157,6 +167,8 @@ settings.render_node_path = None settings.omit_stripe_headers = False # --- Change Detection & Optimization --- +settings.video_min_qp = 0 # CBR QP clamps: 0 = encoder default; max bounds the quality floor, min bounds bit waste on easy content +settings.video_max_qp = 0 settings.use_paint_over_quality = True # Enable paint-over/IDR requests for static regions settings.paint_over_trigger_frames = 15 # Frames of no motion to trigger paint-over settings.damage_block_threshold = 10 # Consecutive changes to trigger "damaged" state @@ -165,6 +177,7 @@ settings.damage_block_duration = 30 # Frames a stripe stays "damaged" # --- Watermarking --- # Must be a bytes object. The path to your PNG image. settings.watermark_path = b"/path/to/your/watermark.png" +settings.cursor_size_cap = 32 # Cap out-of-band hardware-cursor PNGs to this longest edge (<= 0 = uncapped) # 0:None, 1:TopLeft, 2:TopRight, 3:BottomLeft, 4:BottomRight, 5:Middle, 6:Animated settings.watermark_location_enum = 4 ``` @@ -239,6 +252,10 @@ The Wayland backend implements a **Zero-Copy** architecture for hardware encodin **Performance Note:** Software (Pixman) rendering, the absence of a hardware encoder, or utilizing a render node different from the encoding node will force a "Readback" fallback, copying pixels to the CPU and breaking the zero-copy chain (higher latency and CPU load). A watermark does **not** force readback — on the GPU path it is composited into the frame before encoding. +## Built-in MP4 Recorder + +For convenience, the extension ships its own fragmented-MP4 muxer (no `avformat` dependency) with the `start_recording(...)`, `stop_recording()`, and `recording_status()` Python functions, controllable through the `PIXELFLUX_RECORD*` environment variables. Recording taps the encoded full-frame H.264 stream, and HTTP endpoints allow remote trigger/stop/status. + ## Recording Sink The capture session can output the raw H.264 video stream directly to a Unix domain socket for external recording. @@ -260,9 +277,9 @@ ffmpeg -f h264 -i unix:///tmp/pixelflux_record -c:v copy test.h264 ffmpeg -f h264 -framerate 60 -i unix:///tmp/pixelflux_record -c:v libx264 -preset fast -crf 23 -pix_fmt yuv420p test.mp4 ``` -## Computer Use Interface (Wayland) +## Computer Use Interface (X11 and Wayland) -The Wayland backend implements the [Anthropic Computer Use specification](https://github.com/anthropics/claude-quickstarts/tree/main/computer-use-demo), providing an HTTP API for AI agents to control the desktop. Enable it by setting the `PIXELFLUX_CU` environment variable to the port the server should listen on: +Both backends implement the [Anthropic Computer Use specification](https://github.com/anthropics/claude-quickstarts/tree/main/computer-use-demo), providing an HTTP API for AI agents to control the desktop. On Wayland the compositor injects input natively; on X11 the existing display is driven through XTEST with root-window screenshots. Enable it by setting the `PIXELFLUX_CU` environment variable to the port the server should listen on: ```bash export PIXELFLUX_CU=5000 @@ -437,7 +454,7 @@ install at build or runtime beyond the driver. * **X11:** XShm capture via pure-Rust XCB, with XFixes cursor and watermark compositing. * **Wayland:** Modern, secure, headless compositor based on [Smithay](https://github.com/Smithay/smithay). * **Flexible Encoding:** - * **Software:** x264 (H.264, incl. 4:4:4) and JPEG with multi-threaded striping. + * **Software:** x264 (H.264, incl. 4:4:4 — GPL, toggleable) and JPEG with multi-threaded striping, plus BSD-licensed OpenH264 full-frame software H.264. * **Hardware:** NVIDIA NVENC (incl. High 4:4:4, ARGB-direct BT.709, multi-GPU containers, API-version negotiation) and VA-API (Intel/AMD, VA-VPP convert) with Zero-Copy support. * **Driver-aware GPU auto-selection** via the `auto_gpu` setting. * **Zero-Copy Frames (X11 & Wayland):** the native frame object (buffer protocol) hands the encoded buffer to Python with no copy, on every supported Python version (3.9–3.14). @@ -446,13 +463,16 @@ install at build or runtime beyond the driver. * **Paint-Over:** Automatically improves quality for static regions. * **Damage Throttling:** Limits processing during high-motion scenes. * **On-demand keyframes:** `request_idr_frame()` forces an IDR for reconnecting clients. -* **Input Handling:** Built-in input injection for mouse and keyboard (Wayland). +* **Input Handling:** Built-in input injection for mouse and keyboard (Wayland; XTEST on X11 via Computer Use). * **Cursor Compositing:** Hardware cursor planes or software rendering options. * **Dynamic Watermarking:** Overlay PNGs with static positioning or DVD-screensaver style animation. * **Recording Sink:** Direct Unix socket output of full-frame H.264 streams for local capture. +* **Built-in MP4 Recorder:** Crash-safe fragmented-MP4 recording without any FFmpeg `avformat` dependency. * **AI Agent Control:** Computer Use API to dump screenshots and drive all facets of a desktop environment. ## License This project is licensed under the **Mozilla Public License Version 2.0**. A copy of the MPL 2.0 can be found at https://mozilla.org/MPL/2.0/. + +Note that the default build links the GPL-2.0+ `libx264` for striped software H.264; build with `PIXELFLUX_ENABLE_GPL=0` to exclude every GPL-licensed component (the FFmpeg bindings are used LGPL-only, and `openh264` is BSD-licensed). diff --git a/example/screen_to_browser.py b/example/screen_to_browser.py index 6ec8e5f..ba7ced7 100644 --- a/example/screen_to_browser.py +++ b/example/screen_to_browser.py @@ -9,111 +9,44 @@ This script demonstrates the pixelflux library's instance-safe capabilities. It can handle multiple WebSocket clients, each with its own independent screen capture session. The capture region can be controlled via the URL hash. + +The capture callback never blocks and never buffers more than the +bounded per-client queue. When a client cannot drain the stream, the backlog +is dropped and H.264 delivery gates per stripe row until the next keyframe +(requested from the encoder at most once per second), so a slow consumer +costs bounded memory and resumes on a decodable frame. """ # Standard library imports import asyncio import os import mimetypes +import time +import urllib.parse import websockets import websockets.asyncio.server as ws_async -import threading # Third-party library imports from pixelflux import CaptureSettings, ScreenCapture, ensure_wayland_display # ============================================================================== # --- BASE CONFIGURATION SETTINGS --- -# These settings will be used as a template for each new connection. -# Modify the parameters below to test different capture and encoding options. +# These settings are applied to a fresh CaptureSettings for each new +# connection. Modify the parameters below to test different capture and +# encoding options. # ============================================================================== HTTP_PORT = 9001 WS_PORT = 9000 -# Create a default template for capture settings. -base_capture_settings = CaptureSettings() - -# --- Debugging --- -# Enable/disable the continuous FPS and settings log printed to the console. -base_capture_settings.debug_logging = True - -# --- Core Capture --- -base_capture_settings.capture_width = 1920 -base_capture_settings.capture_height = 1080 -base_capture_settings.capture_x = 0 # This can be overridden by the URL -base_capture_settings.capture_y = 0 -base_capture_settings.target_fps = 60.0 -base_capture_settings.capture_cursor = False - -# --- Encoding Mode --- -# Sets the output codec. 0 for JPEG, 1 for H.264. -base_capture_settings.output_mode = 1 -# Force CPU encoding and ignore hardware encoders -base_capture_settings.use_cpu = False - -# --- Zero-copy frames --- -# The native callback delivers a StripeFrame (buffer-protocol object owning the C -# buffer). Every mode now emits its wire header natively (0x03 JPEG / 0x04 H.264), -# so memoryview(frame) is sent with no copy or re-frame on any mode. The buffer -# protocol pins the buffer alive across every Python version (no PEP 688). -ZERO_COPY = True - -# --- H.264 Quality Settings --- -# Constant Rate Factor (0-51, lower is better quality & higher bitrate). -# Good values are typically 18-28. -base_capture_settings.video_crf = 25 -# CRF for H.264 paintover on static content. Used if lower (better) than video_crf. -base_capture_settings.video_paintover_crf = 18 -# Number of high-quality H.264 frames to send in a burst when a paintover is triggered. -base_capture_settings.video_paintover_burst_frames = 5 -# Use I444 (full color) instead of I420. Better quality, higher CPU/bandwidth. -base_capture_settings.video_fullcolor = False -# Encode full frames instead of just changed stripes. -base_capture_settings.video_fullframe = False -# Flag the stream to be in streaming mode to bypass all vnc logic -base_capture_settings.video_streaming_mode = False -# Encoder device index: -2 = auto-detect (lets AUTO_GPU pick), -1 = software encoding, 0+ = specific GPU. -base_capture_settings.encode_node_index = -2 -# Switches to CBR mode and ignores CRF value. Used in conjunction with video_bitrate_kbps. -base_capture_settings.video_cbr_mode = False -# Target bitrate in kbps for CBR mode. Required when video_cbr_mode is enabled. -base_capture_settings.video_bitrate_kbps = 4000 -# Optional VBV buffer size in kilobits for custom buffer size. -base_capture_settings.video_vbv_multiplier = 1.5 # VBV as a multiple of one frame's bit budget (0 = auto policy). -# Allow pixelflux to adjust its capture width and height. Overrides provided width and height when enabled. -base_capture_settings.auto_adjust_screen_capture_size = True -# - -# --- Change Detection & Optimization --- -# Use a higher quality setting for static regions that haven't changed for a while. -base_capture_settings.use_paint_over_quality = True -# Number of frames of no motion in a stripe to trigger a high-quality "paint-over". -base_capture_settings.paint_over_trigger_frames = 15 -# Consecutive changes to a stripe to trigger a "damaged" state (uses base quality). -base_capture_settings.damage_block_threshold = 10 -# Number of frames a stripe stays "damaged" after being triggered. -base_capture_settings.damage_block_duration = 30 - -# --- JPEG Quality Settings --- -# Quality of jpegs under motion -base_capture_settings.jpeg_quality = 40 -# Quality of jpegs on static content paintovers -base_capture_settings.paint_over_jpeg_quality = 90 - -# --- Watermarking --- -# The path MUST be a byte string (b"") and point to a valid PNG file. -#base_capture_settings.watermark_path = b"/path/to/image.png" -# Sets the watermark location on the screen. Default is 0 (disabled). -# Options: 0:None, 1:TopLeft, 2:TopRight, 3:BottomLeft, 4:BottomRight, 5:Middle, 6:Animated -base_capture_settings.watermark_location_enum = 0 - -# --- Recording --- -# When this is set to a valid path (string) will enable a unix socket for recording -# i.e. '/tmp/test' can be recorded with "ffmpeg -f h264 -i unix:///tmp/test -c:v copy test.h264" -# For a clean recording the stream might need a re-encode i.e.: -# "ffmpeg -f h264 -framerate 60 -i unix:///tmp/test -c:v libx264 -preset fast -crf 23 -pix_fmt yuv420p test.mp4" -# This option enables IDR frames every 30 frames and on socket connection -base_capture_settings.recording_socket = None +# Send-side bounds, mirroring selkies' per-client video relay: the queue holds +# at most ~2 seconds of stream at 60 fps; keyframe requests triggered by +# overflow are rate-limited per client. +VIDEO_QUEUE_MAXSIZE = 120 +IDR_REQUEST_FLOOR_SECONDS = 1.0 +# A send that makes no progress for this long marks the client abandoned; the +# connection is closed rather than letting a dead socket linger (drops are +# already handled upstream by the bounded queue). +SEND_TIMEOUT_SECONDS = 1.0 # ============================================================================== # --- ENVIRONMENT OVERRIDES (the same variables selkies accepts) --- @@ -128,41 +61,128 @@ def _env(name, legacy=None): value = os.environ.get(legacy) return value or "" -# Backend: force Wayland/X11; left unset, pixelflux follows WAYLAND_DISPLAY. _wayland = _env("SELKIES_WAYLAND", "PIXELFLUX_WAYLAND").lower() -if _wayland: - base_capture_settings.use_wayland = _wayland in ("1", "true", "yes", "on") -# Compositor render node: an explicit path wins over auto-GPU selection. _render_dri = _env("SELKIES_RENDER_DRI", "DRINODE") -if _render_dri: - base_capture_settings.render_node_path = _render_dri.encode("utf-8") -# "true" (the default) = first GPU; any other token = first GPU matching a vendor -# name, kernel driver name, devicetree prefix, or raw PCI vendor id; "false" disables. -base_capture_settings.auto_gpu = _env("SELKIES_AUTO_GPU", "AUTO_GPU") or "true" -# Encoder node (VA-API/NVENC device), distinct from the render node. _encode_dri = _env("SELKIES_ENCODE_DRI", "DRI_NODE") -if _encode_dri: - base_capture_settings.encode_node_path = _encode_dri.encode("utf-8") - _idx = _encode_dri.rsplit("renderD", 1)[-1] - if _idx.isdigit(): - base_capture_settings.encode_node_index = int(_idx) - 128 _recording = _env("SELKIES_RECORDING_SOCKET", "PIXELFLUX_RECORDING_SOCKET") -if _recording: - base_capture_settings.recording_socket = _recording -# Compositor cursor-theme size in pixels (Wayland backend). _cursor_size = _env("SELKIES_CURSOR_SIZE", "XCURSOR_SIZE") -if _cursor_size.isdigit(): - base_capture_settings.cursor_size = int(_cursor_size) +_auto_gpu = _env("SELKIES_AUTO_GPU", "AUTO_GPU") or "true" + +def build_capture_settings(): + """Return a fresh CaptureSettings for one client. + + Each connection gets its own object: sessions started concurrently must + not see another client's overrides (e.g. the per-URL capture_x below). + """ + cs = CaptureSettings() + + # --- Debugging --- + # Enable/disable the continuous FPS and settings log printed to the console. + cs.debug_logging = True + + # --- Core Capture --- + cs.capture_width = 1920 + cs.capture_height = 1080 + cs.capture_x = 0 # This can be overridden by the URL + cs.capture_y = 0 + cs.target_fps = 60.0 + cs.capture_cursor = False + + # --- Encoding Mode --- + # Sets the output codec. 0 for JPEG, 1 for H.264. + cs.output_mode = 1 + # Force CPU encoding and ignore hardware encoders + cs.use_cpu = False + + # --- H.264 Quality Settings --- + # Constant Rate Factor (0-51, lower is better quality & higher bitrate). + # Good values are typically 18-28. + cs.video_crf = 25 + # CRF for H.264 paintover on static content. Used if lower (better) than video_crf. + cs.video_paintover_crf = 18 + # Number of high-quality H.264 frames to send in a burst when a paintover is triggered. + cs.video_paintover_burst_frames = 5 + # Use I444 (full color) instead of I420. Better quality, higher CPU/bandwidth. + cs.video_fullcolor = False + # Encode full frames instead of just changed stripes. + cs.video_fullframe = False + # Flag the stream to be in streaming mode to bypass all vnc logic + cs.video_streaming_mode = False + # Encoder device index: -2 = auto-detect (lets AUTO_GPU pick), -1 = software encoding, 0+ = specific GPU. + cs.encode_node_index = -2 + # Switches to CBR mode and ignores CRF value. Used in conjunction with video_bitrate_kbps. + cs.video_cbr_mode = False + # Target bitrate in kbps for CBR mode. Required when video_cbr_mode is enabled. + cs.video_bitrate_kbps = 4000 + # Optional VBV buffer size in kilobits for custom buffer size. + cs.video_vbv_multiplier = 1.5 # VBV as a multiple of one frame's bit budget (0 = auto policy). + # Allow pixelflux to adjust its capture width and height. Overrides provided width and height when enabled. + cs.auto_adjust_screen_capture_size = True + + # --- Change Detection & Optimization --- + # Use a higher quality setting for static regions that haven't changed for a while. + cs.use_paint_over_quality = True + # Number of frames of no motion in a stripe to trigger a high-quality "paint-over". + cs.paint_over_trigger_frames = 15 + # Consecutive changes to a stripe to trigger a "damaged" state (uses base quality). + cs.damage_block_threshold = 10 + # Number of frames a stripe stays "damaged" after being triggered. + cs.damage_block_duration = 30 + + # --- JPEG Quality Settings --- + # Quality of jpegs under motion + cs.jpeg_quality = 40 + # Quality of jpegs on static content paintovers + cs.paint_over_jpeg_quality = 90 + + # --- Watermarking --- + # The path MUST be a byte string (b"") and point to a valid PNG file. + #cs.watermark_path = b"/path/to/image.png" + # Sets the watermark location on the screen. Default is 0 (disabled). + # Options: 0:None, 1:TopLeft, 2:TopRight, 3:BottomLeft, 4:BottomRight, 5:Middle, 6:Animated + cs.watermark_location_enum = 0 + + # --- Recording --- + # When this is set to a valid path (string) will enable a unix socket for recording + # i.e. '/tmp/test' can be recorded with "ffmpeg -f h264 -i unix:///tmp/test -c:v copy test.h264" + # For a clean recording the stream might need a re-encode i.e.: + # "ffmpeg -f h264 -framerate 60 -i unix:///tmp/test -c:v libx264 -preset fast -crf 23 -pix_fmt yuv420p test.mp4" + # This option enables IDR frames every 30 frames and on socket connection + cs.recording_socket = None + + # --- Environment overrides --- + # Backend: force Wayland/X11; left unset, pixelflux follows WAYLAND_DISPLAY. + if _wayland: + cs.use_wayland = _wayland in ("1", "true", "yes", "on") + # Compositor render node: an explicit path wins over auto-GPU selection. + if _render_dri: + cs.render_node_path = _render_dri.encode("utf-8") + # "true" (the default) = first GPU; any other token = first GPU matching a vendor + # name, kernel driver name, devicetree prefix, or raw PCI vendor id; "false" disables. + cs.auto_gpu = _auto_gpu + # Encoder node (VA-API/NVENC device), distinct from the render node. + if _encode_dri: + cs.encode_node_path = _encode_dri.encode("utf-8") + _idx = _encode_dri.rsplit("renderD", 1)[-1] + if _idx.isdigit(): + cs.encode_node_index = int(_idx) - 128 + if _recording: + cs.recording_socket = _recording + # Compositor cursor-theme size in pixels (Wayland backend). + if _cursor_size.isdigit(): + cs.cursor_size = int(_cursor_size) + + return cs # Wayland: bring the compositor socket up now, before the capture starts, so apps # launched alongside this script can already connect to WAYLAND_DISPLAY. -if getattr(base_capture_settings, "use_wayland", False): +if _wayland in ("1", "true", "yes", "on"): _dim = lambda name: int(os.environ.get(name) or 0) if str(os.environ.get(name) or "").isdigit() else 0 ensure_wayland_display( width=_dim("SELKIES_MANUAL_WIDTH"), height=_dim("SELKIES_MANUAL_HEIGHT"), render_node=_render_dri, - auto_gpu=base_capture_settings.auto_gpu, + auto_gpu=_auto_gpu, cursor_size=int(_cursor_size) if _cursor_size.isdigit() else -1, ) @@ -175,7 +195,6 @@ def _env(name, legacy=None): # The key is the WebSocket connection object. # The value is another dictionary containing the client's capture module, queue, and task. ACTIVE_CLIENTS = {} -CLIENT_LOCK = threading.Lock() # Lock for thread-safe modifications to ACTIVE_CLIENTS. async def send_stripes_task(websocket, queue): """ @@ -184,22 +203,29 @@ async def send_stripes_task(websocket, queue): """ print(f"Send task started for client {websocket.remote_address}.") try: - # This loop will run until the connection is closed, - # which will raise a ConnectionClosed exception. while True: item = await queue.get() try: - # item == {'data': , 'owner': }. Keeping - # `item` (hence the StripeFrame) referenced for the whole send keeps the C - # buffer alive until the send releases the view, so zero-copy is safe. - await websocket.send(item['data']) + # item == {'data': , 'owner': }. Keeping + # `item` (hence the StripeFrame) referenced for the whole send keeps + # the C buffer alive until the send releases the view, so zero-copy + # is safe. The timeout is liveness-only: overflow is handled by the + # bounded queue upstream, so a send stuck this long means the client + # is gone, not merely slow. + await asyncio.wait_for(websocket.send(item['data']), + SEND_TIMEOUT_SECONDS) + except asyncio.TimeoutError: + print(f"Client {websocket.remote_address} abandoned (send " + f"stalled > {SEND_TIMEOUT_SECONDS}s); closing.") + await websocket.close(code=1001, reason='Send timeout') + break finally: queue.task_done() except websockets.exceptions.ConnectionClosed: # This is the expected, clean way to exit the loop when a client disconnects. print(f"Connection closed for {websocket.remote_address}. Send task stopping.") - + except asyncio.CancelledError: # This happens when the main handler cancels us during cleanup. print(f"Send task was cancelled for {websocket.remote_address}.") @@ -207,7 +233,7 @@ async def send_stripes_task(websocket, queue): except Exception as e: # Catch any other unexpected errors. print(f"[ERROR] Send task for client {websocket.remote_address} failed unexpectedly: {e}") - + finally: print(f"Send task for {websocket.remote_address} has finished.") @@ -221,44 +247,91 @@ async def websocket_handler(websocket): client_module = None send_task = None - # Keep a reference to the callback to prevent it from being garbage collected - on_stripe = None try: # --- 1. Configure Capture for this Specific Client --- - client_settings = base_capture_settings + client_settings = build_capture_settings() try: x_offset = int(path.strip('/')) client_settings.capture_x = x_offset print(f"Client {client_id} requested custom capture at x={x_offset}.") except (ValueError, TypeError): print(f"Client {client_id} using default capture at x=0.") - client_settings.capture_x = 0 # --- 2. Create Resources for this Client --- client_module = ScreenCapture() - client_queue = asyncio.Queue(maxsize=120) + client_queue = asyncio.Queue(maxsize=VIDEO_QUEUE_MAXSIZE) + # live_rows: stripe rows (wire y_start) whose H.264 chain is intact + # for this client. A row's deltas are deliverable only after its IDR; + # an overflow drop clears the set, gating every row until a keyframe + # re-anchors it. One frame can mix IDR and delta stripes (per-stripe + # encoder re-init), so gating is per row, not per frame — the same + # rule as selkies' video relay. Full-frame encoders always emit row 0. + relay_state = {'live_rows': set(), 'last_idr_req': 0.0} # --- 3. Create a unique callback (closure) for this client --- - # This function "closes over" client_queue and g_loop, giving it access - # without needing global lookups or user_data. + # Runs on the native capture thread: it must never block, so it hands + # the frame to the loop and the loop-side enqueue applies the bounded + # relay rules. The memoryview aliases the C buffer and pins the + # StripeFrame alive until the send (or a drop) releases the item. def on_stripe(frame): """Callback invoked by pixelflux with a StripeFrame per video stripe.""" if not (len(frame) > 0 and g_loop and not g_loop.is_closed()): return - if ZERO_COPY: - # Zero-copy: memoryview aliases the C buffer and pins the StripeFrame - # alive. The queued item holds both, so the frame (hence its buffer) - # outlives the send until the view is released. - item_to_queue = {'data': memoryview(frame), 'owner': frame} - else: - # Copy fallback: the frame already carries its native wire header. - item_to_queue = {'data': bytes(frame), 'owner': None} + item_to_queue = {'data': memoryview(frame), 'owner': frame} + + def request_idr_throttled(): + now = time.monotonic() + if now - relay_state['last_idr_req'] >= IDR_REQUEST_FLOOR_SECONDS: + relay_state['last_idr_req'] = now + client_module.request_idr_frame() + + def enqueue(): + data = item_to_queue['data'] + # Wire prefixes: 0x04 = H.264 (byte 1 = encoded picture type, + # 0x01 = IDR; bytes 4:6 = stripe y_start big-endian), 0x03 = + # JPEG. JPEG is never gated: every frame is independently + # decodable, a dropped one is simply repainted by the next. + is_h264 = len(data) >= 10 and data[0] == 0x04 + is_idr = is_h264 and data[1] == 0x01 + row = ((data[4] << 8) | data[5]) if is_h264 else 0 + if is_h264 and not is_idr and row not in relay_state['live_rows']: + # Undecodable until this row re-anchors; ask for a + # keyframe in case none is already on the way. + request_idr_throttled() + return + try: + client_queue.put_nowait(item_to_queue) + if is_idr: + relay_state['live_rows'].add(row) + except asyncio.QueueFull: + # The socket is slower than the stream: drop the whole + # backlog rather than buffer it, then recover via a + # keyframe. Dropping the item references releases the + # underlying StripeFrame buffers. + while not client_queue.empty(): + try: + client_queue.get_nowait() + client_queue.task_done() + except asyncio.QueueEmpty: + break + relay_state['live_rows'].clear() + if is_h264 and not is_idr: + request_idr_throttled() + else: + # Keyframes and JPEG frames go into the just-cleared + # queue so the client resumes immediately. + client_queue.put_nowait(item_to_queue) + if is_idr: + relay_state['live_rows'].add(row) - asyncio.run_coroutine_threadsafe( - client_queue.put(item_to_queue), g_loop - ) + try: + g_loop.call_soon_threadsafe(enqueue) + except RuntimeError: + # Loop closed between the check and the call (teardown race); + # dropping the item's references recycles the frame buffer. + pass # --- 4. Register and Start Resources for this Client --- send_task = asyncio.create_task(send_stripes_task(websocket, client_queue)) @@ -287,12 +360,12 @@ def on_stripe(frame): finally: # --- 7. Clean Up Resources for this Specific Client --- print(f"Cleaning up resources for client {client_id}...") - + if send_task and not send_task.done(): send_task.cancel() try: await send_task except asyncio.CancelledError: pass - + if client_module: loop = asyncio.get_running_loop() await loop.run_in_executor(None, client_module.stop_capture) @@ -300,6 +373,20 @@ def on_stripe(frame): ACTIVE_CLIENTS.pop(websocket, None) print(f"Cleanup complete for client {client_id}. Active clients: {len(ACTIVE_CLIENTS)}") +def _resolve_static_path(script_dir, request_path): + """Return the real path of request_path under script_dir, or None if it + escapes. realpath canonicalizes '..'/symlinks and the root+os.sep boundary + rejects sibling-prefix dirs; the path is URL-decoded first.""" + decoded = urllib.parse.unquote(request_path) + root = os.path.realpath(script_dir) + try: + requested = os.path.realpath(os.path.join(root, decoded.lstrip('/'))) + except ValueError: + return None # e.g. embedded NUL byte ("%00") + if requested != root and not requested.startswith(root + os.sep): + return None + return requested + async def handle_http_request(reader, writer): """Handle HTTP requests by serving static files from the script directory.""" try: @@ -309,7 +396,7 @@ async def handle_http_request(reader, writer): parts = request_line.split() if len(parts) < 2 or parts[0] != b'GET': - writer.write(b'HTTP/1.1 405 Method Not Allowed\r\n\r\n') + writer.write(b'HTTP/1.1 405 Method Not Allowed\r\nContent-Length: 0\r\nConnection: close\r\n\r\n') return path = parts[1].decode().split('#')[0] # Ignore hash part @@ -317,33 +404,34 @@ async def handle_http_request(reader, writer): path = '/index.html' script_dir = os.path.dirname(os.path.abspath(__file__)) - full_path = os.path.join(script_dir, path.lstrip('/')) - - # Security check to prevent directory traversal attacks - if not os.path.realpath(full_path).startswith(os.path.realpath(script_dir)): - writer.write(b'HTTP/1.1 403 Forbidden\r\n\r\n') + full_path = _resolve_static_path(script_dir, path) + + # Security check: reject directory traversal / escapes outside script_dir. + if full_path is None: + writer.write(b'HTTP/1.1 403 Forbidden\r\nContent-Length: 0\r\nConnection: close\r\n\r\n') return if os.path.isfile(full_path): with open(full_path, 'rb') as f: content = f.read() content_type = mimetypes.guess_type(full_path)[0] or 'application/octet-stream' - headers = f'HTTP/1.1 200 OK\r\nContent-Type: {content_type}\r\nContent-Length: {len(content)}\r\n\r\n' + headers = f'HTTP/1.1 200 OK\r\nContent-Type: {content_type}\r\nContent-Length: {len(content)}\r\nConnection: close\r\n\r\n' writer.write(headers.encode()) writer.write(content) else: - writer.write(b'HTTP/1.1 404 Not Found\r\n\r\n') + writer.write(b'HTTP/1.1 404 Not Found\r\nContent-Length: 0\r\nConnection: close\r\n\r\n') except Exception as e: print(f"[HTTP Error] {e}") + writer.write(b'HTTP/1.1 500 Internal Server Error\r\nContent-Length: 0\r\nConnection: close\r\n\r\n') finally: - if not writer.is_closing(): - try: - await writer.drain() - except ConnectionResetError: - pass # Client closed connection before we could finish. - finally: - writer.close() + # The client may already be gone; draining/closing a dead connection + # must not raise out of the handler. + try: + await writer.drain() + except (ConnectionError, OSError): + pass + writer.close() async def main(): @@ -364,26 +452,18 @@ async def main(): await asyncio.Event().wait() except OSError as e: print(f"[FATAL] Could not start server (is port {WS_PORT} in use?): {e}") - except KeyboardInterrupt: - print("\nShutdown signal received.") finally: print("Shutting down all client connections...") - # Create a list of cleanup tasks for all active clients - cleanup_tasks = [] - with CLIENT_LOCK: - clients_to_clean = list(ACTIVE_CLIENTS.keys()) - - for ws in clients_to_clean: - # Closing the websocket connection will trigger its handler's finally block - cleanup_tasks.append(ws.close(code=1001, reason='Server shutting down')) - + # Closing each websocket connection triggers its handler's finally block. + cleanup_tasks = [ws.close(code=1001, reason='Server shutting down') + for ws in list(ACTIVE_CLIENTS.keys())] if cleanup_tasks: await asyncio.gather(*cleanup_tasks, return_exceptions=True) if ws_server: ws_server.close() await ws_server.wait_closed() - + http_server.close() await http_server.wait_closed() print("All servers and connections closed. Goodbye.") diff --git a/pixelflux/Cargo.lock b/pixelflux/Cargo.lock index 529c758..598b15d 100644 --- a/pixelflux/Cargo.lock +++ b/pixelflux/Cargo.lock @@ -10,9 +10,9 @@ checksum = "320119579fcad9c21884f5c4861d16174d0e06250625266f50fe6898340abefa" [[package]] name = "aho-corasick" -version = "1.1.4" +version = "1.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ddd31a130427c27518df266943a5308ed92d4b226cc639f5a8f1002816174301" +checksum = "c982642fa9e8606056828ee9a8505737230110bb1099153c79efe865c59d12ba" dependencies = [ "memchr", ] @@ -1189,12 +1189,14 @@ version = "2.0.0" dependencies = [ "base64", "calloop", + "cfg-if", "crossbeam-channel", "ffmpeg-sys-next", "gbm", "image", "libc", "libloading 0.9.0", + "memmap2", "nvcodec-sys", "openh264", "openh264-sys2", @@ -1205,7 +1207,10 @@ dependencies = [ "smithay", "tiny_http", "turbojpeg", + "wayland-client", "wayland-protocols", + "wayland-protocols-misc", + "wayland-protocols-wlr", "wayland-server", "x11rb", "x264-sys", @@ -1577,9 +1582,9 @@ dependencies = [ [[package]] name = "regex-automata" -version = "0.4.16" +version = "0.4.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8fcfdb36bda0c880c5931cdc7a2bcdc8ba4556847b9d912bca70bc94708711ad" +checksum = "ad8553b9b26413251cbf30e620595c7a41b3887f03da04579c0e6b0d6a06b4b2" dependencies = [ "aho-corasick", "memchr", @@ -2157,6 +2162,18 @@ dependencies = [ "wayland-sys", ] +[[package]] +name = "wayland-client" +version = "0.31.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e3c36a0f861ad76d0901f2800b46321410d9f73f2ea88aac0650d86c32688073" +dependencies = [ + "bitflags", + "rustix 1.1.4", + "wayland-backend", + "wayland-scanner", +] + [[package]] name = "wayland-protocols" version = "0.32.13" @@ -2165,6 +2182,7 @@ checksum = "23d0c813de3daa2ed6520af85a3bd49b0e722a3078506899aa9686fea58dc4b6" dependencies = [ "bitflags", "wayland-backend", + "wayland-client", "wayland-scanner", "wayland-server", ] @@ -2177,6 +2195,7 @@ checksum = "6e9567599ef23e09b8dad6e429e5738d4509dfc46b3b21f32841a304d16b29c8" dependencies = [ "bitflags", "wayland-backend", + "wayland-client", "wayland-protocols", "wayland-scanner", "wayland-server", @@ -2190,6 +2209,7 @@ checksum = "eb04e52f7836d7c7976c78ca0250d61e33873c34156a2a1fc9474828ec268234" dependencies = [ "bitflags", "wayland-backend", + "wayland-client", "wayland-protocols", "wayland-scanner", "wayland-server", diff --git a/pixelflux/Cargo.toml b/pixelflux/Cargo.toml index e52d899..9af43ec 100644 --- a/pixelflux/Cargo.toml +++ b/pixelflux/Cargo.toml @@ -10,12 +10,18 @@ crate-type = ["cdylib"] [dependencies] pyo3 = { version = "0.29.0", features = ["extension-module"] } wayland-server = { version = "0.31.10", features = ["dlopen"] } -wayland-protocols = { version = "0.32", features = ["server"] } +wayland-protocols = { version = "0.32", features = ["server", "client", "staging"] } +wayland-client = "0.31" +wayland-protocols-misc = { version = "0.3", features = ["client"] } +wayland-protocols-wlr = { version = "0.3", features = ["client"] } +memmap2 = "0.9" crossbeam-channel = "0.5" calloop = "0.14" turbojpeg = "1.3" rayon = "1.10" -x264-sys = "0.2.2" +# Striped software H.264 (GPL-2.0+ libx264). Optional, enabled by the default `gpl` +# feature; build with `--no-default-features` (or PIXELFLUX_ENABLE_GPL=0) to exclude it. +x264-sys = { version = "0.2.2", optional = true } # BSD-licensed Cisco OpenH264 as an alternative software H.264 encoder; builds the # vendored C source (needs a C toolchain + nasm), no runtime download. openh264-sys2 # is used for the raw set_option live-bitrate path the safe wrapper doesn't expose. @@ -31,7 +37,7 @@ libloading = "0.9" xcursor = "0.3.1" # X11 host capture (replaces the C++ XShm/XFixes path): shm = XShm zero-copy grab, # xfixes = hardware cursor image. Pure-Rust XCB; links the system libxcb. -x11rb = { version = "0.13", features = ["shm", "xfixes"] } +x11rb = { version = "0.13", features = ["shm", "xfixes", "xtest", "randr"] } image = "=0.25.9" # The crate's build.rs probes the FFmpeg found via pkg-config and emits # per-version cfgs; one crate version supports FFmpeg 3.0 UP TO the release it @@ -45,11 +51,17 @@ base64 = "0.22" serde = { version = "1", features = ["derive"] } serde_json = "1" tiny_http = "0.12" +cfg-if = "1" [dependencies.nvcodec-sys] path = "nvcodec-sys" features = ["cuda"] +[features] +default = ["gpl"] +## GPL-licensed components (GPL-2.0+ libx264 for striped software H.264). +gpl = ["dep:x264-sys"] + [dependencies.smithay] git = "https://github.com/Smithay/smithay" rev = "ca932e042fa9ad150605c150a86275b85f9ad5b3" diff --git a/pixelflux/src/computer_use.rs b/pixelflux/src/computer_use.rs index 98439f2..e8930e6 100644 --- a/pixelflux/src/computer_use.rs +++ b/pixelflux/src/computer_use.rs @@ -8,12 +8,22 @@ //! //! Enabled by setting the `PIXELFLUX_CU` environment variable to the listen port. The server //! handles `POST /computer-use` requests for screenshots, mouse/keyboard injection, scrolling, -//! and cursor position queries — all targeting the Wayland compositor owned by the same process. +//! and cursor position queries. Actions run against a [`CuBackend`], resolved per request: the +//! Wayland compositor owned by this process when one is registered, otherwise the X server named +//! by `DISPLAY` (XTEST injection on a private connection, no active capture required). +//! +//! The same server also exposes the built-in MP4 recorder at `/record_start`, `/record_stop` +//! and `/record_status`, so a headless script can drive a session and record it over plain HTTP. +use std::collections::HashMap; use std::sync::mpsc; +use std::sync::{Mutex, OnceLock}; use std::thread; use std::time::Duration; use std::io::Cursor; +use std::io::Read; + +use smithay::input::keyboard::xkb; use base64::Engine; use base64::engine::general_purpose::STANDARD as BASE64; @@ -21,6 +31,7 @@ use image::{ImageBuffer, Rgba, ImageFormat}; use serde::Deserialize; use tiny_http; +use crate::wayland::keymap::keysym_for_char; use crate::ThreadCommand; fn clamp(v: T, lo: T, hi: T) -> T { @@ -41,60 +52,6 @@ pub fn encode_png_rgba(data: &[u8], width: u32, height: u32) -> Result, Ok(png) } -fn scancode_for_char(c: char) -> Option<(u32, bool)> { - Some(match c { - 'a' | 'A' => (38, c.is_uppercase()), - 'b' | 'B' => (56, c.is_uppercase()), - 'c' | 'C' => (54, c.is_uppercase()), - 'd' | 'D' => (40, c.is_uppercase()), - 'e' | 'E' => (26, c.is_uppercase()), - 'f' | 'F' => (41, c.is_uppercase()), - 'g' | 'G' => (42, c.is_uppercase()), - 'h' | 'H' => (43, c.is_uppercase()), - 'i' | 'I' => (31, c.is_uppercase()), - 'j' | 'J' => (44, c.is_uppercase()), - 'k' | 'K' => (45, c.is_uppercase()), - 'l' | 'L' => (46, c.is_uppercase()), - 'm' | 'M' => (58, c.is_uppercase()), - 'n' | 'N' => (57, c.is_uppercase()), - 'o' | 'O' => (32, c.is_uppercase()), - 'p' | 'P' => (33, c.is_uppercase()), - 'q' | 'Q' => (24, c.is_uppercase()), - 'r' | 'R' => (27, c.is_uppercase()), - 's' | 'S' => (39, c.is_uppercase()), - 't' | 'T' => (28, c.is_uppercase()), - 'u' | 'U' => (30, c.is_uppercase()), - 'v' | 'V' => (55, c.is_uppercase()), - 'w' | 'W' => (25, c.is_uppercase()), - 'x' | 'X' => (53, c.is_uppercase()), - 'y' | 'Y' => (29, c.is_uppercase()), - 'z' | 'Z' => (52, c.is_uppercase()), - '0' => (19, false), '1' => (10, false), '2' => (11, false), - '3' => (12, false), '4' => (13, false), '5' => (14, false), - '6' => (15, false), '7' => (16, false), '8' => (17, false), - '9' => (18, false), - '!' => (10, true), '@' => (11, true), '#' => (12, true), - '$' => (13, true), '%' => (14, true), '^' => (15, true), - '&' => (16, true), '*' => (17, true), '(' => (18, true), - ')' => (19, true), - '-' => (20, false), '_' => (20, true), - '=' => (21, false), '+' => (21, true), - '[' => (34, false), '{' => (34, true), - ']' => (35, false), '}' => (35, true), - ';' => (47, false), ':' => (47, true), - '\'' => (48, false), '"' => (48, true), - '`' => (49, false), '~' => (49, true), - '\\' => (51, false), '|' => (51, true), - ',' => (59, false), '<' => (59, true), - '.' => (60, false), '>' => (60, true), - '/' => (61, false), '?' => (61, true), - ' ' => (65, false), - '\t' => (23, false), - '\n' => (36, false), - _ => return None, - }) -} - fn scancode_for_keyname(name: &str) -> Option { Some(match name.to_lowercase().as_str() { "return" | "enter" => 36, @@ -147,62 +104,239 @@ fn scancode_for_keyname(name: &str) -> Option { }) } +/// Keysym for one component of an xdotool-style key spec that is not a named key: a single +/// literal character (`a`, `7`, `-`, `#`) via the Unicode mapping, or a keysym NAME +/// (`minus`, `bracketleft`, `udiaeresis`, …) via xkb's name lookup (case-tolerant fallback +/// so `Minus` still resolves). Returns 0 when the spec names no keysym. +fn keysym_for_key_spec(name: &str) -> u32 { + let mut chars = name.chars(); + if let (Some(c), None) = (chars.next(), chars.next()) { + return keysym_for_char(c); + } + let sym = xkb::keysym_from_name(name, xkb::KEYSYM_NO_FLAGS).raw(); + if sym != 0 { + return sym; + } + xkb::keysym_from_name(name, xkb::KEYSYM_CASE_INSENSITIVE).raw() +} + fn is_modifier(scancode: u32) -> bool { matches!(scancode, 37 | 105 | 50 | 62 | 64 | 108 | 133 | 134) } -fn send_key(tx: &smithay::reexports::calloop::channel::Sender, scancode: u32, pressed: bool) { - let _ = tx.send(ThreadCommand::KeyboardKey { - scancode, - state: if pressed { 1 } else { 0 }, - }); +/// Semantic mouse-button vocabulary of the action layer; each backend maps it to its native +/// codes (evdev `BTN_` on Wayland, X core buttons 1/2/3 on X11). +#[derive(Clone, Copy)] +pub enum CuButton { + Left, + Right, + Middle, } -fn send_mouse_move(tx: &smithay::reexports::calloop::channel::Sender, x: f64, y: f64) { - let _ = tx.send(ThreadCommand::PointerMotion { x, y }); +/// The desktop-side primitives one Computer Use action decomposes into. Keycodes are X/xkb +/// keycodes (evdev + 8) — the numbering both the smithay seat and XTEST consume — and +/// coordinates are framebuffer/root pixels, already clamped by the action layer. +pub trait CuBackend { + fn name(&self) -> &'static str; + fn fb_size(&self) -> Result<(i32, i32), String>; + /// One display's framebuffer size; 0 = the primary. Unknown ids are an error. The + /// X11 backend serves only the root (display 0); Wayland resolves any live output id. + fn display_fb_size(&self, display: u32) -> Result<(i32, i32), String> { + if display == 0 { + self.fb_size() + } else { + Err(format!("Unknown display: {display}")) + } + } + fn key(&self, scancode: u32, pressed: bool); + fn mouse_move(&self, x: f64, y: f64); + fn button(&self, btn: CuButton, pressed: bool); + fn scroll(&self, dx: f64, dy: f64); + /// PNG of one display's framebuffer; 0 = the primary. Unknown ids are an error. + fn screenshot_png(&self, display: u32) -> Result, String>; + fn cursor_pos(&self) -> Result<(f64, f64), String>; + /// Run `seq` with every keysym in `keysyms` (which `resolve_keysyms` could not place) + /// made temporarily typeable, when the backend can arrange that; `seq` receives + /// keysym -> keycode for the transient bindings (empty when none could be arranged, + /// in which case those keysyms simply stay untypeable). Wayland's `resolve_keysyms` + /// already overlay-binds anything the base keymap lacks, so the default runs `seq` + /// with no bindings; the X11 backend overrides this with a grab-atomic spare-keycode + /// remap. + fn with_transient_keysyms(&self, keysyms: &[u32], seq: &mut dyn FnMut(&HashMap)) { + let _ = keysyms; + seq(&HashMap::new()); + } + /// Resolve keysyms against the backend's ACTIVE keymap: one `(keycode, shift level)` + /// per input keysym, `(0, 0)` where the keymap cannot produce it. Level bit 0 = Shift, + /// bit 1 = AltGr — the order xkb two/four-level key types use. Wayland resolves through + /// the compositor's keymap policy (overlay-binding what the base lacks); X11 through + /// the server's own `GetKeyboardMapping`. + fn resolve_keysyms(&self, keysyms: &[u32]) -> Vec<(u32, u32)>; + /// Keycode synthesized around AltGr-level hits (level bit 1). The default is the + /// pc105 right-Alt position the compositor's seat keymap binds to `ISO_Level3_Shift`; + /// the X11 backend overrides it with the level-3 modifier key found in the server's + /// keymap (0 = none, in which case `resolve_keysyms` reports no AltGr levels). + fn altgr_keycode(&self) -> u32 { + KC_ALTGR + } } const BTN_LEFT: u32 = 0x110; const BTN_RIGHT: u32 = 0x111; const BTN_MIDDLE: u32 = 0x112; -fn send_mouse_button(tx: &smithay::reexports::calloop::channel::Sender, btn: u32, pressed: bool) { - let _ = tx.send(ThreadCommand::PointerButton { - btn, - state: if pressed { 1 } else { 0 }, - }); +/// Reply deadline for compositor round-trips (screenshot readback, cursor position, fb size). +/// Bounded so a wedged render path turns into a JSON error instead of hanging the sequential +/// HTTP loop — and every request behind it — forever. +const REPLY_TIMEOUT: Duration = Duration::from_secs(5); + +/// Wayland implementation: every primitive is a `ThreadCommand` on the compositor's calloop +/// channel, so injection and readback serialize naturally with rendering and encoding. +pub struct CuWaylandBackend { + tx: smithay::reexports::calloop::channel::Sender, } -fn send_scroll(tx: &smithay::reexports::calloop::channel::Sender, x: f64, y: f64) { - let _ = tx.send(ThreadCommand::PointerAxis { x, y }); +impl CuBackend for CuWaylandBackend { + fn name(&self) -> &'static str { + "wayland" + } + + fn fb_size(&self) -> Result<(i32, i32), String> { + self.display_fb_size(0) + } + + fn display_fb_size(&self, display: u32) -> Result<(i32, i32), String> { + let (resp_tx, resp_rx) = mpsc::channel(); + self.tx.send(ThreadCommand::CuGetInfo { display_id: display, resp: resp_tx }) + .map_err(|_| "Failed to request compositor info".to_string())?; + let (w, h, _) = resp_rx.recv_timeout(REPLY_TIMEOUT) + .map_err(|_| "Compositor info request failed".to_string())?; + // CuGetInfo reports zeros for output ids that don't exist (and for an output + // with no mode, which live outputs always have). + if w <= 0 || h <= 0 { + return Err(format!("Unknown display: {display}")); + } + Ok((w, h)) + } + + fn key(&self, scancode: u32, pressed: bool) { + let _ = self.tx.send(ThreadCommand::KeyboardKey { + scancode, + state: if pressed { 1 } else { 0 }, + }); + } + + fn mouse_move(&self, x: f64, y: f64) { + let _ = self.tx.send(ThreadCommand::PointerMotion { x, y }); + } + + fn button(&self, btn: CuButton, pressed: bool) { + let btn = match btn { + CuButton::Left => BTN_LEFT, + CuButton::Right => BTN_RIGHT, + CuButton::Middle => BTN_MIDDLE, + }; + let _ = self.tx.send(ThreadCommand::PointerButton { + btn, + state: if pressed { 1 } else { 0 }, + }); + } + + fn scroll(&self, dx: f64, dy: f64) { + let _ = self.tx.send(ThreadCommand::PointerAxis { x: dx, y: dy }); + } + + fn screenshot_png(&self, display: u32) -> Result, String> { + let (resp_tx, resp_rx) = mpsc::channel(); + self.tx.send(ThreadCommand::CuScreenshot { display_id: display, resp: resp_tx }) + .map_err(|_| "Failed to request screenshot".to_string())?; + resp_rx.recv_timeout(REPLY_TIMEOUT).map_err(|_| "Screenshot failed".to_string())? + } + + fn cursor_pos(&self) -> Result<(f64, f64), String> { + let (resp_tx, resp_rx) = mpsc::channel(); + self.tx.send(ThreadCommand::CuCursorPosition { resp: resp_tx }) + .map_err(|_| "Failed to request cursor position".to_string())?; + resp_rx.recv_timeout(REPLY_TIMEOUT).map_err(|_| "Cursor position failed".to_string()) + } + + fn resolve_keysyms(&self, keysyms: &[u32]) -> Vec<(u32, u32)> { + let (resp_tx, resp_rx) = mpsc::channel(); + if self + .tx + .send(ThreadCommand::BindKeysyms { keysyms: keysyms.to_vec(), reply: resp_tx }) + .is_err() + { + return vec![(0, 0); keysyms.len()]; + } + resp_rx + .recv_timeout(REPLY_TIMEOUT) + .unwrap_or_else(|_| vec![(0, 0); keysyms.len()]) + } } -fn screenshot( - tx: &smithay::reexports::calloop::channel::Sender, -) -> Result, String> { - let (resp_tx, resp_rx) = mpsc::channel(); - tx.send(ThreadCommand::CuScreenshot { resp: resp_tx }) - .map_err(|_| "Failed to request screenshot".to_string())?; - resp_rx.recv().map_err(|_| "Screenshot failed".to_string()) +/// Per-request literal-key resolver: batches keysym lookups against the backend's active +/// keymap and caches them for the request's burst of key events (a fresh backend — and thus +/// a fresh cache — is resolved per HTTP request, so a runtime layout switch is picked up by +/// the next request). +struct KeyResolver<'a> { + backend: &'a dyn CuBackend, + cache: HashMap, } -fn cursor_position( - tx: &smithay::reexports::calloop::channel::Sender, -) -> Result<(f64, f64), String> { - let (resp_tx, resp_rx) = mpsc::channel(); - tx.send(ThreadCommand::CuCursorPosition { resp: resp_tx }) - .map_err(|_| "Failed to request cursor position".to_string())?; - resp_rx.recv().map_err(|_| "Cursor position failed".to_string()) +impl<'a> KeyResolver<'a> { + fn new(backend: &'a dyn CuBackend) -> Self { + Self { backend, cache: HashMap::new() } + } + + /// Resolve a batch up front so a `type` action costs one backend round trip. + fn prefetch(&mut self, keysyms: &[u32]) { + let missing: Vec = { + let mut seen = std::collections::HashSet::new(); + keysyms + .iter() + .copied() + .filter(|&s| s != 0 && !self.cache.contains_key(&s) && seen.insert(s)) + .collect() + }; + if missing.is_empty() { + return; + } + let out = self.backend.resolve_keysyms(&missing); + for (sym, hit) in missing.into_iter().zip(out) { + self.cache.insert(sym, hit); + } + } + + /// `(keycode, shift level)` for `keysym`, or `None` when the active keymap cannot + /// produce it. + fn resolve(&mut self, keysym: u32) -> Option<(u32, u32)> { + if keysym == 0 { + return None; + } + self.prefetch(&[keysym]); + let hit = self.cache.get(&keysym).copied().unwrap_or((0, 0)); + (hit.0 != 0).then_some(hit) + } } -fn get_fb_size( - tx: &smithay::reexports::calloop::channel::Sender, -) -> Result<(i32, i32), String> { - let (resp_tx, resp_rx) = mpsc::channel(); - tx.send(ThreadCommand::CuGetInfo { resp: resp_tx }) - .map_err(|_| "Failed to request compositor info".to_string())?; - let (w, h, _) = resp_rx.recv().map_err(|_| "Compositor info request failed".to_string())?; - Ok((w, h)) +/// Shift and default AltGr (pc105 right Alt / ISO_Level3_Shift) keycodes, held to reach a +/// resolved key's shift level: bit 0 = Shift, bit 1 = AltGr. +const KC_SHIFT: u32 = 50; +const KC_ALTGR: u32 = 108; + +/// The level modifiers `level` requires beyond whatever is already held (`held_mask` uses +/// the same bit layout), in press order. `altgr` is the backend's AltGr keycode +/// ([`CuBackend::altgr_keycode`]). +fn level_modifiers(level: u32, held_mask: u32, altgr: u32) -> Vec { + let mut out = Vec::new(); + if level & 1 != 0 && held_mask & 1 == 0 { + out.push(KC_SHIFT); + } + if level & 2 != 0 && held_mask & 2 == 0 && altgr != 0 { + out.push(altgr); + } + out } #[derive(Deserialize)] @@ -216,17 +350,17 @@ struct CuActionRequest { scroll_amount: Option, duration: Option, region: Option<[f64; 4]>, + /// Display id for `screenshot` / `zoom` (default 0, the primary; `record_start` uses + /// its own `display` field with the same meaning). Unknown ids get an error reply. + display: Option, } fn ok_json() -> String { "{\"result\":\"ok\"}".to_string() } -fn handle_action( - req: CuActionRequest, - tx: &smithay::reexports::calloop::channel::Sender, -) -> String { - let result = handle_action_inner(req, tx); +fn handle_action(req: CuActionRequest, backend: &dyn CuBackend) -> String { + let result = handle_action_inner(req, backend); match result { Ok(response) => response, Err(e) => format!("{{\"error\":\"{}\"}}", e.replace('"', "\\\"")), @@ -236,20 +370,17 @@ fn handle_action( /// Turn one parsed Computer Use request into a real action on the captured desktop — the /// bridge that lets an external AI agent see and drive the session. /// -/// Every action ultimately becomes a compositor thread command or a framebuffer read, so this is -/// where the API's vocabulary meets the running compositor. Coordinates are clamped to the current +/// Every action ultimately becomes a backend input primitive or a framebuffer read, so this is +/// where the API's vocabulary meets the running desktop. Coordinates are clamped to the current /// framebuffer so a mistaken agent click cannot address off-screen pixels; keyboard and pointer /// actions translate into input events; and actions that must look at the screen (`screenshot`, /// `zoom`) request a fresh capture first, so the agent reasons about the current frame rather than /// a stale one. Returns the JSON response, or an error string explaining why the action could not /// run. -fn handle_action_inner( - req: CuActionRequest, - tx: &smithay::reexports::calloop::channel::Sender, -) -> Result { +fn handle_action_inner(req: CuActionRequest, b: &dyn CuBackend) -> Result { let sleep_ms = |ms: u64| thread::sleep(Duration::from_millis(ms)); - let (fb_w, fb_h) = get_fb_size(tx)?; + let (fb_w, fb_h) = b.fb_size()?; let handle_coord = |coord: [f64; 2]| -> (f64, f64) { ( @@ -265,7 +396,7 @@ fn handle_action_inner( match req.action.as_str() { "screenshot" => { - let png = screenshot(tx)?; + let png = b.screenshot_png(req.display.unwrap_or(0))?; let b64 = BASE64.encode(&png); Ok(format!("{{\"data\":\"{}\"}}", b64)) } @@ -273,34 +404,34 @@ fn handle_action_inner( "mouse_move" => { let coord = req.coordinate.ok_or("Missing coordinate")?; let (fx, fy) = handle_coord(coord); - send_mouse_move(tx, fx, fy); + b.mouse_move(fx, fy); Ok(ok_json()) } "left_click" | "right_click" | "middle_click" => { - let btn: u32 = match req.action.as_str() { - "left_click" => BTN_LEFT, - "right_click" => BTN_RIGHT, - _ => BTN_MIDDLE, + let btn = match req.action.as_str() { + "left_click" => CuButton::Left, + "right_click" => CuButton::Right, + _ => CuButton::Middle, }; if let Some(coord) = req.coordinate { let (fx, fy) = handle_coord(coord); - send_mouse_move(tx, fx, fy); + b.mouse_move(fx, fy); sleep_ms(30); } if let Some(ref mod_name) = req.text { if let Some(sc) = handle_modifier(mod_name) { - send_key(tx, sc, true); + b.key(sc, true); sleep_ms(20); } } - send_mouse_button(tx, btn, true); + b.button(btn, true); sleep_ms(20); - send_mouse_button(tx, btn, false); + b.button(btn, false); if let Some(ref mod_name) = req.text { if let Some(sc) = handle_modifier(mod_name) { sleep_ms(10); - send_key(tx, sc, false); + b.key(sc, false); } } Ok(ok_json()) @@ -310,25 +441,25 @@ fn handle_action_inner( let n = if req.action == "double_click" { 2 } else { 3 }; if let Some(coord) = req.coordinate { let (fx, fy) = handle_coord(coord); - send_mouse_move(tx, fx, fy); + b.mouse_move(fx, fy); sleep_ms(30); } if let Some(ref mod_name) = req.text { if let Some(sc) = handle_modifier(mod_name) { - send_key(tx, sc, true); + b.key(sc, true); sleep_ms(20); } } for _ in 0..n { - send_mouse_button(tx, BTN_LEFT, true); + b.button(CuButton::Left, true); sleep_ms(10); - send_mouse_button(tx, BTN_LEFT, false); + b.button(CuButton::Left, false); sleep_ms(10); } if let Some(ref mod_name) = req.text { if let Some(sc) = handle_modifier(mod_name) { sleep_ms(10); - send_key(tx, sc, false); + b.key(sc, false); } } Ok(ok_json()) @@ -339,91 +470,166 @@ fn handle_action_inner( let end = req.coordinate.ok_or("Missing coordinate")?; let (sx, sy) = handle_coord(start); let (ex, ey) = handle_coord(end); - send_mouse_move(tx, sx, sy); + b.mouse_move(sx, sy); sleep_ms(30); - send_mouse_button(tx, BTN_LEFT, true); + b.button(CuButton::Left, true); sleep_ms(30); - send_mouse_move(tx, ex, ey); + b.mouse_move(ex, ey); sleep_ms(30); - send_mouse_button(tx, BTN_LEFT, false); + b.button(CuButton::Left, false); Ok(ok_json()) } "left_mouse_down" => { - send_mouse_button(tx, BTN_LEFT, true); + b.button(CuButton::Left, true); Ok(ok_json()) } "left_mouse_up" => { - send_mouse_button(tx, BTN_LEFT, false); + b.button(CuButton::Left, false); Ok(ok_json()) } "type" => { let text = req.text.as_deref().ok_or("Missing text")?; - for chunk in text.as_bytes().chunks(50) { - let chunk_str = std::str::from_utf8(chunk).map_err(|_| "Invalid UTF-8")?; - for ch in chunk_str.chars() { - if ch == '\n' { - send_key(tx, 36, true); - sleep_ms(10); - send_key(tx, 36, false); - sleep_ms(10); - continue; + // A nested app compositor owns the apps on its own socket; type there as + // a virtual-keyboard client, since keys on pixelflux's own seat carry an + // overlay keymap the inner compositor never sees. Falls through to the + // local seat if the app socket is unreachable. + if b.name() == "wayland" { + if let Some(sock) = app_wayland_socket_path() { + // Failures log once per socket value; every request still + // retries, so a compositor that comes back is used again + // immediately (and re-arms the logging). + static FAILED_SOCK: Mutex> = Mutex::new(None); + match crate::wayland::vkclient::type_text_to(&sock, text) { + Ok(()) => { + *FAILED_SOCK.lock().unwrap() = None; + return Ok(ok_json()); + } + Err(e) => { + let mut last = FAILED_SOCK.lock().unwrap(); + if last.as_deref() != Some(sock.as_str()) { + eprintln!( + "[ComputerUse] app-compositor type via {sock} failed ({e}); using local seat until it is reachable" + ); + *last = Some(sock); + } + } + } + } + } + let mut resolver = KeyResolver::new(b); + let syms: Vec = text.chars().map(keysym_for_char).collect(); + resolver.prefetch(&syms); + // Base+AltGr resolution stays the preferred path; only what the active keymap + // cannot reach at all goes through the backend's transient-bind fallback. + let mut unresolved: Vec = Vec::new(); + for &sym in &syms { + if sym != 0 && resolver.resolve(sym).is_none() && !unresolved.contains(&sym) { + unresolved.push(sym); + } + } + b.with_transient_keysyms(&unresolved, &mut |bound| { + for (i, &sym) in syms.iter().enumerate() { + if i > 0 && i % 50 == 0 { + sleep_ms(20); } - if let Some((sc, need_shift)) = scancode_for_char(ch) { - if need_shift { - send_key(tx, 50, true); + if let Some((sc, level)) = resolver.resolve(sym) { + let level_mods = level_modifiers(level, 0, b.altgr_keycode()); + for &m in &level_mods { + b.key(m, true); sleep_ms(5); - send_key(tx, sc, true); - sleep_ms(10); - send_key(tx, sc, false); - send_key(tx, 50, false); - } else { - send_key(tx, sc, true); - sleep_ms(10); - send_key(tx, sc, false); } + b.key(sc, true); + sleep_ms(10); + b.key(sc, false); + for &m in level_mods.iter().rev() { + b.key(m, false); + } + sleep_ms(8); + } else if let Some(&kc) = bound.get(&sym) { + // Transient binds sit at the plain level: no modifiers needed. + b.key(kc, true); + sleep_ms(10); + b.key(kc, false); sleep_ms(8); } } - sleep_ms(20); - } + }); Ok(ok_json()) } "key" => { let text = req.text.as_deref().ok_or("Missing text")?; - let parts: Vec<&str> = text.split('+').collect(); + let mut resolver = KeyResolver::new(b); let mut mods: Vec = Vec::new(); - let mut main_key: Option = None; - for part in &parts { + let mut main_key: Option<(u32, u32)> = None; + // Keysym of the last main-key spec the active keymap could not place (cleared + // when a later part resolves): typed via the transient-bind fallback. + let mut unresolved_sym: Option = None; + for part in text.split('+') { let trimmed = part.trim(); if let Some(sc) = scancode_for_keyname(trimmed) { if is_modifier(sc) { mods.push(sc); } else { - main_key = Some(sc); + main_key = Some((sc, 0)); + unresolved_sym = None; + } + } else { + let sym = keysym_for_key_spec(trimmed); + if let Some(hit) = resolver.resolve(sym) { + main_key = Some(hit); + unresolved_sym = None; + } else if sym != 0 { + main_key = None; + unresolved_sym = Some(sym); } } } - for &sc in &mods { - send_key(tx, sc, true); - sleep_ms(10); - } - if let Some(sc) = main_key { - send_key(tx, sc, true); - sleep_ms(30); - send_key(tx, sc, false); - } else if let Some(&last_mod) = mods.last() { - send_key(tx, last_mod, true); - sleep_ms(30); - send_key(tx, last_mod, false); - } - for &sc in mods.iter().rev() { - sleep_ms(10); - send_key(tx, sc, false); - } + let unresolved: Vec = unresolved_sym.into_iter().collect(); + b.with_transient_keysyms(&unresolved, &mut |bound| { + let main_key = main_key.or_else(|| { + // Transient binds sit at the plain level: no modifiers needed. + unresolved_sym.and_then(|s| bound.get(&s)).map(|&kc| (kc, 0)) + }); + for &sc in &mods { + b.key(sc, true); + sleep_ms(10); + } + if let Some((sc, level)) = main_key { + // Modifiers reaching the key's shift level are added unless the spec + // already asked for them (Shift as 50/62, AltGr as 108 or the backend's + // resolved level-3 keycode). + let altgr = b.altgr_keycode(); + let held = mods.iter().fold(0u32, |m, &sc| match sc { + 50 | 62 => m | 1, + sc if sc == KC_ALTGR || sc == altgr => m | 2, + _ => m, + }); + let level_mods = level_modifiers(level, held, altgr); + for &m in &level_mods { + b.key(m, true); + sleep_ms(10); + } + b.key(sc, true); + sleep_ms(30); + b.key(sc, false); + for &m in level_mods.iter().rev() { + sleep_ms(10); + b.key(m, false); + } + } else if let Some(&last_mod) = mods.last() { + b.key(last_mod, true); + sleep_ms(30); + b.key(last_mod, false); + } + for &sc in mods.iter().rev() { + sleep_ms(10); + b.key(sc, false); + } + }); Ok(ok_json()) } @@ -431,10 +637,25 @@ fn handle_action_inner( let text = req.text.as_deref().ok_or("Missing text")?; let duration = req.duration.ok_or("Missing duration")?; let duration = duration.min(100.0); - if let Some(sc) = scancode_for_keyname(text) { - send_key(tx, sc, true); + let trimmed = text.trim(); + let mut resolver = KeyResolver::new(b); + let hit = match scancode_for_keyname(trimmed) { + Some(sc) => Some((sc, 0)), + None => resolver.resolve(keysym_for_key_spec(trimmed)), + }; + if let Some((sc, level)) = hit { + let level_mods = level_modifiers(level, 0, b.altgr_keycode()); + for &m in &level_mods { + b.key(m, true); + sleep_ms(10); + } + b.key(sc, true); sleep_ms((duration * 1000.0) as u64); - send_key(tx, sc, false); + b.key(sc, false); + for &m in level_mods.iter().rev() { + sleep_ms(10); + b.key(m, false); + } } Ok(ok_json()) } @@ -444,12 +665,12 @@ fn handle_action_inner( let amount = req.scroll_amount.unwrap_or(1).max(0) as f64; if let Some(coord) = req.coordinate { let (fx, fy) = handle_coord(coord); - send_mouse_move(tx, fx, fy); + b.mouse_move(fx, fy); sleep_ms(30); } if let Some(ref mod_name) = req.text { if let Some(sc) = handle_modifier(mod_name) { - send_key(tx, sc, true); + b.key(sc, true); sleep_ms(20); } } @@ -460,19 +681,19 @@ fn handle_action_inner( "right" => (amount, 0.0), _ => return Err(format!("Invalid scroll_direction: {}", dir)), }; - send_scroll(tx, dx, dy); + b.scroll(dx, dy); sleep_ms(30); if let Some(ref mod_name) = req.text { if let Some(sc) = handle_modifier(mod_name) { sleep_ms(10); - send_key(tx, sc, false); + b.key(sc, false); } } Ok(ok_json()) } "cursor_position" => { - let (x, y) = cursor_position(tx)?; + let (x, y) = b.cursor_pos()?; Ok(format!("{{\"text\":\"X={},Y={}\"}}", x.round() as i32, y.round() as i32)) } @@ -484,14 +705,18 @@ fn handle_action_inner( "zoom" => { let region = req.region.ok_or("Missing region")?; + let display = req.display.unwrap_or(0); + // The region clamps against the TARGET display's own framebuffer, which need + // not match the primary's. + let (dw, dh) = b.display_fb_size(display)?; let (x0, y0, x1, y1) = (region[0], region[1], region[2], region[3]); - let left = clamp(x0.round() as u32, 0, fb_w as u32 - 1); - let top = clamp(y0.round() as u32, 0, fb_h as u32 - 1); - let right = clamp(x1.round() as u32, left + 1, fb_w as u32); - let bottom = clamp(y1.round() as u32, top + 1, fb_h as u32); + let left = clamp(x0.round() as u32, 0, dw as u32 - 1); + let top = clamp(y0.round() as u32, 0, dh as u32 - 1); + let right = clamp(x1.round() as u32, left + 1, dw as u32); + let bottom = clamp(y1.round() as u32, top + 1, dh as u32); let crop_w = right - left; let crop_h = bottom - top; - let png = screenshot(tx)?; + let png = b.screenshot_png(display)?; if crop_w > 0 && crop_h > 0 { if let Ok(img) = image::load_from_memory(&png) { let cropped = img.crop_imm(left, top, crop_w, crop_h); @@ -510,25 +735,180 @@ fn handle_action_inner( } } +static WAYLAND_TX: Mutex>> = + Mutex::new(None); + +/// Body of `POST /record_start`. All fields are optional; unset ones fall back to the +/// `PIXELFLUX_RECORD_*` environment variables and built-in defaults. +#[derive(Deserialize, Default)] +struct RecordStartRequest { + /// Output MP4 path (default: `$PIXELFLUX_RECORD`, else `/tmp/pixelflux-record-.mp4`). + path: Option, + /// Wayland output id to record (default 0; ignored on X11). + display: Option, + /// Capture fps cap for a recorder-owned capture. + fps: Option, + /// Bitrate override in kbps for a recorder-owned capture. + bitrate_kbps: Option, +} + +/// Handle the recorder REST endpoints sharing the CU server: `record_start`, +/// `record_stop` and `record_status`, all one JSON round-trip into the same recorder +/// implementation the Python API and env vars use. Returns `None` for any other URL. +fn handle_record_endpoint(url: &str, body: &str) -> Option { + let reply = match url { + "/record_start" => { + let req: RecordStartRequest = if body.trim().is_empty() { + RecordStartRequest::default() + } else { + match serde_json::from_str(body) { + Ok(r) => r, + Err(e) => return Some(format!("{{\"error\":\"Invalid JSON: {}\"}}", e)), + } + }; + let path = req + .path + .or_else(|| std::env::var("PIXELFLUX_RECORD").ok().filter(|p| !p.is_empty())) + .unwrap_or_else(|| { + let ts = std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .map(|d| d.as_secs()) + .unwrap_or(0); + format!("/tmp/pixelflux-record-{ts}.mp4") + }); + let mut opts = crate::recorder::RecordOptions::from_env(path); + if let Some(d) = req.display { + opts.display_id = d; + } + if let Some(f) = req.fps { + opts.fps = f; + } + if let Some(b) = req.bitrate_kbps { + opts.bitrate_kbps = b; + } + match crate::recorder::start(opts) { + Ok(s) => crate::recorder::status_to_json(&s).to_string(), + Err(e) => serde_json::json!({ "error": e }).to_string(), + } + } + "/record_stop" => match crate::recorder::stop() { + Ok(s) => { + let mut v = crate::recorder::status_to_json(&s); + v["stopped"] = serde_json::Value::Bool(true); + v.to_string() + } + Err(e) => serde_json::json!({ "error": e }).to_string(), + }, + "/record_status" => match crate::recorder::status() { + Some(s) => crate::recorder::status_to_json(&s).to_string(), + None => serde_json::json!({ "active": false }).to_string(), + }, + _ => return None, + }; + Some(reply) +} + +/// Socket of the compositor apps run under when it is nested under pixelflux +/// (labwc/kwin): keys injected into pixelflux's own seat carry an overlay keymap +/// the inner compositor never sees, so CU text is typed as a client of this +/// socket instead. Set by selkies over the ScreenCapture ABI. +static CU_APP_WAYLAND_DISPLAY: Mutex> = Mutex::new(None); + +/// Set (or clear, with None/empty) the app compositor socket for CU text injection. +pub fn set_app_wayland_display(display: Option) { + *CU_APP_WAYLAND_DISPLAY.lock().unwrap() = display.filter(|s| !s.is_empty()); +} + +/// Resolve the app compositor socket PATH for CU typing, or None to type on the +/// local seat. The ABI value selkies set wins; a standalone CU (no selkies) falls +/// back to PIXELFLUX_APP_WAYLAND_DISPLAY. A value naming pixelflux's own +/// compositor means nothing is nested. +fn app_wayland_socket_path() -> Option { + let name = CU_APP_WAYLAND_DISPLAY + .lock() + .unwrap() + .clone() + .or_else(|| std::env::var("PIXELFLUX_APP_WAYLAND_DISPLAY").ok()) + .filter(|s| !s.is_empty())?; + if crate::wait_socket_name(Duration::from_millis(0)).as_deref() == Some(name.as_str()) { + return None; + } + crate::wayland::wlclient::socket_path(&name) +} + +/// Make the Wayland compositor the preferred CU backend: once a live calloop sender is +/// registered, every subsequent request routes to it instead of an X11 connection. +pub fn register_wayland_backend(tx: smithay::reexports::calloop::channel::Sender) { + *WAYLAND_TX.lock().unwrap() = Some(tx); +} + +/// The registered compositor's command channel, if a Wayland compositor is running in this +/// process. The recorder uses it to attach to (or start) a capture without any Python client. +pub(crate) fn wayland_command_sender( +) -> Option> { + WAYLAND_TX.lock().unwrap().clone() +} + +/// Start the CU server if `PIXELFLUX_CU` names a bind (the standalone fallback; +/// a selkies-managed session passes the setting through [`start_cu_server`]). +pub fn spawn_cu_from_env() { + if let Ok(bind) = std::env::var("PIXELFLUX_CU") { + start_cu_server(&bind); + } +} + +/// Start the CU server on `bind`: a bare port listens on all interfaces (the +/// container-deployment default), `host:port` scopes it. Guarded so that exactly +/// one server binds per process no matter how many call sites (module import, +/// Wayland compositor init, the selkies setting) race to spawn it; backend +/// selection stays per-request, so a server bound at import serves a compositor +/// that only starts later. +pub fn start_cu_server(bind: &str) { + let addr = if bind.contains(':') { + bind.to_string() + } else { + match bind.parse::() { + Ok(port) => format!("0.0.0.0:{port}"), + Err(_) => { + println!("[ComputerUse] Invalid bind '{bind}' - expected a port or host:port"); + return; + } + } + }; + static SPAWNED: OnceLock<()> = OnceLock::new(); + let mut first = false; + SPAWNED.get_or_init(|| { + first = true; + }); + if first { + thread::spawn(move || run_cu_server(addr)); + } +} + +/// Pick the backend for one request: the registered Wayland compositor when present, +/// otherwise a fresh private connection to `DISPLAY`. The X11 connection is per-request so a +/// restarted X server never leaves the CU thread holding a dead connection. +fn resolve_backend() -> Result, String> { + if let Some(tx) = WAYLAND_TX.lock().unwrap().clone() { + return Ok(Box::new(CuWaylandBackend { tx })); + } + crate::x11::computer_use::CuX11Backend::connect() + .map(|be| Box::new(be) as Box) +} + /// Expose the captured desktop to an AI agent over HTTP, so a Computer Use client can drive /// the session much as a human viewer would. /// -/// It runs on its own thread listening on `0.0.0.0:` for POST `/computer-use` JSON actions -/// (screenshot, mouse_move, click, key, scroll, …). Framebuffer dimensions are re-queried from the -/// compositor on every request rather than cached, because the stream can start, stop, or resize -/// underneath the agent and a stale size would misplace every coordinate. A screenshot forces a -/// one-frame GPU readback when the pipeline is in zero-copy mode, since that path never otherwise -/// brings host pixels back to the CPU and the agent needs an image to look at. -pub fn run_cu_server( - tx: smithay::reexports::calloop::channel::Sender, - port: u16, -) { - println!( - "[ComputerUse] Server listening on port {}", - port, - ); - - let server = match tiny_http::Server::http(format!("0.0.0.0:{}", port)) { +/// It runs on its own thread listening on `addr` for POST `/computer-use` JSON actions +/// (screenshot, mouse_move, click, key, scroll, …). The backend and the framebuffer dimensions +/// are re-resolved on every request rather than cached, because the compositor can start, the +/// stream can resize, or the X server can restart underneath the agent — a stale size would +/// misplace every coordinate. On Wayland a screenshot forces a one-frame GPU readback when the +/// pipeline is in zero-copy mode; on X11 it is a one-shot `GetImage` of the root window. +pub fn run_cu_server(addr: String) { + println!("[ComputerUse] Server listening on {}", addr); + + let server = match tiny_http::Server::http(addr.as_str()) { Ok(s) => s, Err(e) => { eprintln!("[ComputerUse] Failed to start server: {}", e); @@ -536,9 +916,17 @@ pub fn run_cu_server( } }; + let mut last_backend = ""; for mut request in server.incoming_requests() { + // A CU body is a single input command: cap its size so a hostile client + // of the (unauthenticated) endpoint cannot exhaust memory with a giant POST. + const MAX_CU_BODY: u64 = 4 * 1024 * 1024; let mut body = String::new(); - if let Err(e) = request.as_reader().read_to_string(&mut body) { + if let Err(e) = request + .as_reader() + .take(MAX_CU_BODY + 1) + .read_to_string(&mut body) + { let _ = request.respond(tiny_http::Response::from_string(format!( "{{\"error\":\"{}\"}}", e )) @@ -549,6 +937,29 @@ pub fn run_cu_server( continue; } + if body.len() as u64 > MAX_CU_BODY { + let resp = tiny_http::Response::from_string( + "{\"error\":\"request body too large\"}".to_string(), + ) + .with_status_code(413) + .with_header( + "Content-Type: application/json".parse::().unwrap() + ); + let _ = request.respond(resp); + continue; + } + + if let Some(json) = handle_record_endpoint(request.url(), &body) { + let _ = request.respond( + tiny_http::Response::from_string(json) + .with_status_code(200) + .with_header( + "Content-Type: application/json".parse::().unwrap() + ) + ); + continue; + } + let parsed: CuActionRequest = match serde_json::from_str(&body) { Ok(r) => r, Err(e) => { @@ -563,7 +974,16 @@ pub fn run_cu_server( } }; - let json_response = handle_action(parsed, &tx); + let json_response = match resolve_backend() { + Ok(backend) => { + if backend.name() != last_backend { + last_backend = backend.name(); + println!("[ComputerUse] Using {} backend", last_backend); + } + handle_action(parsed, backend.as_ref()) + } + Err(e) => format!("{{\"error\":\"{}\"}}", e.replace('"', "\\\"")), + }; let _ = request.respond( tiny_http::Response::from_string(json_response) .with_status_code(200) diff --git a/pixelflux/src/encoders/nvenc.rs b/pixelflux/src/encoders/nvenc.rs index bf6a32c..f171e94 100644 --- a/pixelflux/src/encoders/nvenc.rs +++ b/pixelflux/src/encoders/nvenc.rs @@ -521,33 +521,18 @@ impl Drop for NvencEncoder { } } -/// The lowest H.264 level (nvcodec-sys numeric encoding: 52/60/61/62 for 5.2/6.0/6.1/6.2) -/// whose Annex-A limits fit this resolution and frame rate, floored at 5.2. +use super::min_h264_level; + +/// The level an NVENC session advertises for this geometry, floored at 5.2 (`NV_ENC_LEVEL` +/// shares level_idc's numbering: 52/60/61/62 for 5.2/6.0/6.1/6.2). /// -/// A frame's cost is measured two ways against Annex-A Table A-1: **MaxFS**, the frame size in -/// macroblocks, and **MaxMBPS**, the macroblock rate (frame size × fps). The candidate table is -/// ascending and begins at 5.2 — there is no 5.1 entry — so every resolution up to 4K resolves -/// deterministically to High@5.2. Only above 4K does the level step up to 6.0 / 6.1 / 6.2; a flat -/// 5.2 would fail NVENC init there (5.2's MaxFS of 36864 MBs is ≈ 4096×2304). Picking the lowest -/// fitting level makes the SPS advertise the smallest level a decoder must support, and because the -/// level is fixed from frame 1 (with the profile held at High) the stream never bumps its level -/// mid-GOP — which would force some hardware decoders to re-initialize. Above 6.2's limits there is -/// no higher level, so it returns 6.2 as a best effort. -fn min_h264_level(width: u32, height: u32, fps: u32) -> u32 { - let mbs = (width as u64).div_ceil(16) * (height as u64).div_ceil(16); - let mbps = mbs * fps.max(1) as u64; - const LEVELS: [(u32, u64, u64); 4] = [ - (52, 36864, 2073600), - (60, 139264, 4177920), - (61, 139264, 8355840), - (62, 139264, 16711680), - ]; - for &(level, max_fs, max_mbps) in &LEVELS { - if mbs <= max_fs && mbps <= max_mbps { - return level; - } - } - 62 +/// `reconfigure_resolution` resizes a live session in place, so the level has to cover every +/// geometry that session can still reach — a level bump mid-GOP forces some hardware decoders to +/// re-initialize. 5.2's MaxFS of 36864 macroblocks (≈ 4096×2304) spans everything up to 4K, so +/// pinning that floor makes the whole range resolve to High@5.2; only beyond 4K does the shared +/// ladder step up to 6.0 / 6.1 / 6.2. +fn nvenc_h264_level(width: u32, height: u32, fps: u32) -> u32 { + min_h264_level(width, height, fps).max(52) } impl NvencEncoder { @@ -747,9 +732,10 @@ impl NvencEncoder { /// High-4:4:4 profile; CBR (two-pass quarter-resolution for tighter per-frame rate adherence, /// VBV sizing, optional min/max QP clamps) or ConstQP; infinite GOP (`gopLength` / `idrPeriod` /// = `0xFFFFFFFF`); `zeroReorderDelay` plus a bitstream-restriction VUI (`max_num_reorder_frames=0`) - /// so no-reorder decoders don't buffer; an explicit Annex-A level from `min_h264_level` pinned - /// from frame 1 so the level never bumps mid-stream; BT.709 VUI colorimetry; chroma 4:2:0 or - /// 4:4:4 with the matching full-range flag; repeated SPS/PPS; CABAC; no AUD; strict GOP + /// so no-reorder decoders don't buffer; an explicit Annex-A level from `nvenc_h264_level` + /// pinned from frame 1 so the level never bumps mid-stream; BT.709 VUI primaries and + /// transfer for the sRGB source, with an SMPTE170M matrix and limited range to match the + /// hardware ARGB CSC; chroma 4:2:0 or 4:4:4; repeated SPS/PPS; CABAC; no AUD; strict GOP /// target; and lookahead disabled for real-time latency. /// 6. **Initialize with resize headroom**: `maxEncodeWidth` / `maxEncodeHeight` are raised to at /// least 4096×2304 (the 5.2 ceiling) so `reconfigure_resolution` can grow in place; this costs @@ -942,21 +928,28 @@ impl NvencEncoder { config.rcParams.set_zeroReorderDelay(1); config.encodeCodecConfig.h264Config.h264VUIParameters.bitstreamRestrictionFlag = 1; config.encodeCodecConfig.h264Config.level = - min_h264_level(width, height, settings.target_fps as u32); + nvenc_h264_level(width, height, settings.target_fps as u32); config.encodeCodecConfig.h264Config.idrPeriod = 0xFFFFFFFF; config.encodeCodecConfig.h264Config.h264VUIParameters.videoSignalTypePresentFlag = 1; config.encodeCodecConfig.h264Config.h264VUIParameters.videoFormat = NV_ENC_VUI_VIDEO_FORMAT::NV_ENC_VUI_VIDEO_FORMAT_UNSPECIFIED; config.encodeCodecConfig.h264Config.h264VUIParameters.colourDescriptionPresentFlag = 1; + // Primaries and transfer describe the source, which is sRGB desktop pixels — + // sRGB shares BT.709's primaries and transfer function. Only the matrix follows + // the encoder: NVENC's ARGB hardware CSC is fixed at BT.601, so a client that + // inverts BT.709 shifts saturated colour badly. config.encodeCodecConfig.h264Config.h264VUIParameters.colourPrimaries = NV_ENC_VUI_COLOR_PRIMARIES::NV_ENC_VUI_COLOR_PRIMARIES_BT709; config.encodeCodecConfig.h264Config.h264VUIParameters.transferCharacteristics = NV_ENC_VUI_TRANSFER_CHARACTERISTIC::NV_ENC_VUI_TRANSFER_CHARACTERISTIC_BT709; config.encodeCodecConfig.h264Config.h264VUIParameters.colourMatrix = - NV_ENC_VUI_MATRIX_COEFFS::NV_ENC_VUI_MATRIX_COEFFS_BT709; + NV_ENC_VUI_MATRIX_COEFFS::NV_ENC_VUI_MATRIX_COEFFS_SMPTE170M; config.encodeCodecConfig.h264Config.chromaFormatIDC = if is_444 { 3 } else { 1 }; - config.encodeCodecConfig.h264Config.h264VUIParameters.videoFullRangeFlag = - if is_444 { 1 } else { 0 }; + // That same hardware CSC emits limited range in every chroma format. The + // host-planar path could encode full range, but a capture moves between the + // two at runtime when zero-copy demotes to readback, and the declared range + // has to hold across that, so every NVENC session declares limited. + config.encodeCodecConfig.h264Config.h264VUIParameters.videoFullRangeFlag = 0; config.encodeCodecConfig.h264Config.set_repeatSPSPPS(1); config.encodeCodecConfig.h264Config.entropyCodingMode = NV_ENC_H264_ENTROPY_CODING_MODE::NV_ENC_H264_ENTROPY_CODING_MODE_CABAC; @@ -1190,7 +1183,7 @@ impl NvencEncoder { } self.encode_config.encodeCodecConfig.h264Config.level = - min_h264_level(new_w, new_h, settings.target_fps as u32); + nvenc_h264_level(new_w, new_h, settings.target_fps as u32); if is_cbr { let bps = (settings.video_bitrate_kbps.max(0) as u32).saturating_mul(1000); self.encode_config.rcParams.averageBitRate = bps; @@ -1380,7 +1373,7 @@ impl NvencEncoder { if self.init_params.frameRateNum != fps { self.init_params.frameRateNum = fps; self.init_params.frameRateDen = 1; - self.encode_config.encodeCodecConfig.h264Config.level = min_h264_level( + self.encode_config.encodeCodecConfig.h264Config.level = nvenc_h264_level( self.init_params.encodeWidth, self.init_params.encodeHeight, fps, @@ -1417,8 +1410,7 @@ impl NvencEncoder { /// — a `0x04` tag, a picture-type byte derived from the *actual* encoded `pictureType` /// (IDR = `0x01`, I = `0x02`, P = `0x00`) rather than the `force_idr` request, the low 16 bits /// of the frame number, a zero field, and the width and height (all big-endian). - /// 4. **Emit and fan out**: append the encoded bytes, mirror them to the recording sink when one - /// is attached, unlock the bitstream, and return the framed buffer. + /// 4. **Emit**: append the encoded bytes, unlock the bitstream, and return the framed buffer. unsafe fn submit_frame( &mut self, mapped_buffer: NV_ENC_INPUT_PTR, @@ -1632,7 +1624,9 @@ impl NvencEncoder { } /// Encode a host ARGB frame by uploading it straight into the ARGB input surface, with no - /// CPU-side colour conversion. + /// CPU-side colour conversion. NVENC's hardware ARGB→YUV conversion is fixed at BT.601 + /// limited range, which is what the session VUI declares; a CPU prepass to BT.709 would cost + /// this path its zero-copy property. /// /// The packed rows are copied host→device into the registered ARGB surface and NVENC's hardware /// CSC produces the YUV. Bytes must be in NVENC ARGB order (`B,G,R,A` in memory) — the host BGRA @@ -1962,13 +1956,14 @@ mod gpu_tests { /// first post-resize frame is an IDR (steady frames are P), shrink to 480p, exercise the /// rejection cases (beyond headroom, chroma flip, RC-mode flip) and confirm the session still /// encodes afterward, and optionally dump a decodable stream. Ignored by default; run with - /// `cargo test gpu_ -- --ignored --nocapture`. + /// `cargo test gpu_ -- --ignored --nocapture --test-threads=1` — building NVENC sessions + /// concurrently races in the driver and intermittently faults, so the GPU set runs serially. #[test] #[ignore] fn gpu_resolution_reconfigure_roundtrip() { let mut s = settings(1280, 720, 60.0); let t0 = std::time::Instant::now(); - let mut enc = NvencEncoder::new(&s, ptr::null(), None).expect("NVENC init"); + let mut enc = NvencEncoder::new(&s, ptr::null()).expect("NVENC init"); let init_ms = t0.elapsed().as_secs_f64() * 1000.0; let mut stream: Vec = Vec::new(); @@ -2049,7 +2044,7 @@ mod gpu_tests { let mut s = settings(1280, 720, 60.0); s.video_cbr_mode = true; s.video_bitrate_kbps = 4000; - let mut enc = NvencEncoder::new(&s, ptr::null(), None).expect("NVENC init"); + let mut enc = NvencEncoder::new(&s, ptr::null()).expect("NVENC init"); let f720 = frame(1280, 720, 10); for i in 0..3u64 { enc.encode_cpu_argb(&f720, 1280 * 4, i, 25, i == 0) @@ -2082,7 +2077,7 @@ mod gpu_tests { } let s = settings(1920, 1080, 60.0); let before = used_mb(); - let mut enc = NvencEncoder::new(&s, ptr::null(), None).expect("init"); + let mut enc = NvencEncoder::new(&s, ptr::null()).expect("init"); let f = frame(1920, 1080, 5); for i in 0..3u64 { enc.encode_cpu_argb(&f, 1920 * 4, i, 25, i == 0).expect("encode"); @@ -2093,11 +2088,37 @@ mod gpu_tests { /// On a real GPU, a session that starts taller than the default 2304 headroom (portrait /// 4K: 2160×4096, within NVENC's 4096 H.264 cap) takes its own size as the `maxEncode` ceiling /// and encodes at that resolution. Ignored by default. + /// The VUI has to describe what the session actually emits. NVENC converts its ARGB + /// input with a fixed BT.601 matrix at limited range in both chroma formats, so the + /// matrix and range follow the hardware; the primaries and transfer follow the source, + /// which is sRGB desktop pixels and therefore BT.709. A client that inverts the wrong + /// matrix, or expands a limited-range frame as full-range, shifts colour visibly. + #[test] + #[ignore] + fn gpu_vui_describes_the_hardware_csc() { + for fullcolor in [false, true] { + let mut s = settings(1280, 720, 60.0); + s.video_fullcolor = fullcolor; + let enc = NvencEncoder::new(&s, ptr::null()).expect("NVENC init"); + // encodeCodecConfig is a union; a successful init leaves the H.264 arm live. + let h264 = unsafe { &enc.encode_config.encodeCodecConfig.h264Config }; + let vui = &h264.h264VUIParameters; + assert_eq!(vui.colourMatrix as u32, + NV_ENC_VUI_MATRIX_COEFFS::NV_ENC_VUI_MATRIX_COEFFS_SMPTE170M as u32); + assert_eq!(vui.colourPrimaries as u32, + NV_ENC_VUI_COLOR_PRIMARIES::NV_ENC_VUI_COLOR_PRIMARIES_BT709 as u32); + assert_eq!(vui.transferCharacteristics as u32, + NV_ENC_VUI_TRANSFER_CHARACTERISTIC::NV_ENC_VUI_TRANSFER_CHARACTERISTIC_BT709 as u32); + assert_eq!(vui.videoFullRangeFlag, 0, "fullcolor={fullcolor}"); + assert_eq!(h264.chromaFormatIDC, if fullcolor { 3 } else { 1 }); + } + } + #[test] #[ignore] fn gpu_init_above_default_headroom() { let s = settings(2160, 4096, 30.0); - let mut enc = NvencEncoder::new(&s, ptr::null(), None).expect("NVENC init portrait 4K"); + let mut enc = NvencEncoder::new(&s, ptr::null()).expect("NVENC init portrait 4K"); assert_eq!(enc.init_params.maxEncodeWidth, 4096); assert_eq!(enc.init_params.maxEncodeHeight, 4096); let f = frame(2160, 4096, 20); diff --git a/pixelflux/src/encoders/oh264.rs b/pixelflux/src/encoders/oh264.rs index 1d3cff6..1923daa 100644 --- a/pixelflux/src/encoders/oh264.rs +++ b/pixelflux/src/encoders/oh264.rs @@ -134,6 +134,10 @@ impl Openh264Encoder { .skip_frames(true) .adaptive_quantization(false) .background_detection(false) + // Scene-change detection stays on: OpenH264 forces it back on under + // ScreenContentRealTime and warns when asked for anything else, so the + // infinite-GOP contract cannot be tightened from here the way x264's + // `i_scenecut_threshold = 0` allows. .debug(settings.debug_logging) .intra_frame_period(IntraFramePeriod::from_num_frames(INFINITE_INTRA_PERIOD)) .num_threads(threads); @@ -338,16 +342,12 @@ impl Openh264Encoder { /// byte-for-byte the layout the NVENC/VAAPI/x264 full-frame paths emit, which is what lets a /// single client demuxer serve every encoder. Its type byte is read from the *actually /// encoded* picture type (IDR = `0x01`, I = `0x02`, P = `0x00`) rather than from `force_idr`, - /// because OpenH264's scene-change detection can emit an IDR the caller never asked for, and - /// the header must label a real decode entry point, not the request. It also carries the - /// frame number, a zero y-start, and the width/height (all big-endian). With - /// `omit_stripe_headers` the output is bare Annex-B. + /// because the encoder may re-type a frame, and the header must label a real decode entry + /// point, not the request. It also carries the frame number, a zero y-start, and the + /// width/height (all big-endian). With `omit_stripe_headers` the output is bare Annex-B. /// 5. **Empty payload**: if the encoder produced no bitstream (e.g. a skipped frame), an empty /// vec is returned rather than a lone header — a header with no Annex-B behind it would be a /// malformed frame to the client, and the pipeline reads empty as "nothing to send". - /// 6. **Recording sink**: the raw Annex-B payload (past the header) is also written to the - /// recording sink when one is attached, since a recorder wants the bare elementary stream, - /// not the wire framing. pub fn encode_host_argb( &mut self, argb: &[u8], @@ -459,7 +459,7 @@ mod tests { target_fps: 30.0, ..Default::default() }; - let mut enc = Openh264Encoder::new(&s, None).expect("openh264 init"); + let mut enc = Openh264Encoder::new(&s).expect("openh264 init"); let stride = 128 * 4; let idr = enc.encode_host_argb(&busy_frame(128, 96, 0), stride, 0, true, false).expect("encode idr"); assert!(idr.len() > 10, "IDR frame should produce output"); @@ -489,7 +489,7 @@ mod tests { omit_stripe_headers: true, ..Default::default() }; - let mut enc = Openh264Encoder::new(&s, None).expect("openh264 init"); + let mut enc = Openh264Encoder::new(&s).expect("openh264 init"); let out = enc.encode_host_argb(&busy_frame(128, 96, 0), 128 * 4, 0, true, false).expect("encode"); assert!( out.starts_with(&[0, 0, 0, 1]) || out.starts_with(&[0, 0, 1]), @@ -511,7 +511,7 @@ mod tests { target_fps: 30.0, ..Default::default() }; - let mut e = Openh264Encoder::new(&s, None).unwrap(); + let mut e = Openh264Encoder::new(&s).unwrap(); let _ = e.encode_host_argb(&busy_frame(w, h, 0), stride, 0, true, false).unwrap(); (1..24).map(|t| e.encode_host_argb(&busy_frame(w, h, t), stride, t as u64, false, false).unwrap().len()).sum() }; @@ -533,7 +533,7 @@ mod tests { target_fps: 30.0, ..Default::default() }; - let mut e = Openh264Encoder::new(&s, None).unwrap(); + let mut e = Openh264Encoder::new(&s).unwrap(); let _ = e.encode_host_argb(&busy_frame(w, h, 0), stride, 0, true, false).unwrap(); let high: usize = (1..24).map(|t| e.encode_host_argb(&busy_frame(w, h, t), stride, t as u64, false, false).unwrap().len()).sum(); @@ -557,7 +557,7 @@ mod tests { target_fps: 30.0, ..Default::default() }; - let mut e = Openh264Encoder::new(&s, None).unwrap(); + let mut e = Openh264Encoder::new(&s).unwrap(); let _ = e.encode_host_argb(&busy_frame(w, h, 0), stride, 0, true, false).unwrap(); let low: usize = (1..24).map(|t| e.encode_host_argb(&busy_frame(w, h, t), stride, t as u64, false, false).unwrap().len()).sum(); @@ -584,7 +584,7 @@ mod tests { target_fps: 30.0, ..Default::default() }; - let mut e = Openh264Encoder::new(&s, None).unwrap(); + let mut e = Openh264Encoder::new(&s).unwrap(); let mut total = e.encode_host_argb(&gradient_frame(w, h, 0), stride, 0, true, false).unwrap().len(); total += (1..24) .map(|t| e.encode_host_argb(&gradient_frame(w, h, t), stride, t as u64, false, false).unwrap().len()) @@ -614,7 +614,7 @@ mod tests { }; let frame = busy_frame(w, h, 0); for rgba in [false, true] { - let mut e = Openh264Encoder::new(&s, None).unwrap(); + let mut e = Openh264Encoder::new(&s).unwrap(); let out = e.encode_host_argb(&frame, stride, 0, true, rgba).expect("encode"); assert!(out.len() > 10, "rgba_input={rgba} must produce output"); assert_eq!(out[0], 0x04, "rgba_input={rgba} output must carry the wire header"); @@ -642,7 +642,7 @@ mod slice_tests { video_cbr_mode: true, ..Default::default() }; - let mut enc = Openh264Encoder::new(&s, None).expect("encoder"); + let mut enc = Openh264Encoder::new(&s).expect("encoder"); let frame = vec![0x80u8; 640 * 480 * 4]; let out = enc.encode_host_argb(&frame, 640 * 4, 0, true, false).expect("encode"); let mut nals = 0; @@ -705,7 +705,7 @@ mod slice_tests { target_fps: 60.0, ..Default::default() }; - let mut e = Openh264Encoder::new(&s, None).unwrap(); + let mut e = Openh264Encoder::new(&s).unwrap(); let mut total = 0usize; let mut idrs = 0usize; for t in 0..60u64 { @@ -752,7 +752,7 @@ mod slice_tests { target_fps: 60.0, ..Default::default() }; - let mut e = Openh264Encoder::new(&s, None).unwrap(); + let mut e = Openh264Encoder::new(&s).unwrap(); unsafe { let mut p: openh264_sys2::SEncParamExt = std::mem::zeroed(); let ret = e.encoder.raw_api().get_option( @@ -798,7 +798,7 @@ mod rebuild_cost { ..Default::default() }; let t = std::time::Instant::now(); - let mut e = Openh264Encoder::new(&s, None).expect("init"); + let mut e = Openh264Encoder::new(&s).expect("init"); let init_ms = t.elapsed().as_secs_f64() * 1000.0; let frame = vec![128u8; 1920 * 1080 * 4]; let t = std::time::Instant::now(); @@ -806,6 +806,8 @@ mod rebuild_cost { let first_ms = t.elapsed().as_secs_f64() * 1000.0; assert!(!out.is_empty()); println!("openh264 1080p init={init_ms:.1}ms first_frame={first_ms:.1}ms"); - assert!(init_ms < 100.0, "OpenH264 init unexpectedly slow: {init_ms}ms"); + // Headroom for heavily-loaded runners: the assertion exists to catch + // pathological rebuilds (seconds), not a few ms of host contention. + assert!(init_ms < 250.0, "OpenH264 init unexpectedly slow: {init_ms}ms"); } } diff --git a/pixelflux/src/encoders/overlay.rs b/pixelflux/src/encoders/overlay.rs index e4978f8..df0a86d 100644 --- a/pixelflux/src/encoders/overlay.rs +++ b/pixelflux/src/encoders/overlay.rs @@ -21,7 +21,7 @@ use smithay::{ ImportMem, Renderer, Texture, }, }, - utils::{Point, Transform, Physical}, + utils::{Point, Rectangle, Transform, Physical}, }; use std::path::Path; @@ -59,12 +59,14 @@ pub struct OverlayState { wm_height: u32, wm_pos_x: i32, wm_pos_y: i32, + wm_prev_pos: Option<(i32, i32)>, wm_velocity_x: f64, wm_velocity_y: f64, wm_subpixel_x: f64, wm_subpixel_y: f64, wm_loaded: bool, is_animated: bool, + wm_pixels: Vec, render_buffer: Option, } @@ -75,17 +77,60 @@ impl Default for OverlayState { wm_height: 0, wm_pos_x: 0, wm_pos_y: 0, + wm_prev_pos: None, wm_velocity_x: 2.0, wm_velocity_y: 2.0, wm_subpixel_x: 0.0, wm_subpixel_y: 0.0, wm_loaded: false, is_animated: false, + wm_pixels: Vec::new(), render_buffer: None, } } } +/// Alpha-blend a source pixel (pre-split into r,g,b,a) over a BGRA destination pixel. +/// +/// Overlay pixels are overwhelmingly either fully opaque or fully transparent, and this runs per +/// pixel per frame on the CPU, so the two extremes are special-cased to skip the blend arithmetic +/// entirely: an opaque source (`a == 255`) simply overwrites, a fully transparent source (`a == 0`) +/// is left as-is, and only genuine edge pixels pay for the integer source-over. In every case only +/// the B / G / R bytes are written; the destination's alpha byte is left as the capture delivered it. +#[inline] +pub(crate) fn blend_pixel(dst: &mut [u8], r: u8, g: u8, b: u8, a: u8) { + if a == 255 { + dst[0] = b; + dst[1] = g; + dst[2] = r; + } else if a > 0 { + let ia = 255 - a as u32; + dst[0] = ((b as u32 * a as u32 + dst[0] as u32 * ia) / 255) as u8; + dst[1] = ((g as u32 * a as u32 + dst[1] as u32 * ia) / 255) as u8; + dst[2] = ((r as u32 * a as u32 + dst[2] as u32 * ia) / 255) as u8; + } +} + +/// Source-over compositing for PREMULTIPLIED sources (XFixes/Xcursor pixels are +/// premultiplied by format definition): dst = src + dst*(1-a). The straight-alpha +/// `blend_pixel` on premultiplied input multiplies alpha in twice and darkens +/// every translucent pixel (visible dark fringes on the composited cursor). +/// +/// The sum saturates: some toolkits ship cursors whose colour exceeds its alpha, and on those +/// a wrapping cast would turn an over-bright pixel into a dark one. +pub(crate) fn blend_pixel_premultiplied(dst: &mut [u8], r: u8, g: u8, b: u8, a: u8) { + if a == 255 { + dst[0] = b; + dst[1] = g; + dst[2] = r; + } else if a > 0 { + let ia = 255 - a as u32; + dst[0] = (b as u32 + dst[0] as u32 * ia / 255).min(255) as u8; + dst[1] = (g as u32 + dst[1] as u32 * ia / 255).min(255) as u8; + dst[2] = (r as u32 + dst[2] as u32 * ia / 255).min(255) as u8; + } +} + impl OverlayState { /// Load the watermark image from disk; `output_scale` is the output's fractional /// scale, ceiled to the integer buffer scale of the upload. A failed load clears the overlay. @@ -97,20 +142,81 @@ impl OverlayState { self.wm_loaded = true; let buffer_scale = output_scale.ceil().max(1.0) as i32; + let pixels = rgba.into_vec(); self.render_buffer = Some(MemoryRenderBuffer::from_slice( - &rgba.into_vec(), + &pixels, Fourcc::Abgr8888, (self.wm_width as i32, self.wm_height as i32), buffer_scale, Transform::Normal, None, )); + self.wm_pixels = pixels; } else { self.wm_loaded = false; + self.wm_pixels = Vec::new(); self.render_buffer = None; } } + /// Alpha-blend the watermark into a BGRA frame (row `stride` in bytes) at its + /// current position. Clips per pixel at the frame bounds because the animated + /// position — or a watermark larger than the capture — can leave part of the + /// image off-frame, and only the in-frame portion may be written. + pub fn blend_bgra(&self, frame: &mut [u8], stride: usize, frame_w: i32, frame_h: i32) { + if !self.wm_loaded { + return; + } + let (w, h) = (self.wm_width as i32, self.wm_height as i32); + for y in 0..h { + let ty = self.wm_pos_y + y; + if ty < 0 || ty >= frame_h { + continue; + } + for x in 0..w { + let tx = self.wm_pos_x + x; + if tx < 0 || tx >= frame_w { + continue; + } + let src = ((y * w + x) * 4) as usize; + let (r, g, b, a) = ( + self.wm_pixels[src], + self.wm_pixels[src + 1], + self.wm_pixels[src + 2], + self.wm_pixels[src + 3], + ); + let off = ty as usize * stride + tx as usize * 4; + blend_pixel(&mut frame[off..off + 4], r, g, b, a); + } + } + } + + /// Frame-clipped union of the watermark's current and previous rectangles — + /// the region a damage-gated encoder must repaint after a bounce step moved + /// the image (the vacated area needs repainting as much as the new one). + pub fn damage_rect(&self, frame_w: i32, frame_h: i32) -> Option> { + if !self.wm_loaded { + return None; + } + let (w, h) = (self.wm_width as i32, self.wm_height as i32); + let (mut x0, mut y0) = (self.wm_pos_x, self.wm_pos_y); + let (mut x1, mut y1) = (x0 + w, y0 + h); + if let Some((px, py)) = self.wm_prev_pos { + x0 = x0.min(px); + y0 = y0.min(py); + x1 = x1.max(px + w); + y1 = y1.max(py + h); + } + x0 = x0.max(0); + y0 = y0.max(0); + x1 = x1.min(frame_w); + y1 = y1.min(frame_h); + if x1 <= x0 || y1 <= y0 { + return None; + } + Some(Rectangle::new((x0, y0).into(), (x1 - x0, y1 - y0).into())) + } + /// True once a watermark image has been loaded. pub fn is_active(&self) -> bool { self.wm_loaded @@ -135,6 +241,7 @@ impl OverlayState { let w = self.wm_width as i32; let h = self.wm_height as i32; + self.wm_prev_pos = Some((self.wm_pos_x, self.wm_pos_y)); self.is_animated = matches!(loc, WatermarkLocation::AN); match loc { @@ -211,12 +318,15 @@ impl OverlayState { } } - /// Render element for a software cursor `image` at `pos` (logical coords). + /// Render element for a software cursor `image` at logical `pos` on an output with + /// fractional `scale`; the position converts to physical here so the composited cursor + /// lands where the damage tracker and clients (which work in physical pixels) expect it. pub fn get_cursor_element( &self, renderer: &mut R, image: xcursor::parser::Image, - pos: Point, + pos: Point, + scale: f64, ) -> Option> where R: Renderer + ImportMem, @@ -233,7 +343,7 @@ impl OverlayState { let hot: Point = (image.xhot as i32, image.yhot as i32).into(); - let phys_pos: Point = (pos.x, pos.y).into(); + let phys_pos = pos.to_physical(smithay::utils::Scale::from(scale)).to_i32_round(); MemoryRenderBufferRenderElement::from_buffer( renderer, diff --git a/pixelflux/src/encoders/software.rs b/pixelflux/src/encoders/software.rs index 2bf9ed3..36db942 100644 --- a/pixelflux/src/encoders/software.rs +++ b/pixelflux/src/encoders/software.rs @@ -9,14 +9,16 @@ //! Frames are split into horizontal stripes processed in parallel via rayon. Each stripe is //! independently hashed against the previous frame for change detection, and only dirty stripes //! are encoded. The x264 path maintains per-stripe encoder state across frames for inter-prediction; -//! the JPEG path is stateless. An optional recording sink receives the H.264 NAL units for -//! muxing into a file or socket stream. +//! the JPEG path is stateless. use crate::RustCaptureSettings; use rayon::prelude::*; use smithay::utils::{Physical, Rectangle}; +#[cfg(feature = "gpl")] use std::ffi::CString; +#[cfg(feature = "gpl")] use std::ptr; +use std::sync::Arc; use yuv::{BufferStoreMut, YuvConversionMode, YuvPlanarImageMut, YuvRange, YuvStandardMatrix}; /// Upper bound on the horizontal stripes the CPU encoder splits a frame into, so the @@ -152,6 +154,7 @@ thread_local! { /// state and corrupt the heap. The lock is deliberately held only around open and close, never /// around `x264_encoder_encode`, so serializing setup costs nothing in the hot per-stripe encode /// path where the real parallelism lives. +#[cfg(feature = "gpl")] static X264_OPEN_CLOSE_LOCK: std::sync::Mutex<()> = std::sync::Mutex::new(()); /// One long-lived libx264 session for a stripe, holding the raw `x264_t` handle alongside a @@ -165,6 +168,7 @@ static X264_OPEN_CLOSE_LOCK: std::sync::Mutex<()> = std::sync::Mutex::new(()); /// chosen at open and gates which of the live reconfigures apply. The manual `Send` impl exists only /// because a raw pointer is not `Send` by default and the handle must move onto the rayon stripe /// workers; `Drop` closes it under the global open/close lock for the same reason that lock exists. +#[cfg(feature = "gpl")] pub struct H264EncoderWrapper { encoder: *mut x264_sys::x264_t, pub width: i32, @@ -179,8 +183,10 @@ pub struct H264EncoderWrapper { full_range: bool, } +#[cfg(feature = "gpl")] unsafe impl Send for H264EncoderWrapper {} +#[cfg(feature = "gpl")] impl Drop for H264EncoderWrapper { fn drop(&mut self) { if !self.encoder.is_null() { @@ -191,6 +197,7 @@ impl Drop for H264EncoderWrapper { } } +#[cfg(feature = "gpl")] impl H264EncoderWrapper { /// Open an x264 encoder tuned for real-time screen streaming, or `None` on failure. /// @@ -370,10 +377,9 @@ impl H264EncoderWrapper { /// /// The boolean return is load-bearing: `x264_encoder_encode` can legitimately produce nothing on /// a given call, and the caller must forward a stripe only when real bytes exist — never an empty - /// or header-only packet. The output also serves two consumers at once, which is why framing is - /// conditional: the transport needs the pipeline's small wire header to route the stripe, while - /// the optional recording sink needs the *bare* Annex-B elementary stream with no wire header so - /// it can be muxed directly. + /// or header-only packet. Framing is conditional because the transport needs the pipeline's small + /// wire header to route the stripe, while `omit_headers` consumers take the bare Annex-B + /// elementary stream. /// /// 1. **Picture setup**: wraps the borrowed Y/U/V planes and their strides in an /// `x264_picture_t` with the encoder's CSP, stamps the presentation timestamp with `frame_id`, @@ -386,9 +392,8 @@ impl H264EncoderWrapper { /// the client keys its decode-recovery on the frame type it truly received (IDR = `0x01`, /// I = `0x02`, else `0x00`), then the caller's `fixed_header` (frame number, y-start, width, /// height). With `omit_headers` the output is bare Annex-B. - /// 4. **Payload + recording**: every NAL payload is appended to `output_buf`, and each is also - /// forwarded to the recording sink when one is attached — giving the sink raw Annex-B without - /// the wire header. + /// 4. **Payload**: every NAL payload is appended to `output_buf` after the optional header, + /// so the bytes past the wire header are always a contiguous Annex-B access unit. #[allow(clippy::too_many_arguments)] pub fn encode_with_headers( &mut self, @@ -493,10 +498,14 @@ impl H264EncoderWrapper { pub struct StripeState { pub no_motion_frame_count: u32, pub paint_over_sent: bool, + #[cfg(feature = "gpl")] pub h264_encoder: Option, pub h264_burst_frames_remaining: i32, + #[cfg(feature = "gpl")] pub y_buf: Vec, + #[cfg(feature = "gpl")] pub u_buf: Vec, + #[cfg(feature = "gpl")] pub v_buf: Vec, pub packet_buf: Vec, pub last_hash: u64, @@ -580,13 +589,14 @@ impl StripeState { /// /// # Fields /// -/// * `data` - Compressed payload (JPEG or H.264 NAL units). +/// * `data` - Compressed payload (JPEG or H.264 NAL units). `Arc`-shared so every +/// delivery-layer consumer can retain the frame without copying the bytes. /// * `data_type` - Codec tag: **1 = JPEG**, **2 = H.264**. /// * `stripe_y_start` - Y pixel coordinate of the stripe's top edge within the frame. /// * `stripe_height` - Height of the stripe in pixels. /// * `frame_id` - Frame sequence number this stripe belongs to. pub struct EncodedStripe { - pub data: Vec, + pub data: Arc>, pub data_type: i32, pub stripe_y_start: i32, pub stripe_height: i32, @@ -601,8 +611,7 @@ pub struct EncodedStripe { /// parallel stripes; bandwidth is precious, so unchanged stripes are skipped. Each stripe is /// independently hashed against the previous frame for change detection, and only dirty stripes /// are encoded. The x264 path maintains per-stripe encoder state across frames for -/// inter-prediction; the JPEG path is stateless. The recording sink is attached only in -/// single-stripe mode because several independent sub-frame bitstreams cannot be muxed. +/// inter-prediction; the JPEG path is stateless. /// /// # Arguments /// @@ -674,10 +683,10 @@ pub struct EncodedStripe { /// OpenH264 encoder — one fewer than the available cores, clamped to `[1, 4]`. The slice threads /// keep the in-frame encode latency inside the frame budget at high resolutions; the cap is four /// because `zerolatency` makes x264 slice-threaded and more than four slices trips decode -/// glitches in some Chromium builds, and the minus-one leaves headroom for the capture thread. Multiple stripes instead run across the -/// rayon pool with a single x264 thread and one conversion band each, since the parallelism there -/// already comes from encoding the stripes concurrently. The recording sink is attached only in single-stripe mode, because the several -/// independent sub-frame bitstreams of striped mode cannot be muxed into one recording. +/// glitches in some Chromium builds, and the minus-one leaves headroom for the capture thread. +/// Multiple stripes instead run across the rayon pool with a single x264 thread and one +/// conversion band each, since the parallelism there already comes from encoding the stripes +/// concurrently. #[allow(clippy::too_many_arguments)] pub fn encode_cpu( stripes: &mut Vec, @@ -788,6 +797,7 @@ pub fn encode_cpu( let video_crf = settings.video_crf; let video_po_crf = settings.video_paintover_crf; let video_burst = settings.video_paintover_burst_frames; + #[cfg(feature = "gpl")] let video_fullcolor = settings.video_fullcolor; let video_streaming = settings.video_streaming_mode; let jpeg_q = settings.jpeg_quality; @@ -795,12 +805,16 @@ pub fn encode_cpu( let trigger_frames = settings.paint_over_trigger_frames; let use_paint_over = settings.use_paint_over_quality; let burst_crf = if use_paint_over && video_po_crf < video_crf { video_po_crf } else { video_crf }; + #[cfg(feature = "gpl")] let target_fps = settings.target_fps; let omit_headers = settings.omit_stripe_headers; let damage_block_threshold = settings.damage_block_threshold; let damage_block_duration = settings.damage_block_duration as i32; + #[cfg(feature = "gpl")] let video_cbr = settings.video_cbr_mode; + #[cfg(feature = "gpl")] let video_bitrate = settings.video_bitrate_kbps; + #[cfg(feature = "gpl")] let video_vbv = (crate::encoders::vbv_bits( (video_bitrate.max(0) as u32).saturating_mul(1000), target_fps, @@ -811,11 +825,13 @@ pub fn encode_cpu( // Full-frame x264 threads follow the same policy as the OpenH264 encoder: // one fewer than the cores (headroom for the capture thread), clamped to // [1, 4] to match the four-slice ceiling below. + #[cfg(feature = "gpl")] let h264_threads = if n_processing_stripes == 1 { num_cores.saturating_sub(1).clamp(1, 4) as i32 } else { 1 }; + #[cfg(feature = "gpl")] let csc_bands = 1; let stripe_body = |(i, stripe_state): (usize, &mut StripeState)| -> Option { @@ -829,6 +845,9 @@ pub fn encode_cpu( let mut send_this_stripe = false; let mut quality_or_crf = if output_mode == 0 { jpeg_q } else { video_crf }; + // Only the x264 stripe encoder consumes this; without `gpl` there is no + // consumer, so the flag and its sites go away with it. + #[cfg(feature = "gpl")] let mut force_idr = false; let is_dirty = if !hash_damage { stripe_is_dirty[i] @@ -875,7 +894,10 @@ pub fn encode_cpu( send_this_stripe = true; stripe_state.paint_over_sent = true; quality_or_crf = video_po_crf; - force_idr = true; + #[cfg(feature = "gpl")] + { + force_idr = true; + } stripe_state.h264_burst_frames_remaining = video_burst - 1; } } @@ -884,7 +906,10 @@ pub fn encode_cpu( if force_idr_all { send_this_stripe = true; if output_mode == 1 { - force_idr = true; + #[cfg(feature = "gpl")] + { + force_idr = true; + } if stripe_state.h264_burst_frames_remaining <= 0 && video_burst > 0 { stripe_state.paint_over_sent = true; stripe_state.h264_burst_frames_remaining = video_burst; @@ -932,7 +957,7 @@ pub fn encode_cpu( std::mem::take(&mut stripe_state.packet_buf) }; Some(EncodedStripe { - data, + data: Arc::new(data), data_type: 1, stripe_y_start: y_start as i32, stripe_height: actual_height as i32, @@ -940,6 +965,8 @@ pub fn encode_cpu( }) }) } else { + cfg_if::cfg_if! { + if #[cfg(feature = "gpl")] { let needs_reinit = if let Some(ref enc) = stripe_state.h264_encoder { enc.width != width_usize as i32 || enc.height != actual_height as i32 @@ -1025,7 +1052,7 @@ pub fn encode_cpu( &mut stripe_state.packet_buf, ) { Some(EncodedStripe { - data: std::mem::take(&mut stripe_state.packet_buf), + data: Arc::new(std::mem::take(&mut stripe_state.packet_buf)), data_type: 2, stripe_y_start: y_start as i32, stripe_height: actual_height as i32, @@ -1037,6 +1064,15 @@ pub fn encode_cpu( } else { None } + } else { + static GPL_OFF_LOGGED: std::sync::atomic::AtomicBool = + std::sync::atomic::AtomicBool::new(false); + if !GPL_OFF_LOGGED.swap(true, std::sync::atomic::Ordering::Relaxed) { + eprintln!("[software] software H.264 encoding unavailable: pixelflux was built without GPL components (libx264 disabled). Use JPEG, OpenH264 (`use_openh264 = true`), or a hardware encoder."); + } + None + } + } } } else { None @@ -1132,14 +1168,14 @@ mod tests { (w, h).into(), )]; let dirty = super::encode_cpu( - &mut stripes, &pixels, w, h, &full, &settings, 0, false, false, None, false, + &mut stripes, &pixels, w, h, &full, &settings, 0, false, false, false, ); assert!(!dirty.is_empty(), "damaged frame must encode"); let mut fired_at = None; for frame in 1..=20u16 { let out = super::encode_cpu( - &mut stripes, &pixels, w, h, &[], &settings, frame, false, false, None, false, + &mut stripes, &pixels, w, h, &[], &settings, frame, false, false, false, ); if !out.is_empty() { assert!(fired_at.is_none(), "paint-over must fire exactly once"); @@ -1177,14 +1213,14 @@ mod tests { }; let mut stripes = Vec::new(); let first = super::encode_cpu( - &mut stripes, &static_px, w, h, &[], &settings, 0, false, true, None, false, + &mut stripes, &static_px, w, h, &[], &settings, 0, false, true, false, ); assert!(!first.is_empty(), "first frame hashes as changed and encodes"); let mut fired_at = None; for frame in 1..=20u16 { let out = super::encode_cpu( - &mut stripes, &static_px, w, h, &[], &settings, frame, false, true, None, false, + &mut stripes, &static_px, w, h, &[], &settings, frame, false, true, false, ); if !out.is_empty() { assert!(fired_at.is_none(), "paint-over must fire exactly once while static"); @@ -1194,7 +1230,7 @@ mod tests { assert_eq!(fired_at, Some(settings.paint_over_trigger_frames as u16)); let woke = super::encode_cpu( - &mut stripes, &changed_px, w, h, &[], &settings, 21, false, true, None, false, + &mut stripes, &changed_px, w, h, &[], &settings, 21, false, true, false, ); assert!(!woke.is_empty(), "content change after idle must encode"); } @@ -1252,8 +1288,9 @@ mod qp_bound_sweep { //! Invariants under test: the CBR QP clamp reaches libx264/OpenH264 (a max clamp must //! raise worst-case fidelity on rate-starved text at the cost of bitrate overshoot; //! a min clamp must cut spend on over-budgeted content) and defaults (0) leave the - //! encoders' own behavior untouched. The printed table is the measurement behind the - //! shipped guidance for video_min_qp/video_max_qp. + //! encoders' own behavior untouched. Each encoder is swept separately so the OpenH264 + //! sweep still runs without the `gpl` feature, where it is the only software H.264 path. + #[cfg(feature = "gpl")] use super::H264EncoderWrapper; use crate::encoders::oh264::Openh264Encoder; use crate::RustCaptureSettings; @@ -1300,6 +1337,7 @@ mod qp_bound_sweep { /// Encode `FRAMES` scrolling-text luma frames through the x264 stripe encoder at the /// given rate-control settings (constant grey chroma), returning each frame's raw bitstream. + #[cfg(feature = "gpl")] fn encode_x264(cbr: bool, kbps: i32, crf: i32, min_qp: i32, max_qp: i32) -> Vec> { let mut enc = H264EncoderWrapper::new( W as i32, H as i32, crf, false, 60.0, 4, cbr, kbps, 50, min_qp, max_qp, @@ -1313,7 +1351,7 @@ mod qp_bound_sweep { let mut out = Vec::new(); enc.encode_with_headers( &y, &u, &v, W as i32, (W / 2) as i32, (W / 2) as i32, - i as i64, i == 0, &[], true, &mut out, None, + i as i64, i == 0, &[], true, &mut out, ); out }) @@ -1336,7 +1374,7 @@ mod qp_bound_sweep { video_max_qp: max_qp, ..Default::default() }; - let mut enc = Openh264Encoder::new(&s, None).expect("oh264 init"); + let mut enc = Openh264Encoder::new(&s).expect("oh264 init"); (0..FRAMES) .map(|i| { let y = text_luma(i); @@ -1401,53 +1439,67 @@ mod qp_bound_sweep { / 1000.0 } - /// Diagnostic that the CBR QP clamp is actually plumbed through to both x264 and - /// OpenH264, printing a bitrate/PSNR table on scrolling text and asserting the effect. + /// Diagnostic that the CBR QP clamp is actually plumbed through to x264, printing a + /// bitrate/PSNR table on scrolling text and asserting the effect. /// /// Encodes worst-case scrolling text at 2 Mbps CBR across a sweep of `max_qp` values (plus a /// separate `min_qp` sweep on an over-provisioned 12 Mbps budget), measuring luma PSNR against a - /// per-encoder near-lossless CRF/QP-12 reference so colour-conversion differences cancel out. It - /// asserts that capping `max_qp` at 30 on rate-starved content lifts fidelity by more than - /// 0.5 dB over the unclamped run on both encoders — proving the clamp reaches the encoder rather - /// than being silently dropped (paid for in bitrate overshoot). + /// near-lossless CRF-12 reference from the same encoder so colour-conversion differences cancel + /// out. Capping `max_qp` at 30 on rate-starved content must lift fidelity by more than 0.5 dB + /// over the unclamped run — proving the clamp reaches the encoder rather than being silently + /// dropped (paid for in bitrate overshoot). + #[cfg(feature = "gpl")] #[test] - fn cbr_qp_bound_sweep_diagnostic() { - let ref_x264 = decode_luma(&encode_x264(false, 0, 12, 0, 0)); - let ref_oh264 = decode_luma(&encode_oh264(false, 0, 12, 0, 0)); + fn cbr_qp_bound_sweep_x264() { + let reference = decode_luma(&encode_x264(false, 0, 12, 0, 0)); - println!("scrolling-text 720p60 @ 2 Mbps CBR (PSNR vs own CRF/QP-12 decode):"); + println!("scrolling-text 720p60 @ 2 Mbps CBR, x264 (PSNR vs own CRF-12 decode):"); let mut rows = Vec::new(); for &max_qp in &[0i32, 45, 40, 35, 30] { let x = encode_x264(true, 2000, 25, 0, max_qp); - let o = encode_oh264(true, 2000, 25, 0, max_qp); - let px = mean_psnr(&decode_luma(&x), &ref_x264); - let po = mean_psnr(&decode_luma(&o), &ref_oh264); - println!( - " max_qp {:>2}: x264 {:>8.1} kbps / {:>5.2} dB | oh264 {:>8.1} kbps / {:>5.2} dB", - max_qp, kbps(&x), px, kbps(&o), po - ); - rows.push((max_qp, kbps(&x), px, kbps(&o), po)); + let psnr = mean_psnr(&decode_luma(&x), &reference); + println!(" max_qp {:>2}: {:>8.1} kbps / {:>5.2} dB", max_qp, kbps(&x), psnr); + rows.push((max_qp, psnr)); } - println!("scrolling-text 720p60 @ 12 Mbps CBR, min-QP sweep:"); + println!("scrolling-text 720p60 @ 12 Mbps CBR, x264 min-QP sweep:"); for &min_qp in &[0i32, 10, 15] { let x = encode_x264(true, 12000, 25, min_qp, 0); - let px = mean_psnr(&decode_luma(&x), &ref_x264); - println!(" min_qp {:>2}: x264 {:>8.1} kbps / {:>5.2} dB", min_qp, kbps(&x), px); + let psnr = mean_psnr(&decode_luma(&x), &reference); + println!(" min_qp {:>2}: {:>8.1} kbps / {:>5.2} dB", min_qp, kbps(&x), psnr); } - let base = &rows[0]; - let capped = rows.last().unwrap(); + let base = rows[0].1; + let capped = rows.last().unwrap().1; assert!( - capped.2 > base.2 + 0.5, - "x264 max-QP clamp had no effect: {:.2} vs {:.2} dB", - capped.2, - base.2 + capped > base + 0.5, + "x264 max-QP clamp had no effect: {capped:.2} vs {base:.2} dB" ); + } + + /// The OpenH264 counterpart of [`cbr_qp_bound_sweep_x264`], and the only software H.264 + /// sweep that runs in a build without the `gpl` feature. + /// + /// Same scrolling-text workload and 2 Mbps CBR `max_qp` sweep, with luma PSNR measured against + /// this encoder's own near-lossless QP-12 reference. Capping `max_qp` at 30 must lift fidelity + /// by more than 0.5 dB over the unclamped run. + #[test] + fn cbr_qp_bound_sweep_openh264() { + let reference = decode_luma(&encode_oh264(false, 0, 12, 0, 0)); + + println!("scrolling-text 720p60 @ 2 Mbps CBR, oh264 (PSNR vs own QP-12 decode):"); + let mut rows = Vec::new(); + for &max_qp in &[0i32, 45, 40, 35, 30] { + let o = encode_oh264(true, 2000, 25, 0, max_qp); + let psnr = mean_psnr(&decode_luma(&o), &reference); + println!(" max_qp {:>2}: {:>8.1} kbps / {:>5.2} dB", max_qp, kbps(&o), psnr); + rows.push((max_qp, psnr)); + } + + let base = rows[0].1; + let capped = rows.last().unwrap().1; assert!( - capped.4 > base.4 + 0.5, - "oh264 max-QP clamp had no effect: {:.2} vs {:.2} dB", - capped.4, - base.4 + capped > base + 0.5, + "oh264 max-QP clamp had no effect: {capped:.2} vs {base:.2} dB" ); } } diff --git a/pixelflux/src/encoders/vaapi.rs b/pixelflux/src/encoders/vaapi.rs index 2600e53..2e7f810 100644 --- a/pixelflux/src/encoders/vaapi.rs +++ b/pixelflux/src/encoders/vaapi.rs @@ -138,9 +138,8 @@ fn ff_err_str(err: i32) -> String { /// /// `current_qp` / `qp_hysteresis_counter` drive the CQP hysteresis in `update_qp`. `cbr_mode`, /// `current_bitrate_kbps`, `current_vbv_mult`, and `current_kf_s` cache the live rate-control state -/// so `reconfigure_rate` re-opens the codec only when a value actually changes. `recording_sink` is -/// the optional Unix-socket H.264 fan-out; `omit_stripe_headers` drops the 10-byte framing when the -/// consumer wants a bare Annex-B stream. +/// so `reconfigure_rate` re-opens the codec only when a value actually changes. +/// `omit_stripe_headers` drops the 10-byte framing when the consumer wants a bare Annex-B stream. pub struct VaapiEncoder { encoder_ctx: *mut ff::AVCodecContext, codec: *const ff::AVCodec, @@ -295,7 +294,8 @@ impl VaapiEncoder { let width = settings.width; let height = settings.height; - let fps = settings.target_fps as i32; + // AVRational.den must stay nonzero: a sub-1 fps setting would truncate to 0. + let fps = (settings.target_fps as i32).max(1); unsafe { let mut drm_device_ctx: *mut ff::AVBufferRef = ptr::null_mut(); @@ -437,7 +437,11 @@ impl VaapiEncoder { } set_opt(&mut opts, "async_depth", "1"); set_opt(&mut opts, "profile", "high"); - set_opt(&mut opts, "level", "4.1"); + set_opt( + &mut opts, + "level", + &super::min_h264_level(width as u32, height as u32, fps as u32).to_string(), + ); let ret = ff::avcodec_open2(encoder_ctx, codec, &mut opts); ff::av_dict_free(&mut opts); @@ -700,7 +704,12 @@ impl VaapiEncoder { } set_opt(&mut opts, "async_depth", "1"); set_opt(&mut opts, "profile", "high"); - set_opt(&mut opts, "level", "4.1"); + set_opt( + &mut opts, + "level", + &super::min_h264_level(self.width as u32, self.height as u32, self.fps.max(1) as u32) + .to_string(), + ); let ret = ff::avcodec_open2(self.encoder_ctx, self.codec, &mut opts); ff::av_dict_free(&mut opts); @@ -797,7 +806,7 @@ impl VaapiEncoder { /// described to the client as one full-height stripe: tag `0x04`, a keyframe flag (`0x01`/`0x00` /// from `AV_PKT_FLAG_KEY`), the frame number as a big-endian `u16`, a `0` y-start (a whole frame /// starts at the top), then width and height as big-endian `u16`s, followed by the raw Annex-B - /// payload. The recording sink intercepts frames at the delivery layer. + /// payload. /// `omit_stripe_headers` drops the header on the network path too, for a consumer that already /// wants raw Annex-B. The loop drains `avcodec_receive_packet` until the encoder is empty, /// unref'ing each packet before the next iteration. diff --git a/pixelflux/src/lib.rs b/pixelflux/src/lib.rs index 7ca8e2b..48d1fc6 100644 --- a/pixelflux/src/lib.rs +++ b/pixelflux/src/lib.rs @@ -28,6 +28,7 @@ //! | [`x11`] | X11/XShm capture loop, XFixes out-of-band cursor monitor, and stripe dispatch | //! | [`pipeline`] | Frame-processing policy shared by both backends (send/QP/keyframe decisions) | //! | [`recording_sink`] | Unix-socket H.264 fan-out for external recording | +//! | [`recorder`] | Built-in MP4 recorder (fMP4 muxer + Python/env/REST control surfaces) | //! | [`computer_use`] | HTTP API for AI-agent desktop control (screenshots, input injection) | //! | [`nvgpufilter`] | Multi-GPU NVENC device filtering via ioctl | //! @@ -80,11 +81,13 @@ use smithay::{ element::{ memory::MemoryRenderBufferRenderElement, surface::WaylandSurfaceRenderElement, - AsRenderElements, Wrap, + AsRenderElements, Element, RenderElement, Wrap, }, gles::GlesRenderer, pixman::PixmanRenderer, - Bind, ImportAll, ImportEgl, ImportMem, + sync::SyncPoint, + Bind, ExportMem, Frame as _, ImportAll, ImportDma, ImportEgl, ImportMem, + Renderer as _, }, }, desktop::{space::SpaceRenderElements, Space}, @@ -96,7 +99,7 @@ use smithay::{ output::{Mode as OutputMode, Output, PhysicalProperties, Scale as OutputScale, Subpixel}, reexports::{ calloop::{ - channel::Event as CalloopEvent, generic::Generic, timer::{TimeoutAction, Timer}, + generic::Generic, timer::{TimeoutAction, Timer}, EventLoop, Interest, Mode, PostAction, }, pixman, @@ -113,7 +116,6 @@ use smithay::{ shell::xdg::XdgShellState, shm::ShmState, socket::ListeningSocketSource, - virtual_keyboard::VirtualKeyboardManagerState, pointer_warp::PointerWarpManager, relative_pointer::RelativePointerManagerState, pointer_constraints::PointerConstraintsState, @@ -137,6 +139,35 @@ pub mod encoders { pub mod software; /// VA-API hardware H.264 encoder for Intel / AMD GPUs via FFmpeg. pub mod vaapi; + /// Lowest H.264 level whose Annex-A Table A-1 limits admit a `width` x `height` stream at + /// `fps`, as level_idc (41 = 4.1, 52 = 5.2, 62 = 6.2). + /// + /// A frame is charged two ways: **MaxFS**, its size in macroblocks, and **MaxMBPS**, that + /// size times the frame rate. Advertising the lowest fitting level asks the least of a + /// decoder, so clients that gate hardware decode on the level accept the widest range of + /// streams. One shared ladder keeps two encoders from disagreeing on the same geometry; + /// it starts at 4.1 and a caller needing more raises the floor itself. Above 6.2's limits + /// there is no higher level, so it returns 62 as a best effort. + pub(crate) fn min_h264_level(width: u32, height: u32, fps: u32) -> u32 { + let mbs = (width as u64).div_ceil(16) * (height as u64).div_ceil(16); + let mbps = mbs * fps.max(1) as u64; + const LEVELS: [(u32, u64, u64); 8] = [ + (41, 8192, 245760), + (42, 8704, 522240), + (50, 22080, 589824), + (51, 36864, 983040), + (52, 36864, 2073600), + (60, 139264, 4177920), + (61, 139264, 8355840), + (62, 139264, 16711680), + ]; + for &(level, max_fs, max_mbps) in &LEVELS { + if mbs <= max_fs && mbps <= max_mbps { + return level; + } + } + 62 + } /// Size the CBR VBV/HRD buffer so rate control has enough slack to hold quality steady /// without letting end-to-end latency drift upward. @@ -173,6 +204,8 @@ pub mod encoders { pub mod wayland; /// Unix-socket H.264 recording fan-out for external capture tools. pub mod recording_sink; +/// Built-in MP4 recorder: independent capture-to-file with Python/env/REST control. +pub mod recorder; /// HTTP server implementing the Anthropic Computer Use spec for AI agent desktop control. pub mod computer_use; /// Frame-processing policy shared by the X11 and Wayland backends. @@ -209,6 +242,24 @@ fn get_shm_usage_bytes() -> u64 { total_size } +/// Memory reads for the throttle watchdog, answered from a 100 ms cache: the +/// render timer re-arms at 1 ms while an encode pool is saturated, and a +/// per-tick /proc read plus a full /dev/shm directory walk would spin the +/// syscall count of an otherwise deliberate busy-wait. The watchdog is a +/// hysteresis and tolerates the staleness. +fn get_cached_mem_usage() -> (usize, u64) { + static CACHE: std::sync::Mutex> = std::sync::Mutex::new(None); + let mut guard = CACHE.lock().unwrap_or_else(|e| e.into_inner()); + if let Some((at, rss, shm)) = *guard { + if at.elapsed() < Duration::from_millis(100) { + return (rss, shm); + } + } + let sampled = (Instant::now(), get_process_rss_bytes(), get_shm_usage_bytes()); + *guard = Some(sampled); + (sampled.1, sampled.2) +} + fn calculate_memory_threshold(width: i32, height: i32) -> usize { let frame_size = (width.max(0) as usize) .saturating_mul(height.max(0) as usize) @@ -226,7 +277,9 @@ use encoders::overlay::OverlayState; use encoders::software::MAX_STRIPE_CAPACITY; use encoders::vaapi::VaapiEncoder; -use wayland::cursor::Cursor; +use smithay::reexports::wayland_protocols_misc::zwp_virtual_keyboard_v1::server::zwp_virtual_keyboard_manager_v1::ZwpVirtualKeyboardManagerV1; + +use wayland::cursor::{Cursor, CursorJob}; use wayland::frontend::{AppState, ClientState, FocusTarget, GpuEncoder, next_serial, wayland_time, wayland_utime}; smithay::backend::renderer::element::render_elements! { @@ -244,7 +297,7 @@ smithay::backend::renderer::element::render_elements! { /// the buffer is described precisely enough (fd, stride, DRM modifier) for the importer to interpret /// it. One ARGB8888 plane is all that is carried because the compositor's offscreen target is exactly /// that single-plane format. -fn create_dmabuf_from_bo(bo: &BufferObject<()>) -> Dmabuf { +pub(crate) fn create_dmabuf_from_bo(bo: &BufferObject<()>) -> Dmabuf { let fd = bo.fd().expect("Failed to get FD from GBM BO"); let modifier = bo.modifier(); let stride = bo.stride(); @@ -304,6 +357,9 @@ pub struct RustCaptureSettings { pub debug_logging: bool, pub auto_adjust_screen_capture_size: bool, pub recording_socket: String, + /// Wayland display of an EXTERNAL compositor to capture (host-capture mode); + /// empty composites own clients as usual. + pub wayland_host_display: String, /// When true, encoders emit the raw payload without the per-stripe header byte block; /// stripe metadata is then carried only on the frame attributes. pub omit_stripe_headers: bool, @@ -314,7 +370,7 @@ pub struct RustCaptureSettings { /// infinite GOP, 3 when scheduled keyframes are enabled. pub video_vbv_multiplier: f64, /// Seconds between scheduled recovery keyframes; `<= 0` keeps the GOP infinite - /// (IDRs only on demand: client join / reset, recording cadence). + /// (IDRs only on demand: client join / reset, recorder connect). pub keyframe_interval_s: f64, /// Rate-controlled (CBR) QP clamp: `video_max_qp` bounds the quality FLOOR (screen text stays /// legible under motion at the cost of overshooting impossible targets), `video_min_qp` bounds @@ -341,6 +397,7 @@ pub struct LiveTunables { pub video_streaming_mode: bool, pub keyframe_interval_s: f64, pub capture_cursor: bool, + pub cursor_size_cap: i32, } impl LiveTunables { @@ -357,6 +414,7 @@ impl LiveTunables { video_streaming_mode: s.video_streaming_mode, keyframe_interval_s: s.keyframe_interval_s, capture_cursor: s.capture_cursor, + cursor_size_cap: s.cursor_size_cap, } } @@ -371,6 +429,7 @@ impl LiveTunables { s.video_paintover_burst_frames = self.video_paintover_burst_frames; s.video_streaming_mode = self.video_streaming_mode; s.keyframe_interval_s = self.keyframe_interval_s; + s.cursor_size_cap = self.cursor_size_cap; s.capture_cursor = self.capture_cursor; } } @@ -407,6 +466,7 @@ impl Default for RustCaptureSettings { debug_logging: false, auto_adjust_screen_capture_size: false, recording_socket: String::new(), + wayland_host_display: String::new(), omit_stripe_headers: false, video_cbr_mode: false, video_bitrate_kbps: 4000, @@ -451,15 +511,29 @@ pub(crate) fn extract_settings(settings: &Bound<'_, PyAny>) -> PyResult i32 { + if v <= 0 { 0 } else { v.min(MAX_CAPTURE_DIM) } + }; + let sanitize_fps = |v: f64| -> f64 { + if v.is_finite() && v > 0.0 { v.min(MAX_FPS) } else { DEFAULT_FPS } + }; + let sanitize_scale = |v: f64| -> f64 { + if v.is_finite() && v > 0.0 { v.min(MAX_SCALE) } else { 1.0 } + }; + Ok(RustCaptureSettings { - width: settings.getattr("capture_width")?.extract()?, - height: settings.getattr("capture_height")?.extract()?, - scale, + width: sanitize_dim(settings.getattr("capture_width")?.extract()?), + height: sanitize_dim(settings.getattr("capture_height")?.extract()?), + scale: sanitize_scale(scale), capture_x: settings.getattr("capture_x")?.extract()?, capture_y: settings.getattr("capture_y")?.extract()?, - target_fps: settings.getattr("target_fps")?.extract()?, - jpeg_quality: settings.getattr("jpeg_quality")?.extract()?, - paint_over_jpeg_quality: settings.getattr("paint_over_jpeg_quality")?.extract()?, + target_fps: sanitize_fps(settings.getattr("target_fps")?.extract()?), + jpeg_quality: settings.getattr("jpeg_quality")?.extract::()?.clamp(1, 100), + paint_over_jpeg_quality: settings.getattr("paint_over_jpeg_quality")?.extract::()?.clamp(1, 100), use_paint_over_quality: settings.getattr("use_paint_over_quality")?.extract()?, paint_over_trigger_frames: settings.getattr("paint_over_trigger_frames")?.extract()?, damage_block_threshold: settings.getattr("damage_block_threshold")?.extract()?, @@ -496,9 +570,11 @@ pub(crate) fn extract_settings(settings: &Bound<'_, PyAny>) -> PyResult().ok()) - .filter(|s| !s.is_empty()) - .or_else(|| std::env::var("PIXELFLUX_RECORDING_SOCKET").ok().filter(|s| !s.is_empty())) - .or_else(|| std::env::var("SELKIES_RECORDING_SOCKET").ok().filter(|s| !s.is_empty())) + .unwrap_or_default(), + wayland_host_display: settings + .getattr("wayland_host_display") + .ok() + .and_then(|v| v.extract::().ok()) .unwrap_or_default(), omit_stripe_headers: settings .getattr("omit_stripe_headers") @@ -530,6 +606,14 @@ pub(crate) fn extract_settings(settings: &Bound<'_, PyAny>) -> PyResult) -> PyResult, RustCaptureSettings), - StopCapture, + /// Start (or in-place reconfigure) the capture bound to output `display_id`. + /// `callback` is the Python per-frame delivery target; `None` starts an internal + /// capture with no Python consumer (the built-in recorder taps the delivery layer). + StartCapture { display_id: u32, callback: Option>, settings: RustCaptureSettings }, + /// Stop the capture bound to output `display_id` (other displays keep running). + StopCapture { display_id: u32 }, + /// Create an additional output: `WxH` physical pixels at fractional `scale`, mapped + /// into the layout at offset `(x, y)`. Replies false when the id is taken/reserved or + /// the GPU render target cannot be allocated. + CreateOutput { + id: u32, + width: i32, + height: i32, + x: i32, + y: i32, + scale: f64, + reply: std::sync::mpsc::Sender, + }, + /// Destroy a secondary output: its capture ends, its windows relocate to the primary + /// output. Replies false for the primary (id 0) or an unknown id. + DestroyOutput { id: u32, reply: std::sync::mpsc::Sender }, + /// Remap an existing output (the primary included) to layout offset `(x, y)`: the + /// Space mapping, the offsets used for absolute input injection and cursor + /// compositing, and the windows placed on it all follow, and the output is damaged so + /// the next frames render correctly. Replies false for an unknown id. + RepositionOutput { id: u32, x: i32, y: i32, reply: std::sync::mpsc::Sender }, + /// Reply with every live output as `(id, x, y, width, height, scale, capturing)`. + ListOutputs { reply: std::sync::mpsc::Sender> }, + /// Move the window with the given id onto output `output_id` (fullscreened there). + MoveWindowToOutput { window_id: u32, output_id: u32, reply: std::sync::mpsc::Sender }, + /// Reply with every mapped window as `(window_id, title, app_id, output_id)`. + ListWindows { reply: std::sync::mpsc::Sender> }, SetCursorCallback(Py), SetClipboardCallback(Py), /// Server-side clipboard offer: the compositor owns the selection and serves `data` as /// `mime` (plus text aliases) to pasting clients. SetClipboard { mime: String, data: Vec }, KeyboardKey { scancode: u32, state: u32 }, - /// Swap the seat keyboard's xkb keymap (XKB_KEYMAP_FORMAT_TEXT_V1 text); the caller owns - /// keysym-to-keycode policy and injects the keycodes it defined via `KeyboardKey`. + /// A whole ordered run of key events in one message. Typing a paste one event at a + /// time costs a channel send and a calloop wake per event, which competes with the + /// render loop on the same thread; the caller still decides the sequence. + KeyboardKeys { events: Vec<(u32, u32)> }, + /// Set the seat's BASE keymap from a full XKB_KEYMAP_FORMAT_TEXT_V1 string. The + /// compositor's keymap policy rebuilds on top: overlay binds are re-spliced onto the new + /// base (same keycodes) and the combined keymap is applied in one swap. SetKeymapString(String), + /// Set the seat's BASE layout from RMLVO names (empty strings = xkbcommon defaults); + /// replies whether compilation succeeded. Overlay binds rebuild on top as for + /// `SetKeymapString`. + SetXkbLayout { + rules: String, + model: String, + layout: String, + variant: String, + options: String, + reply: std::sync::mpsc::Sender, + }, + /// Resolve keysyms to `(keycode, level)` against the seat keymap, overlay-binding every + /// keysym the base cannot produce — ONE keymap swap for the whole batch, and a keycode that + /// is currently pressed is never recycled. `(0, 0)` marks an unbindable keysym. Serves + /// computer-use only; selkies resolves its own keysyms and injects plain keycodes. + BindKeysyms { + keysyms: Vec, + reply: std::sync::mpsc::Sender>, + }, + /// Bind explicit `(keycode, keysym)` pairs onto the CURRENT base keymap and deliver it, + /// in one swap. The caller decides the assignment; this only assembles and applies, so the + /// base is neither re-sent nor recompiled and its reverse map is not rebuilt. Ordered with + /// key events on the one command channel, so the keys that need the new binds cannot + /// overtake it — no reply is awaited, and the caller's loop is never blocked on the swap. + SetKeymapOverlay { binds: Vec<(u32, u32)> }, + /// Debug/verification readback: currently pressed xkb keycodes plus the modifier state + /// bitmask (1 ctrl, 2 shift, 4 alt, 8 logo, 16 caps, 32 num, 64 altgr, 128 level5). + GetKeyboardState { + reply: std::sync::mpsc::Sender<(Vec, u32)>, + }, /// Reply with the smithay keyboard's keymap as an XKB_KEYMAP_FORMAT_TEXT_V1 string so a /// consumer (selkies) can build its reverse keysym map from the IDENTICAL keymap. GetXkbKeymap { reply: std::sync::mpsc::Sender }, + /// Ack once every previously queued command has been fully processed (the channel is + /// FIFO). The atexit sweep sends StopCapture + Barrier and waits, so the interpreter never + /// exits while the calloop thread is still mid-teardown (an NVENC/CUDA session drop racing + /// process exit segfaults). + Barrier { reply: std::sync::mpsc::Sender<()> }, PointerMotion { x: f64, y: f64 }, PointerRelativeMotion { dx: f64, dy: f64 }, /// `btn` is an evdev `BTN_` code by contract (e.g. 272 = BTN_LEFT, 273 = BTN_RIGHT, @@ -559,19 +713,29 @@ pub enum ThreadCommand { PointerButton { btn: u32, state: u32 }, PointerAxis { x: f64, y: f64 }, UpdateCursorConfig { render_on_framebuffer: bool }, - /// On-demand keyframe request (client reconnect / decoder reset): forces a send and an IDR - /// even on a static screen. - RequestIdr, - /// Live rate-control change for the Wayland calloop thread (parity with the X11 `rate_dirty` + /// Recreate the cursor theme handles at a new pixel size — the calloop's compositing + /// helper (the burned-in cursor) and, through its job channel, the `wl-cursor` worker's + /// (named-cursor PNG delivery). Replies false for a non-positive size. + SetCursorSize { size: i32, reply: std::sync::mpsc::Sender }, + /// On-demand keyframe request (client reconnect / decoder reset) for one display's + /// capture: forces a send and an IDR even on a static screen. + RequestIdr { display_id: u32 }, + /// Live rate-control change for one display's capture (parity with the X11 `rate_dirty` /// path). Each field is `None` when that dimension is unchanged. - UpdateRate { bitrate_kbps: Option, vbv_multiplier: Option, fps: Option }, - /// Live per-frame tunables (quality / paint-over / streaming / cursor), applied to the - /// calloop's settings and mirrored to the readback encode thread — no capture or encoder - /// restart. - UpdateTunables(LiveTunables), - CuScreenshot { resp: std::sync::mpsc::Sender> }, + UpdateRate { + display_id: u32, + bitrate_kbps: Option, + vbv_multiplier: Option, + fps: Option, + }, + /// Live per-frame tunables (quality / paint-over / streaming / cursor) for one + /// display's capture, mirrored to its readback encode thread — no restart. + UpdateTunables { display_id: u32, tunables: LiveTunables }, + /// One-shot PNG of one output's next rendered frame (0 = primary); an unknown + /// display id replies with an error immediately. + CuScreenshot { display_id: u32, resp: std::sync::mpsc::Sender, String>> }, CuCursorPosition { resp: std::sync::mpsc::Sender<(f64, f64)> }, - CuGetInfo { resp: std::sync::mpsc::Sender<(i32, i32, f64)> }, + CuGetInfo { display_id: u32, resp: std::sync::mpsc::Sender<(i32, i32, f64)> }, } /// Read the kernel driver bound to a render node for encoder routing. @@ -924,6 +1088,14 @@ impl WlEncodeControls { /// publish slot, so a deeper pool would only add latency (staler frames), never overlap. const WL_POOL_SURFACES: usize = 2; +/// Upper bounds for Python-supplied capture geometry: keeps a hostile or buggy setting from +/// turning into a multi-GB `vec![]` allocation that would abort the process. 16384 covers every +/// real display wall; the fps/scale bounds keep timing math finite and sane. +const MAX_CAPTURE_DIM: i32 = 16384; +const MAX_FPS: f64 = 1000.0; +const DEFAULT_FPS: f64 = 60.0; +const MAX_SCALE: f64 = 8.0; + /// Shared capture stats: whichever thread owns the encoders counts frames/stripes and /// composes `desc` + `n_stripes` (the encoder half of the 1 s debug log line); the calloop /// log loads, prints and resets the counters. @@ -950,6 +1122,8 @@ impl WlEncodeStats { /// flow through `controls`; damage and the IDR request arrive per-frame in `WlFrame`. struct WlEncodeConfig { settings: RustCaptureSettings, + /// Output/display id this encode loop serves; keys the recorder's delivery-layer tap. + display_id: u32, /// GLES readback is RGBA; the pixman framebuffer is BGRA. Selects CSC + encoder input kind. use_gpu: bool, /// Attempt a HW (NVENC/VAAPI) readback session before falling back to the CPU encoders. @@ -974,6 +1148,10 @@ struct WlEncodeConfig { /// compatible NVENC session is handed over from the previous encode thread it is reconfigured in /// place (milliseconds) rather than rebuilt, which would stall the stream. The EGL display is /// always null here: readback mode never imports dmabufs. +/// +/// Without the `gpl` feature there is no striped x264: an `output_mode == 1` request that reaches +/// the software fallback is served by the BSD-licensed OpenH264 full-frame encoder instead, so +/// `(None, None)` then means the striped JPEG path alone. fn build_readback_encoders( settings: &RustCaptureSettings, try_gpu: bool, @@ -986,12 +1164,23 @@ fn build_readback_encoders( return (None, Some(e)); } None => { + #[cfg(feature = "gpl")] eprintln!("[Wayland] OpenH264 init failed; falling back to x264 software."); + #[cfg(not(feature = "gpl"))] + eprintln!("[Wayland] OpenH264 init failed; no software H.264 encoder is available in this GPL-free build."); return (None, None); } } } if !try_gpu { + #[cfg(not(feature = "gpl"))] + if settings.output_mode == 1 && !settings.use_openh264 { + println!("[Wayland] pixelflux was built without GPL components (libx264 disabled); substituting the BSD-licensed OpenH264 software encoder."); + if let Some(e) = crate::encoders::oh264::Openh264Encoder::new(settings) { + return (None, Some(e)); + } + eprintln!("[Wayland] OpenH264 init failed; no software H.264 encoder is available in this GPL-free build."); + } return (None, None); } let encode_driver = get_gpu_driver(settings.encode_node_index.max(0)); @@ -1033,6 +1222,14 @@ fn build_readback_encoders( } } } + #[cfg(not(feature = "gpl"))] + if settings.output_mode == 1 && !settings.use_openh264 { + println!("[Wayland] pixelflux was built without GPL components (libx264 disabled); substituting the BSD-licensed OpenH264 software encoder."); + if let Some(e) = crate::encoders::oh264::Openh264Encoder::new(settings) { + return (None, Some(e)); + } + eprintln!("[Wayland] OpenH264 init failed; no software H.264 encoder is available in this GPL-free build."); + } (None, None) } @@ -1053,9 +1250,10 @@ fn build_readback_encoders( /// 2. **Dispatch by encoder**: a hardware session runs `decide_hw_fullframe`, then colorspace- /// converts only the frames actually being encoded (RGBA vs BGRA source chosen by the renderer) /// into the reused NV12/YUV444 buffer and calls `encode_raw`; OpenH264 encodes host pixels -/// directly; otherwise `encode_cpu` runs the striped x264/JPEG path with compositor damage. The -/// software H.264 path keeps an infinite GOP, forcing an IDR only on an explicit request or the -/// configured interval, and an explicit request also forces a full JPEG resend for joiners. +/// directly; otherwise `encode_cpu` runs the striped x264/JPEG path with compositor damage +/// (JPEG alone in a build without the `gpl` feature). The software H.264 path keeps an infinite +/// GOP, forcing an IDR only on an explicit request or the configured interval, and an explicit +/// request also forces a full JPEG resend for joiners. /// 3. **Recycle then deliver**: the capture buffer is recycled BEFORE delivery so a slow consumer /// never pins one, then the stripes go to the delivery thread through a single-slot `send` whose /// blocking is the backpressure that overlaps delivery with the next render + encode. @@ -1118,7 +1316,14 @@ fn wayland_encode_loop(pool: &WlFramePool, cfg: WlEncodeConfig) -> Option = Vec::new(); if let Some(ref mut encoder) = video_encoder { @@ -1133,6 +1338,19 @@ fn wayland_encode_loop(pool: &WlFramePool, cfg: WlEncodeConfig) -> Option YuvStandardMatrix::Bt601, + GpuEncoder::Vaapi(_) => YuvStandardMatrix::Bt709, + }; + // Limited range suits both: NVENC's ARGB CSC emits limited whatever the + // chroma format, so its session declares limited, and VA-API's own + // convert targets tv range as well. + let range = YuvRange::Limited; if settings.video_fullcolor { let y_size = (w * h) as usize; let (y_plane, rest) = nv12_buffer.split_at_mut(y_size); @@ -1148,9 +1366,9 @@ fn wayland_encode_loop(pool: &WlFramePool, cfg: WlEncodeConfig) -> Option Option Option out.push(EncodedStripe { - data, + data: Arc::new(data), data_type: 2, stripe_y_start: 0, stripe_height: height, @@ -1207,7 +1425,7 @@ fn wayland_encode_loop(pool: &WlFramePool, cfg: WlEncodeConfig) -> Option out.push(EncodedStripe { - data, + data: Arc::new(data), data_type: 2, stripe_y_start: 0, stripe_height: height, @@ -1241,17 +1459,18 @@ fn wayland_encode_loop(pool: &WlFramePool, cfg: WlEncodeConfig) -> Option settings.video_fullcolor, }; let cs_str = if is_444 { "CS_IN:I444" } else { "CS_IN:I420" }; - let range_str = if is_444 { "FR" } else { "LR" }; + // Only the software encoder carries 4:4:4 at full range; NVENC's hardware CSC is + // limited-range whatever the chroma format. + let range_str = if is_444 && video_encoder.is_none() { "FR" } else { "LR" }; let frame_str = if video_encoder.is_some() || openh264 || settings.video_fullframe { "FF" } else { @@ -1372,7 +1593,14 @@ fn log_stream_settings( log_msg.push_str(" Streaming"); } - log_msg.push_str(&format!(" | CRF: {}", settings.video_crf)); + if settings.video_cbr_mode { + log_msg.push_str(&format!(" | CBR {}", settings.video_bitrate_kbps)); + } else { + log_msg.push_str(&format!(" | CRF: {}", settings.video_crf)); + if settings.video_bitrate_kbps > 0 { + log_msg.push_str(&format!(" | VBV: {} kbps", settings.video_bitrate_kbps)); + } + } if settings.use_paint_over_quality { log_msg.push_str(&format!( @@ -1391,10 +1619,10 @@ fn log_stream_settings( } }; - let range_str = if is_actually_444 { - "I444 (Full Range)" - } else { - "I420 (Limited Range)" + let range_str = match (is_actually_444, video_encoder.is_none()) { + (true, true) => "I444 (Full Range)", + (true, false) => "I444 (Limited Range)", + _ => "I420 (Limited Range)", }; log_msg.push_str(&format!(" | Colorspace: {}", range_str)); } @@ -1407,242 +1635,443 @@ fn log_stream_settings( println!("{}", log_msg); } -/// The main execution loop of the Wayland backend. -/// -/// This function is the central nervous system of the backend. It runs on its own thread and owns -/// the entire lifecycle of the headless Wayland compositor: -/// -/// 1. **Initialization**: builds the `calloop` event loop and the Wayland display, raises -/// libwayland's per-client buffer limit when the newer setter is available (resolved at runtime -/// so the module still loads against older libwayland), and brings up the rendering pipeline — -/// GBM/EGL hardware acceleration on the resolved DRM render node, falling back to software -/// rendering (Pixman) when no node is usable. -/// 2. **State management**: constructs and holds the `AppState` — the Wayland globals (compositor, -/// seat, SHM, shell, dmabuf, selections, and the rest), the single virtual `HEADLESS-1` output, -/// the frame buffer and encode pools, and the window / cursor / encode bookkeeping. -/// 3. **Event dispatch**: -/// - **Command channel**: control messages from the Python thread — start/stop, input injection, -/// keymap and clipboard operations, live rate / tunable changes, and the computer-use queries. -/// - **Wayland socket**: accepts client connections and drives the compositor protocol. -/// 4. **StartCapture reconfigure**: reprograms the virtual output's mode / scale / refresh, resizes -/// the framebuffer and offscreen GBM buffer, and fullscreens mapped toplevels. The encode device -/// is resolved here: an operator's explicit `encode_node_index` (-1 software, >= 0 a device) -/// always wins, and only the unset `-2` sentinel is filled from the auto-picked render node. -/// H.264 output masks the dimensions even, because 4:2:0 needs even width and height. -/// 5. **Encode-path choice + render loop**: only a same-GPU GLES session encodes zero-copy on this -/// calloop thread, because the dmabuf and its EGL context are calloop-affine; every readback -/// flavor (OpenH264, striped x264/JPEG, Pixman, or a cross-GPU hardware encoder) builds its -/// encoders on the dedicated encode thread instead. A high-frequency timer composites the mapped -/// windows plus cursor and watermark, applies the shared paint-over / recovery-IDR policy, and -/// delivers the encoded stripes back to Python through the frame callback. The zero-copy encode -/// waits the GL render fence first, so a hardware encoder reading the dmabuf through CUDA/VA -/// never maps a half-rasterized (torn) frame. -/// 6. **Thread lifecycle**: the Python frame callback runs on a dedicated delivery thread so its -/// GIL never stalls calloop input / control dispatch, and in readback mode the encoders run on -/// the `wl-encode` thread. On a restart or stop the encode thread is torn down before the -/// delivery thread — it feeds the delivery sender and must be gone first — and the retained -/// callbacks are dropped and gated by a process-shutdown flag so nothing fires into a finalizing -/// interpreter. -fn run_wayland_thread( - command_rx: smithay::reexports::calloop::channel::Channel, - command_tx: smithay::reexports::calloop::channel::Sender, - initial_width: i32, - initial_height: i32, - explicit_dri_node: String, - auto_gpu_selected: bool, - cursor_size: i32, +/// Tear down a capture's encode/delivery threads and pools, returning any readback +/// hardware session for in-place reuse by a following start. The encode thread is joined +/// before the delivery sender drops (it feeds that sender), and the zero-copy session (if +/// any) is left on the capture for the caller to reuse or drop. +fn teardown_capture(cap: &mut wayland::frontend::WlCapture) -> Option { + if let Some(p) = cap.encode_pool.take() { + p.shutdown(); + } + let prior = cap.encode_join.take().and_then(|j| j.join().ok()).flatten(); + if let Some(tx) = cap.deliver_tx.take() { + drop(tx); + } + if let Some(j) = cap.deliver_join.take() { + let _ = j.join(); + } + cap.pending_hw_delivery = None; + cap.pending_hw_damage = false; + cap.recording_sink = None; + prior +} + +/// Stop the capture bound to `display_id`, leaving the output (and every other display's +/// capture) running. +fn stop_capture_on_display(state: &mut AppState, display_id: u32) { + let Some(idx) = state.node_idx_for_id(display_id) else { return }; + if let Some(mut cap) = state.output_nodes[idx].capture.take() { + println!("[Wayland] Capture loop stopped (display {display_id})."); + cap.video_encoder = None; + let _ = teardown_capture(&mut cap); + } + wayland_alive().lock().unwrap().remove(&display_id); +} + +/// Start (or in-place reconfigure) the capture bound to output `display_id`: reprogram the +/// output's mode/scale/refresh, size the render targets, fullscreen the display's windows at +/// the new logical size, resolve the encode path (zero-copy vs readback), and spawn the +/// delivery (and readback-mode encode) threads. The single-display behavior of the former +/// global StartCapture is preserved exactly for display 0. +/// Bring up the readback encode path (pixman readback → pool → encode thread) for a +/// capture whose zero-copy session is absent or just died. Mirrors the start-capture +/// bootstrap: u64::MAX content generations mark every pool slot stale so each one is +/// read back before its first publish, whatever the damage says. +fn bootstrap_readback_pool( + cap: &mut wayland::frontend::WlCapture, + display_id: u32, + use_gpu: bool, + try_gpu: bool, + prior: Option, ) { - let width: i32 = if initial_width > 0 { initial_width } else { 1024 }; - let height: i32 = if initial_height > 0 { initial_height } else { 768 }; + let Some(deliver_tx) = cap.deliver_tx.clone() else { + return; + }; + let settings = cap.settings.clone(); + let pool = Arc::new(WlFramePool::new( + WL_POOL_SURFACES, + (settings.width.max(0) as usize) * (settings.height.max(0) as usize) * 4, + )); + cap.pool_last_render = vec![0; WL_POOL_SURFACES]; + cap.render_seq = 0; + cap.pool_content_gen = vec![u64::MAX; WL_POOL_SURFACES]; + cap.content_gen = 0; + let c = &cap.encode_controls; + c.bitrate_kbps.store(settings.video_bitrate_kbps, Ordering::Relaxed); + c.vbv_mult_milli.store( + (settings.video_vbv_multiplier * 1000.0).round() as i32, + Ordering::Relaxed, + ); + c.fps_milli.store( + (settings.target_fps.max(1.0) * 1000.0) as u64, + Ordering::Relaxed, + ); + let cfg = WlEncodeConfig { + settings: settings.clone(), + display_id, + use_gpu, + try_gpu, + prior, + recording_sink: cap.recording_sink.clone(), + deliver_tx, + controls: cap.encode_controls.clone(), + stats: cap.encode_stats.clone(), + }; + let pool2 = pool.clone(); + cap.encode_join = Some( + thread::Builder::new() + .name(format!("wl-encode-{display_id}")) + .spawn(move || wayland_encode_loop(&pool2, cfg)) + .expect("failed to spawn wl-encode thread"), + ); + cap.encode_pool = Some(pool); +} - let mut event_loop = EventLoop::::try_new().expect("Unable to create event_loop"); - let display: Display = Display::new().unwrap(); - let dh: DisplayHandle = display.handle(); - unsafe { - if let Ok(lib) = libloading::Library::new("libwayland-server.so.0") { - if let Ok(set_max) = lib.get::( - b"wl_display_set_default_max_buffer_size\0", - ) { - set_max( - dh.backend_handle().display_ptr() as *mut std::ffi::c_void, - 10 * 1024 * 1024, - ); - } - std::mem::forget(lib); - } +/// Rebuild a broken zero-copy hardware session with the startup construction (driver +/// match, EGL display hand-over); VAAPI stays excluded under fullcolor exactly like +/// startup. `None` means unrecoverable — the caller demotes to readback. +fn rebuild_zerocopy_encoder( + cap: &wayland::frontend::WlCapture, + state: &mut AppState, +) -> Option { + let settings = &cap.settings; + let encode_driver = get_gpu_driver(settings.encode_node_index.max(0)); + if encode_driver.contains("nvidia") { + let egl_display = state + .gles_renderer + .as_ref() + .map(|r| r.egl_context().display().get_display_handle().handle) + .unwrap_or(std::ptr::null()); + NvencEncoder::new(settings, egl_display) + .ok() + .map(GpuEncoder::Nvenc) + } else if !settings.video_fullcolor { + VaapiEncoder::new(settings).ok().map(GpuEncoder::Vaapi) + } else { + None } +} - let dri_node = explicit_dri_node; +/// Consecutive encode failures before the zero-copy path recovers (~0.5s at 60fps): +/// a hiccup outlasts it, anything longer starts to look like a dead session. +const HW_ERROR_RECOVERY_THRESHOLD: u32 = 30; - let mut use_gpu = !dri_node.is_empty(); - let render_node_path = dri_node.clone(); +fn start_capture_on_display( + state: &mut AppState, + display_id: u32, + cb: Option>, + mut settings: RustCaptureSettings, +) { + use smithay::wayland::fractional_scale::with_fractional_scale; + + // The cursor worker outlives individual captures, so the starting settings have to + // reach it here — same point X11 applies the cap — or it keeps the previous capture's. + let _ = state.cursor_tx.send(CursorJob::SetSizeCap(settings.cursor_size_cap)); + + // Host-capture mode: connect on first use and point this display's capture + // thread at the requested size. The compositor keeps running (CU, clipboard + // callbacks, input fallbacks) but its renderer is bypassed. + if !settings.wayland_host_display.is_empty() { + if state.host.is_none() { + // Capture buffers come from the same render node the encoder imports + // from, resolved via its live fd (the path string is not retained in + // auto mode); each capture thread opens its own device handle. + let gbm_path = if state.use_gpu { + state.gbm_device.as_ref().and_then(|dev| { + use std::os::fd::{AsFd as _, AsRawFd as _}; + let fd = dev.as_fd().as_raw_fd(); + std::fs::read_link(format!("/proc/self/fd/{fd}")).ok() + }) + } else { + None + }; + match crate::wayland::host::HostSession::connect(&settings.wayland_host_display, gbm_path) + { + Ok(h) => { + println!( + "[HostCapture] capturing host compositor '{}' ({} outputs).", + settings.wayland_host_display, + h.output_count() + ); + state.host = Some(h); + } + Err(e) => eprintln!( + "[HostCapture] connect '{}' failed: {e}", + settings.wayland_host_display + ), + } + } + if let Some(host) = &state.host { + // The node's layout offset rides along so the host's heads mirror + // selkies' union layout (input coordinates already assume it). + if let Some(idx) = state.node_idx_for_id(display_id) { + let pos = state.output_nodes[idx].pos; + host.set_layout(display_id, pos.0, pos.1); + } + host.start_capture(display_id, settings.width, settings.height); + } + } - let mut gles_renderer = None; - let mut pixman_renderer = None; - let mut offscreen_buffer: Option<(BufferObject<()>, Dmabuf)> = None; - let mut dmabuf_global = None; - let mut gbm_device_raw = None; - let mut dmabuf_state = DmabufState::new(); + let Some(node_idx) = state.node_idx_for_id(display_id) else { + eprintln!("[Wayland] StartCapture: no output with display id {display_id}."); + return; + }; + let mut node = state.output_nodes.remove(node_idx); - let mut gpu_success = false; - if use_gpu { - println!("[Wayland] Initializing GL Renderer using device: {}", dri_node); - let init_res: Result<(), String> = (|| { - let device_path = std::path::Path::new(&dri_node); - let file = File::options().read(true).write(true).open(device_path) - .map_err(|e| format!("Failed to open render device: {}", e))?; - let file_for_alloc = file.try_clone() - .map_err(|e| format!("Failed to clone file for GBM Allocator: {}", e))?; - let gbm_allocator = RawGbmDevice::new(file_for_alloc) - .map_err(|_| "Failed to create Raw GBM Device")?; - let gbm = GbmDevice::new(file) - .map_err(|_| "Failed to create GBM device")?; - let egl = unsafe { EGLDisplay::new(gbm) } - .map_err(|_| "Failed to create EGL display")?; - let context = EGLContext::new(&egl) - .map_err(|_| "Failed to create EGL context")?; - let mut renderer = unsafe { GlesRenderer::new(context) } - .map_err(|_| "Failed to init GlesRenderer")?; - - if let Err(e) = renderer.bind_wl_display(&dh) { - println!("[Wayland] Warning: Failed to bind EGL to Wayland Display (Optional): {:?}", e); + if state.auto_gpu_selected && settings.encode_node_index < -1 { + if let Some(idx_str) = state.render_node_path.strip_prefix("/dev/dri/renderD") { + if let Ok(idx) = idx_str.parse::() { + settings.encode_node_index = idx - 128; } + } + } - let formats = Bind::::supported_formats(&renderer) - .ok_or("Failed to query formats")? - .into_iter() - .collect::>(); + if settings.output_mode == 1 { + settings.width &= !1; + settings.height &= !1; + } - let node = DrmNode::from_path(device_path) - .map_err(|_| "Failed to create DrmNode")?; - let dmabuf_default_feedback = DmabufFeedbackBuilder::new(node.dev_id(), formats.clone()).build(); + // Tear down this display's previous capture first; its hardware sessions are the + // reuse candidates below (zero-copy inline, readback via the encode config). + let mut prior_zero_copy: Option = None; + let mut prior_readback_encoder: Option = None; + if let Some(mut old) = node.capture.take() { + prior_zero_copy = old.video_encoder.take(); + prior_readback_encoder = teardown_capture(&mut old); + } - dmabuf_global = Some(if let Ok(default_feedback) = dmabuf_default_feedback { - dmabuf_state.create_global_with_default_feedback::(&dh, &default_feedback) + // Bind only after the old capture is gone: its sink unlinks the socket path in + // Drop, which would strip a fresh bind's filesystem name and leave every later + // recorder connect with ENOENT. + let recording_sink = crate::recording_sink::RecordingSink::try_bind(&settings.recording_socket); + + { + // Never panic the compositor thread: an output momentarily without a current + // mode falls back to the requested geometry so the reconfigure below is a + // no-op for size/refresh instead of unwrap-panicking. + let target_refresh = (settings.target_fps * 1000.0).round() as i32; + let (current_w, current_h, current_refresh) = match node.output.current_mode() { + Some(m) => (m.size.w, m.size.h, m.refresh), + None => (settings.width, settings.height, target_refresh), + }; + let current_scale = node.output.current_scale().fractional_scale(); + + if current_w != settings.width + || current_h != settings.height + || (current_scale - settings.scale).abs() > 0.001 + || current_refresh != target_refresh + { + // Allocate the GPU backing for the new dimensions BEFORE committing + // anything: if the driver refuses (VRAM exhaustion, dimensions it will + // not back), the whole reconfigure is skipped and the previous mode + + // buffers stay live. A failed resize must degrade to "no resize", never + // panic the compositor thread. + let mut new_offscreen = None; + let mut gbm_resize_failed = false; + if state.use_gpu { + if let Some(gbm) = state.gbm_device.as_mut() { + match gbm.create_buffer_object( + settings.width as u32, + settings.height as u32, + GbmFormat::Argb8888, + BufferObjectFlags::RENDERING, + ) { + Ok(bo) => { + let dmabuf = create_dmabuf_from_bo(&bo); + new_offscreen = Some((bo, dmabuf)); + } + Err(e) => { + eprintln!( + "[Wayland] GBM buffer resize to {}x{} failed ({:?}); keeping previous output mode.", + settings.width, settings.height, e + ); + gbm_resize_failed = true; + } + } + } + } + if gbm_resize_failed { + // The mode commit below is skipped wholesale, so the rest of this + // StartCapture (encoder setup, stored settings) must see the + // dimensions actually live. + settings.width = current_w; + settings.height = current_h; + settings.scale = current_scale; + settings.target_fps = current_refresh as f64 / 1000.0; } else { - dmabuf_state.create_global::(&dh, formats) - }); + println!( + "[Wayland] Configuring Output {} ({}): {}x{} @ {:.2} FPS (Scale {:.2})", + display_id, node.output.name(), + settings.width, settings.height, settings.target_fps, settings.scale + ); + let new_mode = OutputMode { + size: (settings.width, settings.height).into(), + refresh: target_refresh, + }; + node.output.change_current_state( + Some(new_mode), + Some(Transform::Normal), + Some(OutputScale::Fractional(settings.scale)), + Some(Point::from(node.pos)), + ); + node.output.set_preferred(new_mode); - let bo = gbm_allocator.create_buffer_object( - width as u32, height as u32, GbmFormat::Argb8888, BufferObjectFlags::RENDERING - ).map_err(|_| "Failed to allocate GBM buffer")?; + let pixel_count = + (settings.width.max(0) as usize) * (settings.height.max(0) as usize); + node.frame_buffer = vec![0u8; pixel_count * 4]; - let dmabuf = create_dmabuf_from_bo(&bo); - offscreen_buffer = Some((bo, dmabuf)); - gbm_device_raw = Some(gbm_allocator); - gles_renderer = Some(renderer); - Ok(()) - })(); + if let Some(off) = new_offscreen.take() { + node.offscreen_buffer = Some(off); + } + } + } - match init_res { - Ok(_) => gpu_success = true, - Err(e) => { - println!("[Wayland] GPU Initialization failed: {}. Falling back to Software Renderer (Pixman).", e); - use_gpu = false; + let scale = settings.scale.max(0.1); + let logical_width = (settings.width as f64 / scale).round() as i32; + let logical_height = (settings.height as f64 / scale).round() as i32; + + for window in state.space.elements() { + if wayland::frontend::window_output_id(window) != display_id { + continue; + } + if let Some(surface) = window.wl_surface() { + node.output.enter(&surface); + with_states(&surface, |states| { + with_fractional_scale(states, |fs| { + fs.set_preferred_scale(scale); + }); + }); + } + if let Some(toplevel) = window.toplevel() { + toplevel.with_pending_state(|state| { + use smithay::reexports::wayland_protocols::xdg::shell::server::xdg_toplevel::State; + state.states.set(State::Fullscreen); + state.states.set(State::Activated); + state.size = Some((logical_width, logical_height).into()); + }); + toplevel.send_configure(); } } } - if !gpu_success { - if !dri_node.is_empty() && !use_gpu { - } else { - println!("[Wayland] No render node. Initializing Software Renderer (Pixman)."); - } - pixman_renderer = Some(PixmanRenderer::new().expect("Failed to init PixmanRenderer")); - use_gpu = false; + let use_cpu_explicit = settings.use_cpu || settings.encode_node_index == -1; + let gpu_intent = settings.output_mode == 1 && !settings.use_openh264 && !use_cpu_explicit; + if use_cpu_explicit && !(settings.output_mode == 1 && settings.use_openh264) { + println!("[Wayland] CPU encoding selected (use_cpu=true or encode_node_index=-1)."); } - let compositor_state = CompositorState::new::(&dh); - let fractional_scale_state = FractionalScaleManagerState::new::(&dh); - let shm_state = ShmState::new::(&dh, vec![]); - let output_state = OutputManagerState::new_with_xdg_output::(&dh); - let mut seat_state = SeatState::new(); - let shell_state = XdgShellState::new::(&dh); - let space = Space::default(); - let layer_shell_state = WlrLayerShellState::new::(&dh); - let data_device_state = DataDeviceState::new::(&dh); - let data_control_state = DataControlState::new::(&dh, None, |_| true); - let virtual_keyboard_state = VirtualKeyboardManagerState::new::(&dh, |_client| true); - let pointer_warp_state = PointerWarpManager::new::(&dh); - let relative_pointer_state = RelativePointerManagerState::new::(&dh); - let pointer_constraints_state = PointerConstraintsState::new::(&dh); + let mut different_gpu = false; + if gpu_intent { + let encode_node_idx = settings.encode_node_index.max(0); + if !state.render_node_path.is_empty() + && !state.render_node_path.contains(&format!("renderD{}", 128 + encode_node_idx)) + { + different_gpu = true; + } + } - let foreign_toplevel_list = ForeignToplevelListState::new::(&dh); - let xdg_decoration_state = XdgDecorationState::new::(&dh); - let single_pixel_buffer = SinglePixelBufferState::new::(&dh); - let viewporter_state = ViewporterState::new::(&dh); - let presentation_state = PresentationState::new::(&dh, 1); - let xdg_activation_state = XdgActivationState::new::(&dh); - let primary_selection_state = PrimarySelectionState::new::(&dh); - let popups = PopupManager::default(); + let mut video_encoder: Option = None; + if gpu_intent && state.use_gpu && !different_gpu { + let encode_driver = get_gpu_driver(settings.encode_node_index.max(0)); + println!( + "[Wayland] Encode Node Index: {} | Driver: {}", + settings.encode_node_index.max(0), encode_driver + ); - let mut seat = seat_state.new_wl_seat(&dh, "seat0"); - seat.add_keyboard(XkbConfig::default(), 200, 25) - .expect("Failed to init keyboard"); - seat.add_pointer(); + if encode_driver.contains("nvidia") { + let reused = match prior_zero_copy.as_mut() { + Some(GpuEncoder::Nvenc(enc)) => match enc.reconfigure_resolution(&settings) { + Ok(()) => { + println!("[Wayland] NVENC session reconfigured in place."); + true + } + Err(e) => { + eprintln!("[Wayland] NVENC in-place reconfigure unavailable ({e}); rebuilding."); + false + } + }, + _ => false, + }; + if reused { + video_encoder = prior_zero_copy.take(); + } else { + prior_zero_copy = None; + println!("[Wayland] Nvidia Encoder detected. Initializing NVENC..."); + let egl_display = if let Some(renderer) = state.gles_renderer.as_ref() { + renderer.egl_context().display().get_display_handle().handle + } else { + std::ptr::null() + }; - let mut state = AppState { - compositor_state, - fractional_scale_state, - viewporter_state, - presentation_state, - shm_state, - single_pixel_buffer, - dmabuf_state, - dmabuf_global, - output_state, - seat_state, - shell_state, - layer_shell_state, - space, - data_device_state, - data_control_state, - dh: dh.clone(), - seat, - virtual_keyboard_state, - pointer_warp_state, - relative_pointer_state, - pointer_constraints_state, - outputs: Vec::new(), - pending_windows: Vec::new(), - foreign_toplevel_list, - xdg_decoration_state, - xdg_activation_state, - primary_selection_state, - popups, - frame_buffer: vec![0u8; (width.max(0) as usize) * (height.max(0) as usize) * 4], - gles_renderer, - pixman_renderer, - gbm_device: gbm_device_raw, - offscreen_buffer, - is_capturing: false, - settings: RustCaptureSettings { - width, - height, - ..RustCaptureSettings::default() - }, - callback: None, - cursor_callback: None, - clipboard_callback: None, - pending_clipboard_read: None, - last_log_time: Instant::now(), - start_time: Instant::now(), - clock: Clock::new(), - frame_counter: 0, - pending_force_idr: false, - needs_full_render: true, - use_gpu, - video_encoder: None, + match NvencEncoder::new(&settings, egl_display) { + Ok(encoder) => { + video_encoder = Some(GpuEncoder::Nvenc(encoder)); + println!("[Wayland] NVENC Encoder initialized successfully."); + } + Err(e) => eprintln!( + "[Wayland] Failed to init NVENC: {}. Falling back to CPU.", + e + ), + } + } + } else { + prior_zero_copy = None; + println!("[Wayland] Initializing Unified VAAPI Encoder..."); + if settings.video_fullcolor { + println!("[Wayland] 4:4:4 Fullcolor requested. VAAPI does not support this profile reliably. Falling back to CPU."); + } else { + match VaapiEncoder::new(&settings) { + Ok(encoder) => { + video_encoder = Some(GpuEncoder::Vaapi(encoder)); + println!("[Wayland] VAAPI Encoder initialized successfully."); + } + Err(e) => eprintln!( + "[Wayland] Failed to init VAAPI: {}. Falling back to CPU.", + e + ), + } + } + } + } + drop(prior_zero_copy); + + if different_gpu { + println!("[Wayland] Decision: Rendering and Encoding GPUs differ -> Forcing Readback (CPU path for pixels)."); + } + if video_encoder.is_none() { + println!("[Wayland] Decision: Readback path (encode thread) active."); + } else if !different_gpu { + println!("[Wayland] Decision: Zero-Copy path active."); + } + + if recording_sink.is_some() && settings.output_mode == 0 { + eprintln!( + "[recording_sink] WARNING: recording_socket is set but output_mode is JPEG (0). \ + The recording socket requires a single H.264 stream. Please set output_mode=1 \ + on the Python CaptureSettings to produce a recordable output." + ); + } + + // Every display's capture composites its own watermark, uploaded at this output's + // scale and placed against this output's frame dimensions. + let watermark_output_scale = node.output.current_scale().fractional_scale(); + node.overlay_state + .load_watermark(&settings.watermark_path, watermark_output_scale); + if display_id == 0 { + state.settings = settings.clone(); + if state.cursor_callback_set { + if let Some(icon) = state.current_cursor_icon.clone() { + state.send_cursor_image(&icon); + } + } + } + state.render_cursor_on_framebuffer = settings.capture_cursor; + + let mut cap = wayland::frontend::WlCapture { + settings: settings.clone(), + video_encoder, vaapi_state: StripeState::default(), - cursor_helper: Cursor::load(cursor_size), - overlay_state: OverlayState::default(), - current_cursor_icon: None, - cursor_buffer: None, - cursor_cache: std::collections::HashMap::new(), - render_cursor_on_framebuffer: false, - render_node_path, - auto_gpu_selected, - recording_sink: None, + recording_sink, deliver_tx: None, deliver_join: None, pending_hw_delivery: None, @@ -1655,435 +2084,1803 @@ fn run_wayland_thread( render_seq: 0, pool_content_gen: Vec::new(), content_gen: 0, - pending_screenshot: None, + frame_counter: 0, + pending_force_idr: false, + needs_full_render: true, + last_tick: None, + hw_error_streak: 0, + hw_rebuilt: false, }; - let output = Output::new( - "HEADLESS-1".into(), - PhysicalProperties { - size: (width, height).into(), - subpixel: Subpixel::Unknown, - make: "Pixelflux".into(), - model: "Virtual".into(), - serial_number: "001".into(), - }, - ); - output.change_current_state( - Some(OutputMode { - size: (width, height).into(), - refresh: 60_000, - }), - Some(Transform::Normal), - Some(OutputScale::Fractional(1.0)), - Some((0, 0).into()), - ); - output.set_preferred(OutputMode { - size: (width, height).into(), - refresh: 60_000, - }); - state.space.map_output(&output, (0, 0)); - state.outputs.push(output.clone()); - let _global = output.create_global::(&dh); - let mut damage_tracker = OutputDamageTracker::from_output(&output); + { + let (tx, rx) = std::sync::mpsc::sync_channel::>(1); + // With no Python callback (internal recorder-owned capture) the delivery thread + // only drains the channel: the recorder already consumed the frames at the + // delivery-layer tap, upstream of this per-consumer handoff. + let join = match cb { + Some(cb) => thread::spawn(move || { + crate::boost_thread_priority(-10); + while let Ok(stripes) = rx.recv() { + if PY_SHUTDOWN.load(Ordering::Relaxed) { continue; } + Python::attach(|py| { + for s in stripes { + match Py::new(py, StripeFrame::new_owned_meta( + s.data, s.data_type, s.stripe_y_start, + s.stripe_height, s.frame_id, + )) { + Ok(f) => { if let Err(e) = cb.call1(py, (f,)) { e.print(py); } } + Err(e) => eprintln!("[wayland] frame alloc error: {e:?}"), + } + } + }); + } + }), + None => thread::spawn(move || while rx.recv().is_ok() {}), + }; + cap.deliver_tx = Some(tx); + cap.deliver_join = Some(join); + } - event_loop - .handle() - .insert_source(command_rx, move |event, _, state| { - match event { - CalloopEvent::Msg(ThreadCommand::StartCapture(cb, mut settings)) => { - if state.auto_gpu_selected && settings.encode_node_index < -1 { - if let Some(idx_str) = state.render_node_path.strip_prefix("/dev/dri/renderD") { - if let Ok(idx) = idx_str.parse::() { - settings.encode_node_index = idx - 128; + if cap.video_encoder.is_none() { + bootstrap_readback_pool( + &mut cap, + display_id, + state.use_gpu, + gpu_intent && (!state.use_gpu || different_gpu), + prior_readback_encoder.take(), + ); + } else { + cap.encode_stats.n_stripes.store(1, Ordering::Relaxed); + *cap.encode_stats.desc.lock().unwrap() = + encoder_desc(&settings, cap.video_encoder.as_ref(), false, true); + log_stream_settings(&settings, 1, cap.video_encoder.as_ref(), false); + } + drop(prior_readback_encoder); + // Force the keyframe unconditionally: the damage tracker and offscreen buffer + // stay warm across stop/start, so a restarted capture on a static screen + // otherwise produces no damage, no first frame, and no IDR in either path. + cap.request_idr(); + + node.capture = Some(cap); + state.output_nodes.insert(node_idx, node); + wayland_alive().lock().unwrap().insert(display_id); +} + +/// One output's render + capture tick: composite the elements overlapping this output +/// (positions made output-local by subtracting its layout origin), track damage, feed the +/// display's own encode path, and answer a pending screenshot on the primary. Returns true +/// when the tick was skipped because this display's encode pool was still busy (the caller +/// then retries shortly instead of waiting a full frame interval). +/// Stamp the watermark element onto `target` in place — no clear, so the +/// captured content underneath stays — returning the draw's sync point for the +/// encoder to wait on. +fn draw_host_watermark( + renderer: &mut GlesRenderer, + overlay: &crate::encoders::overlay::OverlayState, + target: &mut Dmabuf, + size: (i32, i32), +) -> Result { + let elem = overlay + .get_watermark_element(renderer) + .ok_or("watermark element unavailable")?; + let mut fb = renderer.bind(target).map_err(|e| format!("bind: {e:?}"))?; + let mut frame = renderer + .render(&mut fb, (size.0, size.1).into(), Transform::Normal) + .map_err(|e| format!("render: {e:?}"))?; + let dst = elem.geometry(1.0.into()); + let local = Rectangle::from_size(dst.size); + elem.draw(&mut frame, elem.src(), dst, &[local], &[]) + .map_err(|e| format!("draw: {e:?}"))?; + frame.finish().map_err(|e| format!("finish: {e:?}")) +} + +/// Re-compose `src` (the retained host frame) plus the watermark into `target`: +/// the path a moving watermark needs, since re-drawing over the same retained +/// buffer would leave trails. +fn compose_host_watermark( + renderer: &mut GlesRenderer, + overlay: &crate::encoders::overlay::OverlayState, + src: &Dmabuf, + target: &mut Dmabuf, + size: (i32, i32), +) -> Result { + let elem = overlay + .get_watermark_element(renderer) + .ok_or("watermark element unavailable")?; + let tex = renderer + .import_dmabuf(src, None) + .map_err(|e| format!("import: {e:?}"))?; + let mut fb = renderer.bind(target).map_err(|e| format!("bind: {e:?}"))?; + let full: Rectangle = Rectangle::from_size((size.0, size.1).into()); + let mut frame = renderer + .render(&mut fb, (size.0, size.1).into(), Transform::Normal) + .map_err(|e| format!("render: {e:?}"))?; + frame + .render_texture_from_to( + &tex, + Rectangle::from_size((size.0 as f64, size.1 as f64).into()), + full, + &[full], + // Opaque: the capture format's undefined alpha must not blend, it + // would leave the target's previous content (and stamped + // watermarks) underneath. + &[full], + Transform::Normal, + 1.0, + None, + &[], + ) + .map_err(|e| format!("texture: {e:?}"))?; + let dst = elem.geometry(1.0.into()); + let local = Rectangle::from_size(dst.size); + elem.draw(&mut frame, elem.src(), dst, &[local], &[]) + .map_err(|e| format!("draw: {e:?}"))?; + frame.finish().map_err(|e| format!("finish: {e:?}")) +} + +fn warn_once_host_watermark(e: &str) { + static WARNED: std::sync::atomic::AtomicBool = std::sync::atomic::AtomicBool::new(false); + if !WARNED.swap(true, Ordering::Relaxed) { + eprintln!("[HostCapture] watermark compositing failed: {e}"); + } +} + +fn render_node_tick( + state: &mut AppState, + node: &mut wayland::frontend::OutputNode, + is_memory_throttling: bool, +) -> bool { + let take_screenshot = state + .pending_screenshot + .as_ref() + .is_some_and(|(id, _)| *id == node.id); + if node.capture.is_none() && !take_screenshot { + return false; + } + + // Per-display frame pacing under the one shared timer (which fires at the fastest + // active capture's rate). + if let Some(cap) = node.capture.as_ref() { + if !take_screenshot { + let fps = (if is_memory_throttling { 5.0 } else { cap.settings.target_fps }).max(1.0); + if let Some(last) = cap.last_tick { + if last.elapsed().as_secs_f64() < (1.0 / fps) * 0.9 { + return false; + } + } + } + } + + let output = node.output.clone(); + let origin: Point = node.pos.into(); + let output_scale_val = output.current_scale().fractional_scale(); + let (width, height) = match node.capture.as_ref() { + Some(c) => (c.settings.width, c.settings.height), + None => output + .current_mode() + .map(|m| (m.size.w, m.size.h)) + .unwrap_or((0, 0)), + }; + if width <= 0 || height <= 0 { + return false; + } + if node.frame_buffer.len() < (width as usize) * (height as usize) * 4 { + node.frame_buffer = vec![0u8; (width as usize) * (height as usize) * 4]; + } + let logical_w = (width as f64 / output_scale_val).round(); + let logical_h = (height as f64 / output_scale_val).round(); + + // A recorder connecting counts as an IDR request, kept armed across skipped ticks. + if let Some(cap) = node.capture.as_mut() { + if cap + .recording_sink + .as_ref() + .map(|s| s.should_force_idr()) + .unwrap_or(false) + { + cap.request_idr(); + } + } + let requested_idr = node.capture.as_ref().map(|c| c.pending_force_idr).unwrap_or(false); + // A client keyframe request lands on the hardware path's atomic (RequestIdr sets + // it whenever an encode pool exists); host mode consults it — without consuming — + // to decide whether a static screen must re-encode its retained frame. + let hw_idr_pending = node + .capture + .as_ref() + .map(|c| c.encode_controls.force_idr.load(Ordering::Relaxed)) + .unwrap_or(false); + let want_idr_for_host = requested_idr || hw_idr_pending; + + let mut pool_slot: Option<(usize, Vec)> = None; + if let Some(cap) = node.capture.as_ref() { + if let Some(ref pool) = cap.encode_pool { + if !is_memory_throttling { + pool_slot = pool.try_begin(); + if pool_slot.is_none() { + return true; + } + } + } + } + + let loc_enum = node + .capture + .as_ref() + .map(|c| c.settings.watermark_location_enum) + .unwrap_or(state.settings.watermark_location_enum); + node.overlay_state.update_position(width, height, loc_enum); + + if let Some(cap) = node.capture.as_mut() { + cap.last_tick = Some(Instant::now()); + } + + // The cursor is composited only on the output the pointer is on, at that output's + // scale; its position is output-local. + let pointer_local: Option> = state + .seat + .get_pointer() + .map(|p| p.current_location()) + .and_then(|pos| { + let rect = Rectangle::::new( + origin.to_f64(), + (logical_w, logical_h).into(), + ); + if rect.contains(pos) { + Some(pos - origin.to_f64()) + } else { + None + } + }); + + let mut render_success = false; + let mut render_sync = None; + let mut damage_rects: Vec> = Vec::new(); + let needs_full = node.capture.as_ref().map(|c| c.needs_full_render).unwrap_or(true); + + // Host-capture mode: the host compositor already blitted this display's frame + // into one of our buffers (screencopy); adopt it in place of compositing. + let host_mode = state.host.as_ref().map(|h| h.has_output_for(node.id)).unwrap_or(false); + if state.host.is_some() && !host_mode { + // No host output backs this display (start_capture already warned): + // produce nothing rather than the compositor's own empty content. + if let Some((id, buf)) = pool_slot.take() { + if let Some(cap) = node.capture.as_ref() { + if let Some(ref pool) = cap.encode_pool { + pool.cancel(id, buf); + } + } + } + return false; + } + // Dmabuf handed to the GPU encoder in host mode (from the new or retained frame). + let mut host_enc_dmabuf: Option = None; + if host_mode { + const RETAINED_OK: u8 = 0; + const RETAINED_NONE: u8 = 1; + const RETAINED_MISMATCH: u8 = 2; + let host_idx = node.id; + let gpu_encoder = node + .capture + .as_ref() + .map(|c| c.video_encoder.is_some()) + .unwrap_or(false); + // The session steps out of `state` while frames are adopted so the + // renderer can composite the watermark / serve screenshot readbacks. + let host = state.host.take().unwrap(); + let new_frame = host.try_take_frame(host_idx); + let have_new = new_frame.is_some(); + // Streaming mode wants a constant-rate stream (the client's decoder pipeline + // is built for it), so re-encode the retained frame every tick like the + // compositor path does. Outside streaming mode, stay damage-driven, waking + // only for a pending IDR (a viewer opening its keyframe gate), a screenshot + // request, or a bouncing watermark that must keep moving. + let streaming = node + .capture + .as_ref() + .map(|c| c.settings.video_streaming_mode) + .unwrap_or(false); + let wm_active = node.overlay_state.is_active(); + let wm_animated = wm_active && node.overlay_state.is_animated(); + if !have_new && !want_idr_for_host && !streaming && !take_screenshot && !wm_animated { + if let Some((id, buf)) = pool_slot.take() { + if let Some(cap) = node.capture.as_ref() { + if let Some(ref pool) = cap.encode_pool { + pool.cancel(id, buf); + } + } + } + state.host = Some(host); + return false; + } + if let Some(f) = new_frame { + host.retain_frame(host_idx, f); + } + // Consume the (new or prior) retained frame's content into the pool slot / + // GPU dmabuf. The frame stays retained for the next IDR. The CPU path + // blends the watermark in place; the GPU path composites it below. Damage + // is the fresh blit's; a re-encode of retained content has none of its own. + let mut wm_drawn = false; + let outcome = host.with_retained(host_idx, |r| { + let Some(f) = r else { return RETAINED_NONE }; + damage_rects = if have_new { f.damage.clone() } else { Vec::new() }; + if let Some(cpu) = f.cpu.as_ref() { + if gpu_encoder { + return RETAINED_MISMATCH; // software frames, GPU encoder + } + if let Some((_, ref mut buf)) = pool_slot { + cpu.write_bgra(f.width, f.height, buf); + if wm_active { + node.overlay_state.blend_bgra(buf, (f.width as usize) * 4, f.width, f.height); + wm_drawn = true; + } + } + cpu.write_bgra(f.width, f.height, &mut node.frame_buffer); + if wm_active { + node.overlay_state + .blend_bgra(&mut node.frame_buffer, (f.width as usize) * 4, f.width, f.height); + } + } else if let Some(dmabuf) = f.dmabuf.as_ref() { + if !gpu_encoder { + return RETAINED_MISMATCH; // GPU frames, CPU encoder + } + host_enc_dmabuf = Some(dmabuf.clone()); + } + RETAINED_OK + }); + // GPU path: composite the watermark and serve screenshot readbacks with + // the renderer. A bouncing watermark re-composes retained content into + // this display's offscreen target every tick (drawing in place would + // trail); anchored watermarks are stamped once onto each fresh blit and + // ride along with retained re-encodes. + if let Some(src) = host_enc_dmabuf.clone() { + if wm_active { + if let Some(renderer) = state.gles_renderer.as_mut() { + if wm_animated { + if let Some((_, target)) = node.offscreen_buffer.as_mut() { + match compose_host_watermark( + renderer, + &node.overlay_state, + &src, + target, + (width, height), + ) { + Ok(sync) => { + render_sync = Some(sync); + host_enc_dmabuf = Some(target.clone()); + wm_drawn = true; + } + Err(e) => warn_once_host_watermark(&e), + } + } + } else if have_new { + let mut target = src.clone(); + match draw_host_watermark( + renderer, + &node.overlay_state, + &mut target, + (width, height), + ) { + Ok(sync) => { + render_sync = Some(sync); + wm_drawn = true; } + Err(e) => warn_once_host_watermark(&e), } } - - if settings.output_mode == 1 { - settings.width &= !1; - settings.height &= !1; + } + } + if take_screenshot { + if let Some(renderer) = state.gles_renderer.as_mut() { + let mut shot = host_enc_dmabuf.clone().unwrap_or(src); + match renderer.bind(&mut shot) { + Ok(fb) => { + let rect = Rectangle::new((0, 0).into(), (width, height).into()); + match renderer.copy_framebuffer(&fb, rect, Fourcc::Abgr8888) { + Ok(mapping) => match renderer.map_texture(&mapping) { + Ok(data) => { + let n = data.len().min(node.frame_buffer.len()); + node.frame_buffer[..n].copy_from_slice(&data[..n]); + } + Err(e) => eprintln!("[HostCapture] screenshot map: {e:?}"), + }, + Err(e) => eprintln!("[HostCapture] screenshot copy: {e:?}"), + } + } + Err(e) => eprintln!("[HostCapture] screenshot bind: {e:?}"), + }; + } + } + } + if wm_drawn { + if let Some(rect) = node.overlay_state.damage_rect(width, height) { + damage_rects.push(rect); + } + } + state.host = Some(host); + if outcome != RETAINED_OK { + if outcome == RETAINED_MISMATCH { + static WARNED: std::sync::atomic::AtomicBool = + std::sync::atomic::AtomicBool::new(false); + if !WARNED.swap(true, Ordering::Relaxed) { + eprintln!( + "[HostCapture] host frame type and encoder mismatch \ + (GPU host needs a GPU encoder; software host needs a CPU encoder)." + ); + } + } + if let Some((id, buf)) = pool_slot.take() { + if let Some(cap) = node.capture.as_ref() { + if let Some(ref pool) = cap.encode_pool { + pool.cancel(id, buf); } + } + } + return false; + } + render_success = true; + } - state.recording_sink = - crate::recording_sink::RecordingSink::try_bind(&settings.recording_socket); - - if let Some(output) = state.outputs.first() { - let current_mode = output.current_mode().unwrap(); - let current_w = current_mode.size.w; - let current_h = current_mode.size.h; - let current_scale = output.current_scale().fractional_scale(); - let current_refresh = current_mode.refresh; - let target_refresh = (settings.target_fps * 1000.0).round() as i32; - - let scale = settings.scale.max(0.1); - let logical_width = (settings.width as f64 / scale).round() as i32; - let logical_height = (settings.height as f64 / scale).round() as i32; - - if current_w != settings.width - || current_h != settings.height - || (current_scale - settings.scale).abs() > 0.001 - || current_refresh != target_refresh - { - // Allocate the GPU backing for the new dimensions BEFORE - // committing anything: if the driver refuses (VRAM - // exhaustion, dimensions it will not back), the whole - // reconfigure is skipped and the previous mode + buffers - // stay live. A failed resize must degrade to "no resize", - // never panic the compositor thread — that would drop - // every Wayland client with no recovery short of a - // process restart. (The INITIAL allocation path already - // degrades gracefully, to the software renderer.) - let mut new_offscreen = None; - let mut gbm_resize_failed = false; - if state.use_gpu { - if let Some(gbm) = state.gbm_device.as_mut() { - match gbm.create_buffer_object( - settings.width as u32, - settings.height as u32, - GbmFormat::Argb8888, - BufferObjectFlags::RENDERING, - ) { - Ok(bo) => { - let dmabuf = create_dmabuf_from_bo(&bo); - new_offscreen = Some((bo, dmabuf)); - } - Err(e) => { - eprintln!( - "[Wayland] GBM buffer resize to {}x{} failed ({:?}); keeping previous output mode.", - settings.width, settings.height, e - ); - gbm_resize_failed = true; + if !host_mode && state.use_gpu { + if let Some(renderer) = state.gles_renderer.as_mut() { + let mut cap = node.capture.as_mut(); + if let Some((_bo, dmabuf)) = node.offscreen_buffer.as_mut() { + let render_age = if node.overlay_state.is_animated() || needs_full { 0 } else { 1 }; + match renderer.bind(dmabuf) { + Ok(mut frame) => { + let mut elements: Vec>> = Vec::new(); + + if state.render_cursor_on_framebuffer { + if let Some(pos) = pointer_local { + let scale = Scale::from(output_scale_val); + + if let Some(CursorImageStatus::Named(icon)) = &state.current_cursor_icon { + let name = wayland::frontend::cursor_icon_to_str(icon); + let time = Duration::from_millis(state.clock.now().as_millis() as u64); + if let Some(image) = state.cursor_helper.get_image_by_name(name, output_scale_val.round() as u32, time) { + if let Some(elem) = node.overlay_state.get_cursor_element(renderer, image, pos, output_scale_val) { + elements.push(CompositionElements::Cursor(elem)); } } + } else if let Some(CursorImageStatus::Surface(surface)) = &state.current_cursor_icon { + let phys_pos = pos.to_physical(scale); + let elem_result = with_states(surface, |states| { + WaylandSurfaceRenderElement::from_surface(renderer, surface, states, phys_pos, 1.0, smithay::backend::renderer::element::Kind::Cursor) + }); + if let Ok(Some(cursor_elem)) = elem_result { + elements.push(CompositionElements::Surface(cursor_elem)); + } + } else if state.current_cursor_icon.is_none() { + let time = Duration::from_millis(state.clock.now().as_millis() as u64); + let image = state.cursor_helper.get_image(output_scale_val.round() as u32, time); + if let Some(elem) = node.overlay_state.get_cursor_element(renderer, image, pos, output_scale_val) { + elements.push(CompositionElements::Cursor(elem)); + } } } - if gbm_resize_failed { - // The mode commit below is skipped wholesale, so the - // rest of this StartCapture (encoder setup, stored - // settings) must also see the dimensions that are - // actually live — otherwise the encoder and the - // still-old buffers disagree on frame geometry. - settings.width = current_w; - settings.height = current_h; - settings.scale = current_scale; - settings.target_fps = current_refresh as f64 / 1000.0; - } else { - println!( - "[Wayland] Configuring Output: {}x{} @ {:.2} FPS (Scale {:.2})", - settings.width, settings.height, settings.target_fps, settings.scale - ); - let new_mode = OutputMode { - size: (settings.width, settings.height).into(), - refresh: target_refresh, - }; - output.change_current_state( - Some(new_mode), - Some(Transform::Normal), - Some(OutputScale::Fractional(settings.scale)), - Some((0, 0).into()), - ); - output.set_preferred(new_mode); + } - let pixel_count = - (settings.width.max(0) as usize) * (settings.height.max(0) as usize); - state.frame_buffer = vec![0u8; pixel_count * 4]; + if let Some(elem) = node.overlay_state.get_watermark_element(renderer) { + elements.push(CompositionElements::Cursor(elem)); + } - if let Some(off) = new_offscreen.take() { - state.offscreen_buffer = Some(off); + { + let layer_map = layer_map_for_output(&output); + + let draw_layer = |renderer: &mut GlesRenderer, elements: &mut Vec>>, target_layer: smithay::wayland::shell::wlr_layer::Layer| { + for surface in layer_map.layers().rev() { + let current_layer = surface.layer(); + if current_layer == target_layer { + if let Some(geo) = layer_map.layer_geometry(surface) { + let elem = smithay::wayland::compositor::with_states(surface.wl_surface(), |states| { + WaylandSurfaceRenderElement::from_surface( + renderer, surface.wl_surface(), states, + geo.loc.to_physical_precise_round(output_scale_val), 1.0, + smithay::backend::renderer::element::Kind::Unspecified + ) + }); + if let Ok(Some(e)) = elem { + elements.push(CompositionElements::Surface(e)); + } + } + } } - } + }; + + draw_layer(renderer, &mut elements, smithay::wayland::shell::wlr_layer::Layer::Overlay); + draw_layer(renderer, &mut elements, smithay::wayland::shell::wlr_layer::Layer::Top); } - for window in state.space.elements() { + for window in state.space.elements_for_output(&output).collect::>().into_iter().rev() { + let window_loc = state.space.element_location(window).unwrap_or_default() - origin; + if let Some(surface) = window.wl_surface() { - output.enter(&surface); - with_states(&surface, |states| { - smithay::wayland::fractional_scale::with_fractional_scale(states, |fs| { - fs.set_preferred_scale(scale); + let popups = PopupManager::popups_for_surface(&surface); + for (popup, location) in popups { + let popup_surface = popup.wl_surface(); + let popup_pos = window_loc + location; + let elem = smithay::wayland::compositor::with_states(popup_surface, |states| { + WaylandSurfaceRenderElement::from_surface( + renderer, + popup_surface, + states, + popup_pos.to_physical_precise_round(output_scale_val), + 1.0, + smithay::backend::renderer::element::Kind::Unspecified + ) }); - }); - } - if let Some(toplevel) = window.toplevel() { - toplevel.with_pending_state(|state| { - use smithay::reexports::wayland_protocols::xdg::shell::server::xdg_toplevel::State; - state.states.set(State::Fullscreen); - state.states.set(State::Activated); - state.size = Some((logical_width, logical_height).into()); - }); - toplevel.send_configure(); + if let Ok(Some(e)) = elem { + elements.push(CompositionElements::Surface(e)); + } + } } - } - } - let use_cpu_explicit = settings.use_cpu || settings.encode_node_index == -1; - let gpu_intent = - settings.output_mode == 1 && !settings.use_openh264 && !use_cpu_explicit; - if use_cpu_explicit && !(settings.output_mode == 1 && settings.use_openh264) { - println!("[Wayland] CPU encoding selected (use_cpu=true or encode_node_index=-1)."); - } - - let mut different_gpu = false; - if gpu_intent { - let encode_node_idx = settings.encode_node_index.max(0); - if !state.render_node_path.is_empty() - && !state.render_node_path.contains(&format!("renderD{}", 128 + encode_node_idx)) - { - different_gpu = true; + elements.extend(window.render_elements(renderer, window_loc.to_physical_precise_round(output_scale_val), Scale::from(output_scale_val), 1.0).into_iter().map(CompositionElements::Space)); } - } - if gpu_intent && state.use_gpu && !different_gpu { - let encode_driver = get_gpu_driver(settings.encode_node_index.max(0)); - println!( - "[Wayland] Encode Node Index: {} | Driver: {}", - settings.encode_node_index.max(0), encode_driver - ); - - if encode_driver.contains("nvidia") { - let reused = match state.video_encoder.as_mut() { - Some(GpuEncoder::Nvenc(enc)) => { - match enc.reconfigure_resolution(&settings) { - Ok(()) => { - println!("[Wayland] NVENC session reconfigured in place."); - true - } - Err(e) => { - eprintln!("[Wayland] NVENC in-place reconfigure unavailable ({e}); rebuilding."); - false + { + let layer_map = layer_map_for_output(&output); + + let draw_layer = |renderer: &mut GlesRenderer, elements: &mut Vec>>, target_layer: smithay::wayland::shell::wlr_layer::Layer| { + for surface in layer_map.layers() { + let current_layer = surface.layer(); + if current_layer == target_layer { + if let Some(geo) = layer_map.layer_geometry(surface) { + let elem = smithay::wayland::compositor::with_states(surface.wl_surface(), |states| { + WaylandSurfaceRenderElement::from_surface( + renderer, surface.wl_surface(), states, + geo.loc.to_physical_precise_round(output_scale_val), 1.0, + smithay::backend::renderer::element::Kind::Unspecified + ) + }); + if let Ok(Some(e)) = elem { + elements.push(CompositionElements::Surface(e)); + } } } } - _ => false, }; - if !reused { - state.video_encoder = None; - println!("[Wayland] Nvidia Encoder detected. Initializing NVENC..."); - let egl_display = if let Some(renderer) = state.gles_renderer.as_ref() { - renderer.egl_context().display().get_display_handle().handle - } else { - std::ptr::null() - }; - match NvencEncoder::new(&settings, egl_display) { - Ok(encoder) => { - state.video_encoder = Some(GpuEncoder::Nvenc(encoder)); - println!("[Wayland] NVENC Encoder initialized successfully."); - } - Err(e) => eprintln!( - "[Wayland] Failed to init NVENC: {}. Falling back to CPU.", - e - ), + draw_layer(renderer, &mut elements, smithay::wayland::shell::wlr_layer::Layer::Bottom); + draw_layer(renderer, &mut elements, smithay::wayland::shell::wlr_layer::Layer::Background); + } + match node.damage_tracker.render_output(renderer, &mut frame, render_age, &elements, [0.1, 0.1, 0.1, 1.0]) { + Ok(result) => { + render_success = true; + if let Some(damage) = result.damage { + damage_rects = damage.clone(); + } + render_sync = Some(result.sync); + if let Some(c) = cap.as_deref_mut() { + c.needs_full_render = false; } + }, + Err(e) => eprintln!("Render error: {:?}", e) + } + if let Some(c) = cap { + if !damage_rects.is_empty() { + c.content_gen += 1; } - } else { - state.video_encoder = None; - println!("[Wayland] Initializing Unified VAAPI Encoder..."); - if settings.video_fullcolor { - println!("[Wayland] 4:4:4 Fullcolor requested. VAAPI does not support this profile reliably. Falling back to CPU."); - } else { - match VaapiEncoder::new(&settings) { - Ok(encoder) => { - state.video_encoder = Some(GpuEncoder::Vaapi(encoder)); - println!( - "[Wayland] VAAPI Encoder initialized successfully." + if let Some((id, ref mut buf)) = pool_slot { + // No-damage ticks skip the readback, so a pooled buffer can + // lag the offscreen target whenever the encoder held the + // other slot across a tick; one catch-up readback keeps + // every published buffer current. + if render_success && c.pool_content_gen[id] != c.content_gen { + let _ = renderer.with_context(|gl| unsafe { + gl.ReadPixels( + 0, + 0, + width, + height, + smithay::backend::renderer::gles::ffi::RGBA, + smithay::backend::renderer::gles::ffi::UNSIGNED_BYTE, + buf.as_mut_ptr() as *mut std::ffi::c_void, ); - } - Err(e) => eprintln!( - "[Wayland] Failed to init VAAPI: {}. Falling back to CPU.", - e - ), + }); + c.pool_content_gen[id] = c.content_gen; } } } - } else { - state.video_encoder = None; - } + if pool_slot.is_none() && take_screenshot { + let _ = renderer.with_context(|gl| unsafe { + gl.ReadPixels( + 0, + 0, + width, + height, + smithay::backend::renderer::gles::ffi::RGBA, + smithay::backend::renderer::gles::ffi::UNSIGNED_BYTE, + node.frame_buffer.as_mut_ptr() as *mut std::ffi::c_void, + ); + }); + } + }, + Err(e) => eprintln!("Failed to bind buffer: {:?}", e) + } + } + } + } else if !host_mode { + if let Some(renderer) = state.pixman_renderer.as_mut() { + let mut cap = node.capture.as_mut(); + let (ptr, buf_age) = match pool_slot { + Some((id, ref mut buf)) => { + let age = cap + .as_ref() + .map(|c| { + if c.pool_last_render[id] == 0 { + 0 + } else { + (c.render_seq + 1 - c.pool_last_render[id]) as usize + } + }) + .unwrap_or(0); + (buf.as_mut_ptr() as *mut u32, age) + } + None => (node.frame_buffer.as_mut_ptr() as *mut u32, 0), + }; + let mut image = unsafe { + pixman::Image::from_raw_mut(pixman::FormatCode::A8R8G8B8, width as usize, height as usize, ptr, (width as usize) * 4, false).expect("Failed to create pixman image") + }; + match renderer.bind(&mut image) { + Ok(mut frame) => { + let mut elements: Vec>> = Vec::new(); + + if state.render_cursor_on_framebuffer { + if let Some(pos) = pointer_local { + let scale = Scale::from(output_scale_val); + + if let Some(CursorImageStatus::Named(icon)) = &state.current_cursor_icon { + let name = wayland::frontend::cursor_icon_to_str(icon); + let time = Duration::from_millis(state.clock.now().as_millis() as u64); + if let Some(image) = state.cursor_helper.get_image_by_name(name, output_scale_val.round() as u32, time) { + if let Some(elem) = node.overlay_state.get_cursor_element(renderer, image, pos, output_scale_val) { + elements.push(CompositionElements::Cursor(elem)); + } + } + } else if let Some(CursorImageStatus::Surface(surface)) = &state.current_cursor_icon { + let phys_pos = pos.to_physical(scale); + let elem_result = with_states(surface, |states| { + WaylandSurfaceRenderElement::from_surface(renderer, surface, states, phys_pos, 1.0, smithay::backend::renderer::element::Kind::Cursor) + }); + if let Ok(Some(cursor_elem)) = elem_result { + elements.push(CompositionElements::Surface(cursor_elem)); + } + } else if state.current_cursor_icon.is_none() { + let time = Duration::from_millis(state.clock.now().as_millis() as u64); + let image = state.cursor_helper.get_image(output_scale_val.round() as u32, time); + if let Some(elem) = node.overlay_state.get_cursor_element(renderer, image, pos, output_scale_val) { + elements.push(CompositionElements::Cursor(elem)); + } + } + } + } - if different_gpu { - println!("[Wayland] Decision: Rendering and Encoding GPUs differ -> Forcing Readback (CPU path for pixels)."); - } - if state.video_encoder.is_none() { - println!("[Wayland] Decision: Readback path (encode thread) active."); - } else if !different_gpu { - println!("[Wayland] Decision: Zero-Copy path active."); - } + if let Some(elem) = node.overlay_state.get_watermark_element(renderer) { + elements.push(CompositionElements::Cursor(elem)); + } + + { + let layer_map = layer_map_for_output(&output); + + let draw_layer = |renderer: &mut PixmanRenderer, elements: &mut Vec>>, target_layer: smithay::wayland::shell::wlr_layer::Layer| { + for surface in layer_map.layers() { + let current_layer = surface.layer(); + if current_layer == target_layer { + if let Some(geo) = layer_map.layer_geometry(surface) { + let elem = smithay::wayland::compositor::with_states(surface.wl_surface(), |states| { + WaylandSurfaceRenderElement::from_surface( + renderer, surface.wl_surface(), states, + geo.loc.to_physical_precise_round(output_scale_val), 1.0, + smithay::backend::renderer::element::Kind::Unspecified + ) + }); + if let Ok(Some(e)) = elem { + elements.push(CompositionElements::Surface(e)); + } + } + } + } + }; + + draw_layer(renderer, &mut elements, smithay::wayland::shell::wlr_layer::Layer::Overlay); + draw_layer(renderer, &mut elements, smithay::wayland::shell::wlr_layer::Layer::Top); + } + + for window in state.space.elements_for_output(&output).collect::>().into_iter().rev() { + let loc = state.space.element_location(window).unwrap_or_default() - origin; + + if let Some(surface) = window.wl_surface() { + let popups = PopupManager::popups_for_surface(&surface); + for (popup, location) in popups { + let popup_surface = popup.wl_surface(); { + let popup_pos = loc + location; + let elem = smithay::wayland::compositor::with_states(popup_surface, |states| { + WaylandSurfaceRenderElement::from_surface( + renderer, + popup_surface, + states, + popup_pos.to_physical_precise_round(output_scale_val), + 1.0, + smithay::backend::renderer::element::Kind::Unspecified + ) + }); + if let Ok(Some(e)) = elem { + elements.push(CompositionElements::Surface(e)); + } + } + } + } + + elements.extend(window.render_elements(renderer, loc.to_physical_precise_round(output_scale_val), Scale::from(output_scale_val), 1.0).into_iter().map(CompositionElements::Space)); + } + + { + let layer_map = layer_map_for_output(&output); + + let draw_layer = |renderer: &mut PixmanRenderer, elements: &mut Vec>>, target_layer: smithay::wayland::shell::wlr_layer::Layer| { + for surface in layer_map.layers() { + let current_layer = surface.layer(); + if current_layer == target_layer { + if let Some(geo) = layer_map.layer_geometry(surface) { + let elem = smithay::wayland::compositor::with_states(surface.wl_surface(), |states| { + WaylandSurfaceRenderElement::from_surface( + renderer, surface.wl_surface(), states, + geo.loc.to_physical_precise_round(output_scale_val), 1.0, + smithay::backend::renderer::element::Kind::Unspecified + ) + }); + if let Ok(Some(e)) = elem { + elements.push(CompositionElements::Surface(e)); + } + } + } + } + }; + + draw_layer(renderer, &mut elements, smithay::wayland::shell::wlr_layer::Layer::Bottom); + draw_layer(renderer, &mut elements, smithay::wayland::shell::wlr_layer::Layer::Background); + } + + let render_age = if node.overlay_state.is_animated() || needs_full { 0 } else { buf_age }; + match node.damage_tracker.render_output(renderer, &mut frame, render_age, &elements, [0.1, 0.1, 0.1, 1.0]) { + Ok(result) => { + render_success = true; + if let Some(c) = cap.as_deref_mut() { + c.needs_full_render = false; + } + if let Some(damage) = result.damage { damage_rects = damage.clone(); } + }, + Err(e) => eprintln!("Render error: {:?}", e) + } + if let Some(c) = cap { + c.render_seq += 1; + if render_success { + if let Some((id, _)) = pool_slot { + c.pool_last_render[id] = c.render_seq; + } + } + } + }, + Err(e) => eprintln!("Failed to bind pixman image: {:?}", e) + } + } + } + + if render_success { + let time = state.clock.now(); + for window in state.space.elements_for_output(&output).cloned().collect::>() { + window.send_frame(&output, time, Some(Duration::ZERO), |_, _| Some(output.clone())); + } + + if let Some(cap) = node.capture.as_mut() { + // A dead encode thread (panic, unexpected exit) cannot drain the pool; + // every publish would park the slot and each tick would silently skip + // while is_capturing still reports true. Rebuild the readback path in + // place, reusing a cleanly handed-back encoder session if any. + if cap.encode_join.as_ref().is_some_and(|j| j.is_finished()) { + let prior = cap + .encode_join + .take() + .and_then(|j| j.join().ok().flatten()); + if let Some(pool) = cap.encode_pool.take() { + pool.shutdown(); + } + let s = &cap.settings; + let try_gpu = s.output_mode == 1 + && !s.use_openh264 + && !(s.use_cpu || s.encode_node_index == -1); + eprintln!("[Wayland] encode thread died; rebuilding the readback path."); + bootstrap_readback_pool(cap, node.id, state.use_gpu, try_gpu, prior); + cap.request_idr(); + } + if is_memory_throttling { + if cap.encode_pool.is_none() { + cap.frame_counter = cap.frame_counter.wrapping_add(1); + } + } else if cap.encode_pool.is_some() { + if take_screenshot { + if let Some((_, ref buf)) = pool_slot { + let n = buf.len().min(node.frame_buffer.len()); + node.frame_buffer[..n].copy_from_slice(&buf[..n]); + } + } + if let Some((id, buf)) = pool_slot.take() { + let is_animated = node.overlay_state.is_animated(); + cap.encode_pool.as_ref().unwrap().publish(WlFrame { + id, + buf, + frame_id: cap.frame_counter, + damage: std::mem::take(&mut damage_rects), + is_animated, + }); + cap.frame_counter = cap.frame_counter.wrapping_add(1); + } + } else if let Some(ref mut encoder) = cap.video_encoder { + // Deliver the parked frame (if any) first, WITHOUT blocking: this runs + // on the calloop thread, and a blocking send would freeze + // input/command/Wayland dispatch for as long as the Python consumer + // stalls. While a frame stays parked, no new frame is encoded — an + // encoded frame joins the H.264 reference chain and can never be + // dropped — and the tick's damage is latched so the pause never loses + // a change. + let slot_free = match cap.pending_hw_delivery.take() { + None => true, + Some(pending) => match cap.deliver_tx.as_ref() { + None => true, + Some(tx) => match tx.try_send(pending) { + Ok(()) => true, + Err(std::sync::mpsc::TrySendError::Full(p)) => { + cap.pending_hw_delivery = Some(p); + false + } + Err(std::sync::mpsc::TrySendError::Disconnected(_)) => true, + }, + }, + }; + if !slot_free { + if !damage_rects.is_empty() { + cap.pending_hw_damage = true; + } + } else { + let is_animated = node.overlay_state.is_animated(); + let had_damage = !damage_rects.is_empty() + || std::mem::take(&mut cap.pending_hw_damage); + let decision = crate::pipeline::decide_hw_fullframe( + &mut cap.vaapi_state, + &cap.settings, + cap.frame_counter, + had_damage, + is_animated, + requested_idr, + ); + let send_frame = decision.send; + let force_idr = decision.force_idr; + let target_qp = decision.target_qp; + + let mut frame_out = false; + if send_frame { + if let Some(sync) = render_sync.take() { + let _ = sync.wait(); + } + // Host-capture frames encode from the buffer the host blitted + // into; otherwise from this display's own composited buffer. + let enc_dmabuf: Option = host_enc_dmabuf + .clone() + .or_else(|| node.offscreen_buffer.as_ref().map(|(_, d)| d.clone())); + let result = match encoder { + GpuEncoder::Nvenc(enc) => { + if let Some(ref dmabuf) = enc_dmabuf { + enc.encode(dmabuf, cap.frame_counter as u64, target_qp, force_idr) + } else { + Err("NVENC ZeroCopy requires offscreen buffer (GPU context)".to_string()) + } + }, + GpuEncoder::Vaapi(enc) => { + if let Some(ref dmabuf) = enc_dmabuf { + enc.encode_dmabuf(dmabuf, cap.frame_counter as u64, target_qp, force_idr) + } else { + Err("Vaapi ZeroCopy requires offscreen buffer (GPU context)".to_string()) + } + } + }; + + if let Ok(data) = result { + cap.hw_error_streak = 0; + cap.hw_rebuilt = false; + if !data.is_empty() { + frame_out = true; + cap.encode_stats.frames.fetch_add(1, Ordering::Relaxed); + cap.encode_stats.stripes.fetch_add(1, Ordering::Relaxed); + if let Some(ref tx) = cap.deliver_tx { + let stripes = vec![EncodedStripe { + data: Arc::new(data), data_type: 2, stripe_y_start: 0, + stripe_height: height, frame_id: cap.frame_counter as i32, + }]; + if let Some(ref socket) = cap.recording_sink { + socket.write_frame(&stripes, height); + } + crate::recorder::wayland_tap(node.id, &stripes); + // Non-blocking: a full slot parks the frame (delivered + // ahead of any new encode above). + match tx.try_send(stripes) { + Ok(()) => {} + Err(std::sync::mpsc::TrySendError::Full(s)) => { + cap.pending_hw_delivery = Some(s); + } + Err(std::sync::mpsc::TrySendError::Disconnected(_)) => {} + } + } + } + } else if let Err(e) = result { + eprintln!("HW Encode Error: {}", e); + cap.hw_error_streak = cap.hw_error_streak.saturating_add(1); + if cap.hw_error_streak == HW_ERROR_RECOVERY_THRESHOLD { + // The zero-copy session persistently fails after having + // worked (driver hiccup, CUDA pressure from a co-tenant): + // rebuild the session once, else demote to the readback + // path. Streaming black frames forever is not an option. + // A session whose encodes keep failing still constructs, so + // the rebuild only counts as recovery until the next streak; + // otherwise the stream would rebuild in a loop and never demote. + let rebuilt = if cap.hw_rebuilt { + None + } else { + rebuild_zerocopy_encoder(cap, state) + }; + match rebuilt { + Some(enc) => { + cap.video_encoder = Some(enc); + cap.pending_force_idr = true; + cap.hw_rebuilt = true; + eprintln!("[Wayland] zero-copy HW encoder rebuilt after repeated encode errors."); + } + None => { + eprintln!("[Wayland] zero-copy HW encoder unrecoverable; demoting to readback encode."); + cap.video_encoder = None; + cap.hw_rebuilt = false; + // Mirror the startup intent: readback still + // tries the GPU unless the operator opted out. + let s = &cap.settings; + let try_gpu = s.output_mode == 1 + && !s.use_openh264 + && !(s.use_cpu || s.encode_node_index == -1); + bootstrap_readback_pool( + cap, node.id, state.use_gpu, try_gpu, None, + ); + } + } + cap.hw_error_streak = 0; + } + } + } + // An unserved request stays armed: on an infinite GOP an IDR lost to an + // encode error would never self-heal. + cap.pending_force_idr = requested_idr && !frame_out; + cap.frame_counter = cap.frame_counter.wrapping_add(1); + } + } + } + if take_screenshot { + if let Some((_, resp)) = state.pending_screenshot.take() { + if !node.frame_buffer.is_empty() { + let w = width as u32; + let h = height as u32; + let png = if state.use_gpu { + crate::computer_use::encode_png_rgba(&node.frame_buffer, w, h) + } else { + let mut rgba = node.frame_buffer.clone(); + for px in rgba.chunks_exact_mut(4) { + px.swap(0, 2); + } + crate::computer_use::encode_png_rgba(&rgba, w, h) + }; + match png { + Ok(data) => { let _ = resp.send(Ok(data)); } + Err(e) => { + let _ = resp.send(Err(format!("PNG encode error: {e}"))); + eprintln!("[ComputerUse] PNG encode error: {}", e); + } + } + } else { + let _ = resp.send(Err("Screenshot render produced no pixels".to_string())); + } + } + } + } + if let Some((id, buf)) = pool_slot.take() { + if let Some(cap) = node.capture.as_ref() { + if let Some(ref pool) = cap.encode_pool { + pool.cancel(id, buf); + } + } + } + false +} + +/// True when the rectangles `(x, y, w, h)` overlap: strict interior intersection, so +/// touching edges do not count and empty (non-positive-dimension) rectangles never +/// overlap anything. Arithmetic is widened so extreme coordinates cannot wrap. +fn rects_overlap(a: (i32, i32, i32, i32), b: (i32, i32, i32, i32)) -> bool { + let (ax, ay, aw, ah) = (a.0 as i64, a.1 as i64, a.2 as i64, a.3 as i64); + let (bx, by, bw, bh) = (b.0 as i64, b.1 as i64, b.2 as i64, b.3 as i64); + aw > 0 && ah > 0 && bw > 0 && bh > 0 + && ax < bx + bw && bx < ax + aw + && ay < by + bh && by < ay + ah +} + +/// An overlapping output as `(id, flavor, rect)`, where flavor names which rectangle +/// pair matched and rect is `(x, y, width, height)`. +type OutputOverlap = (u32, &'static str, (i32, i32, i32, i32)); + +/// The first live output (excluding `skip_id`) whose rectangle overlaps a candidate +/// placement. Both rectangle flavors are checked — logical +/// (Space layout, scale-divided) and physical (mode pixels at the same origin) — because +/// input injection and cursor compositing key off the physical rects while window layout +/// keys off the logical ones, and neither may overlap. +fn find_output_overlap( + nodes: &[wayland::frontend::OutputNode], + skip_id: Option, + logical: (i32, i32, i32, i32), + physical: (i32, i32, i32, i32), +) -> Option { + for n in nodes { + if Some(n.id) == skip_id { + continue; + } + if let Some(geo) = n.logical_geometry() { + let other = (geo.loc.x, geo.loc.y, geo.size.w, geo.size.h); + if rects_overlap(logical, other) { + return Some((n.id, "logical", other)); + } + } + if let Some(mode) = n.output.current_mode() { + let other = (n.pos.0, n.pos.1, mode.size.w, mode.size.h); + if rects_overlap(physical, other) { + return Some((n.id, "physical", other)); + } + } + } + None +} + +/// Create an additional output mapped into the layout at `(x, y)`. Fails (false) on a +/// duplicate id, non-positive geometry/scale, a rectangle overlapping a live output, or a +/// GPU render-target allocation failure. Only Create/Reposition placements are validated: +/// a capture reconfigure (StartCapture on an existing output) resizes UNVALIDATED, so +/// keeping a multi-step relayout overlap-free at every step is the caller's ordering +/// responsibility. +fn create_output_on( + state: &mut AppState, + id: u32, + width: i32, + height: i32, + x: i32, + y: i32, + scale: f64, +) -> bool { + if state.node_idx_for_id(id).is_some() || width <= 0 || height <= 0 || scale <= 0.0 { + return false; + } + let logical_size = ( + (width as f64 / scale).round() as i32, + (height as f64 / scale).round() as i32, + ); + if let Some((oid, flavor, other)) = find_output_overlap( + &state.output_nodes, + None, + (x, y, logical_size.0, logical_size.1), + (x, y, width, height), + ) { + eprintln!( + "[Wayland] CreateOutput {id}: rejected, {flavor} rect {}x{}+{x}+{y} overlaps output {oid} at {}x{}+{}+{}.", + if flavor == "logical" { logical_size.0 } else { width }, + if flavor == "logical" { logical_size.1 } else { height }, + other.2, other.3, other.0, other.1, + ); + return false; + } + let output = Output::new( + format!("HEADLESS-{}", id + 1), + PhysicalProperties { + size: (width, height).into(), + subpixel: Subpixel::Unknown, + make: "Pixelflux".into(), + model: "Virtual".into(), + serial_number: format!("{:03}", id + 1), + }, + ); + let mode = OutputMode { size: (width, height).into(), refresh: 60_000 }; + output.change_current_state( + Some(mode), + Some(Transform::Normal), + Some(OutputScale::Fractional(scale)), + Some((x, y).into()), + ); + output.set_preferred(mode); + let mut offscreen = None; + if state.use_gpu { + let Some(gbm) = state.gbm_device.as_mut() else { return false }; + match gbm.create_buffer_object( + width as u32, + height as u32, + GbmFormat::Argb8888, + BufferObjectFlags::RENDERING, + ) { + Ok(bo) => { + let dmabuf = create_dmabuf_from_bo(&bo); + offscreen = Some((bo, dmabuf)); + } + Err(e) => { + eprintln!("[Wayland] CreateOutput {id}: GBM allocation {width}x{height} failed ({e:?})."); + return false; + } + } + } + state.space.map_output(&output, (x, y)); + let global = output.create_global::(&state.dh); + let damage_tracker = OutputDamageTracker::from_output(&output); + if let Some(host) = state.host.as_ref() { + host.set_layout(id, x, y); + } + println!("[Wayland] Output {id} created: {width}x{height} @ ({x}, {y}) scale {scale:.2}."); + state.output_nodes.push(wayland::frontend::OutputNode { + id, + output, + global, + pos: (x, y), + damage_tracker, + frame_buffer: vec![0u8; (width.max(0) as usize) * (height.max(0) as usize) * 4], + offscreen_buffer: offscreen, + overlay_state: OverlayState::default(), + capture: None, + }); + // A nested session opens one host toplevel per screen and the extras wait + // parked until a display exists for them: hand the newest waiting window to + // the new output. Windows stacked on an output rather than parked — anything + // placed before this compositor started parking them — remain candidates. + let mut counts: Vec<(u32, usize)> = Vec::new(); + for w in state.space.elements() { + let oid = wayland::frontend::window_output_id(w); + match counts.iter_mut().find(|(o, _)| *o == oid) { + Some((_, c)) => *c += 1, + None => counts.push((oid, 1)), + } + } + let newest = |pred: &dyn Fn(&smithay::desktop::Window) -> bool| { + state + .space + .elements() + .filter(|w| pred(w)) + .max_by_key(|w| wayland::frontend::window_meta(w).map(|m| m.id).unwrap_or(0)) + .cloned() + }; + let adopt = newest(&|w| { + wayland::frontend::window_meta(w) + .map(|m| m.parked.load(std::sync::atomic::Ordering::Relaxed)) + .unwrap_or(false) + }) + .or_else(|| { + newest(&|w| { + let oid = wayland::frontend::window_output_id(w); + counts.iter().any(|(o, c)| *o == oid && *c >= 2) + }) + }); + if let Some(window) = adopt { + state.place_window_on_output(&window, id); + println!( + "[Wayland] Output {id}: adopted waiting window {}.", + wayland::frontend::window_meta(&window).map(|m| m.id).unwrap_or(0) + ); + } + true +} + +/// Move an existing output (the primary included) to layout offset `(x, y)`. The output's +/// advertised position, its Space mapping, and the windows placed on it (mapped at the +/// output's origin — window positions are output-relative under forced fullscreen) all +/// follow, so absolute input injection and cursor compositing — both keyed off `node.pos` — +/// resolve against the new layout immediately. A destination overlapping another live +/// output is refused (false). As with `CreateOutput`, only the placement itself is +/// validated: a capture reconfigure (StartCapture on an existing output) resizes +/// UNVALIDATED, so keeping a multi-step relayout overlap-free at every step is the +/// caller's ordering responsibility. +fn reposition_output_on(state: &mut AppState, id: u32, x: i32, y: i32) -> bool { + let Some(idx) = state.node_idx_for_id(id) else { return false }; + let output = state.output_nodes[idx].output.clone(); + if state.output_nodes[idx].pos == (x, y) { + return true; + } + let logical_size = state.output_nodes[idx] + .logical_geometry() + .map(|g| (g.size.w, g.size.h)) + .unwrap_or((0, 0)); + let physical_size = output.current_mode().map(|m| (m.size.w, m.size.h)).unwrap_or((0, 0)); + if let Some((oid, flavor, other)) = find_output_overlap( + &state.output_nodes, + Some(id), + (x, y, logical_size.0, logical_size.1), + (x, y, physical_size.0, physical_size.1), + ) { + eprintln!( + "[Wayland] RepositionOutput {id}: rejected, {flavor} rect {}x{}+{x}+{y} overlaps output {oid} at {}x{}+{}+{}.", + if flavor == "logical" { logical_size.0 } else { physical_size.0 }, + if flavor == "logical" { logical_size.1 } else { physical_size.1 }, + other.2, other.3, other.0, other.1, + ); + return false; + } + state.output_nodes[idx].pos = (x, y); + if let Some(host) = state.host.as_ref() { + host.set_layout(id, x, y); + } + output.change_current_state(None, None, None, Some((x, y).into())); + state.space.map_output(&output, (x, y)); + let windows: Vec = state + .space + .elements() + .filter(|w| wayland::frontend::window_output_id(w) == id) + .cloned() + .collect(); + for window in &windows { + state.space.map_element(window.clone(), (x, y), false); + } + if let Some(cap) = state.output_nodes[idx].capture.as_mut() { + cap.needs_full_render = true; + } + println!("[Wayland] Output {id} repositioned to ({x}, {y})."); + true +} + +/// Destroy a secondary output: end its capture, relocate its windows onto the primary +/// output, unmap it from the space, and retract its global. The primary (id 0) is refused. +fn destroy_output_on(state: &mut AppState, id: u32) -> bool { + if id == 0 { + return false; + } + let Some(_) = state.node_idx_for_id(id) else { return false }; + stop_capture_on_display(state, id); + if let Some(host) = state.host.as_ref() { + host.idle_output(id); + } + wayland_owners().lock().unwrap().remove(&id); + // Relocate while the node is still registered so output leave/enter both resolve. + let windows: Vec = state + .space + .elements() + .filter(|w| wayland::frontend::window_output_id(w) == id) + .cloned() + .collect(); + for window in &windows { + // The primary keeps whichever screen it already shows: a nested session's + // second screen parks again rather than covering the first. + if state.would_cover_screen(window, 0) { + state.park_window(window, 0); + } else { + state.place_window_on_output(window, 0); + } + } + for w in &state.pending_windows { + if let Some(meta) = wayland::frontend::window_meta(w) { + if meta.output.load(Ordering::Relaxed) == id { + meta.output.store(0, Ordering::Relaxed); + } + } + } + let idx = state.node_idx_for_id(id).unwrap(); + let node = state.output_nodes.remove(idx); + state.space.unmap_output(&node.output); + state.dh.remove_global::(node.global); + println!( + "[Wayland] Output {id} destroyed; {} window(s) relocated to primary.", + windows.len() + ); + true +} +/// Startup inputs for the compositor thread: its two calloop channels, the sender it hands +/// to computer-use, the initial geometry, and the GPU / cursor policy. +struct WaylandThreadConfig { + command_rx: smithay::reexports::calloop::channel::Channel, + wake_rx: smithay::reexports::calloop::channel::Channel<()>, + command_tx: smithay::reexports::calloop::channel::Sender, + initial_width: i32, + initial_height: i32, + explicit_dri_node: String, + auto_gpu_selected: bool, + cursor_size: i32, +} + + +/// The main execution loop of the Wayland backend. +/// +/// This function is the central nervous system of the backend. It runs on its own thread and owns +/// the entire lifecycle of the headless Wayland compositor: +/// +/// 1. **Initialization**: builds the `calloop` event loop and the Wayland display, raises +/// libwayland's per-client buffer limit when the newer setter is available (resolved at runtime +/// so the module still loads against older libwayland), and brings up the rendering pipeline — +/// GBM/EGL hardware acceleration on the resolved DRM render node, falling back to software +/// rendering (Pixman) when no node is usable. +/// 2. **State management**: constructs and holds the `AppState` — the Wayland globals (compositor, +/// seat, SHM, shell, dmabuf, selections, and the rest) plus the output registry: the primary +/// virtual `HEADLESS-1` output at layout (0, 0), extended at runtime by CreateOutput with +/// additional outputs at their layout offsets, each `OutputNode` owning its damage tracker, +/// render targets, and (at most one) capture pipeline. +/// 3. **Event dispatch**: +/// - **Command channel**: control messages from the Python thread — per-display start/stop, +/// output lifecycle (create/destroy/list/move-window), input injection routed across the +/// output layout, keymap and clipboard operations, live rate / tunable changes, and the +/// computer-use queries. +/// - **Wayland socket**: accepts client connections and drives the compositor protocol. +/// 4. **StartCapture reconfigure** (per display): reprograms that output's mode / scale / refresh, +/// resizes its framebuffer and offscreen GBM buffer, and fullscreens the toplevels placed on +/// it. The encode device is resolved here: an operator's explicit `encode_node_index` +/// (-1 software, >= 0 a device) always wins, and only the unset `-2` sentinel is filled from +/// the auto-picked render node. H.264 output masks the dimensions even, because 4:2:0 needs +/// even width and height. +/// 5. **Encode-path choice + render loop**: only a same-GPU GLES session encodes zero-copy on this +/// calloop thread, because the dmabuf and its EGL context are calloop-affine; every readback +/// flavor (OpenH264, striped x264/JPEG, Pixman, or a cross-GPU hardware encoder) builds its +/// encoders on that display's dedicated encode thread instead. A shared timer (paced at the +/// fastest active capture) renders each capturing output — its windows, popups and layers made +/// output-local, the cursor only on the pointer's output — applies the shared paint-over / +/// recovery-IDR policy per display, and delivers each display's encoded stripes through its own +/// frame callback. The zero-copy encode waits the GL render fence first, so a hardware encoder +/// reading the dmabuf through CUDA/VA never maps a half-rasterized (torn) frame. +/// 6. **Thread lifecycle**: the Python frame callback runs on a dedicated delivery thread so its +/// GIL never stalls calloop input / control dispatch, and in readback mode the encoders run on +/// the `wl-encode` thread. On a restart or stop the encode thread is torn down before the +/// delivery thread — it feeds the delivery sender and must be gone first — and the retained +/// callbacks are dropped and gated by a process-shutdown flag so nothing fires into a finalizing +/// interpreter. +fn run_wayland_thread(cfg: WaylandThreadConfig) { + let WaylandThreadConfig { + command_rx, + wake_rx, + command_tx, + initial_width, + initial_height, + explicit_dri_node, + auto_gpu_selected, + cursor_size, + } = cfg; + let width: i32 = if initial_width > 0 { initial_width } else { 1024 }; + let height: i32 = if initial_height > 0 { initial_height } else { 768 }; + + let mut event_loop = match EventLoop::::try_new() { + Ok(l) => l, + Err(e) => { + eprintln!("[Wayland] compositor thread aborting: event loop init failed: {e}"); + return; + } + }; + let display: Display = match Display::new() { + Ok(d) => d, + Err(e) => { + eprintln!("[Wayland] compositor thread aborting: display init failed: {e}"); + return; + } + }; + let dh: DisplayHandle = display.handle(); + unsafe { + if let Ok(lib) = libloading::Library::new("libwayland-server.so.0") { + if let Ok(set_max) = lib.get::( + b"wl_display_set_default_max_buffer_size\0", + ) { + set_max( + dh.backend_handle().display_ptr() as *mut std::ffi::c_void, + 10 * 1024 * 1024, + ); + } + std::mem::forget(lib); + } + } + + let dri_node = explicit_dri_node; + + let mut use_gpu = !dri_node.is_empty(); + let render_node_path = dri_node.clone(); + + let mut gles_renderer = None; + let mut pixman_renderer = None; + let mut offscreen_buffer: Option<(BufferObject<()>, Dmabuf)> = None; + let mut dmabuf_global = None; + let mut gbm_device_raw = None; + let mut dmabuf_state = DmabufState::new(); + + let mut gpu_success = false; + if use_gpu { + println!("[Wayland] Initializing GL Renderer using device: {}", dri_node); + let init_res: Result<(), String> = (|| { + let device_path = std::path::Path::new(&dri_node); + let file = File::options().read(true).write(true).open(device_path) + .map_err(|e| format!("Failed to open render device: {}", e))?; + let file_for_alloc = file.try_clone() + .map_err(|e| format!("Failed to clone file for GBM Allocator: {}", e))?; + let gbm_allocator = RawGbmDevice::new(file_for_alloc) + .map_err(|_| "Failed to create Raw GBM Device")?; + let gbm = GbmDevice::new(file) + .map_err(|_| "Failed to create GBM device")?; + let egl = unsafe { EGLDisplay::new(gbm) } + .map_err(|_| "Failed to create EGL display")?; + let context = EGLContext::new(&egl) + .map_err(|_| "Failed to create EGL context")?; + let mut renderer = unsafe { GlesRenderer::new(context) } + .map_err(|_| "Failed to init GlesRenderer")?; + + if let Err(e) = renderer.bind_wl_display(&dh) { + println!("[Wayland] Warning: Failed to bind EGL to Wayland Display (Optional): {:?}", e); + } + + let formats = Bind::::supported_formats(&renderer) + .ok_or("Failed to query formats")? + .into_iter() + .collect::>(); + + let node = DrmNode::from_path(device_path) + .map_err(|_| "Failed to create DrmNode")?; + let dmabuf_default_feedback = DmabufFeedbackBuilder::new(node.dev_id(), formats.clone()).build(); + + dmabuf_global = Some(if let Ok(default_feedback) = dmabuf_default_feedback { + dmabuf_state.create_global_with_default_feedback::(&dh, &default_feedback) + } else { + dmabuf_state.create_global::(&dh, formats) + }); + + let bo = gbm_allocator.create_buffer_object( + width as u32, height as u32, GbmFormat::Argb8888, BufferObjectFlags::RENDERING + ).map_err(|_| "Failed to allocate GBM buffer")?; + + let dmabuf = create_dmabuf_from_bo(&bo); + offscreen_buffer = Some((bo, dmabuf)); + gbm_device_raw = Some(gbm_allocator); + gles_renderer = Some(renderer); + Ok(()) + })(); + + match init_res { + Ok(_) => gpu_success = true, + Err(e) => { + println!("[Wayland] GPU Initialization failed: {}. Falling back to Software Renderer (Pixman).", e); + use_gpu = false; + } + } + } + + if !gpu_success { + if !dri_node.is_empty() && !use_gpu { + } else { + println!("[Wayland] No render node. Initializing Software Renderer (Pixman)."); + } + pixman_renderer = Some(PixmanRenderer::new().expect("Failed to init PixmanRenderer")); + use_gpu = false; + } + + let compositor_state = CompositorState::new::(&dh); + let fractional_scale_state = FractionalScaleManagerState::new::(&dh); + let shm_state = ShmState::new::(&dh, vec![]); + let output_state = OutputManagerState::new_with_xdg_output::(&dh); + let mut seat_state = SeatState::new(); + let shell_state = XdgShellState::new::(&dh); + let space = Space::default(); + let layer_shell_state = WlrLayerShellState::new::(&dh); + let data_device_state = DataDeviceState::new::(&dh); + let data_control_state = DataControlState::new::(&dh, None, |_| true); + let _vk_global = dh.create_global::(1, ()); + let pointer_warp_state = PointerWarpManager::new::(&dh); + let relative_pointer_state = RelativePointerManagerState::new::(&dh); + let pointer_constraints_state = PointerConstraintsState::new::(&dh); + + let foreign_toplevel_list = ForeignToplevelListState::new::(&dh); + let xdg_decoration_state = XdgDecorationState::new::(&dh); + let single_pixel_buffer = SinglePixelBufferState::new::(&dh); + let viewporter_state = ViewporterState::new::(&dh); + let presentation_state = PresentationState::new::(&dh, 1); + let xdg_activation_state = XdgActivationState::new::(&dh); + let primary_selection_state = PrimarySelectionState::new::(&dh); + let popups = PopupManager::default(); + + let mut seat = seat_state.new_wl_seat(&dh, "seat0"); + seat.add_keyboard(XkbConfig::default(), 200, 25) + .expect("Failed to init keyboard"); + seat.add_pointer(); + + let mut state = AppState { + compositor_state, + fractional_scale_state, + viewporter_state, + presentation_state, + shm_state, + single_pixel_buffer, + dmabuf_state, + dmabuf_global, + output_state, + seat_state, + shell_state, + layer_shell_state, + space, + data_device_state, + data_control_state, + dh: dh.clone(), + seat, + pointer_warp_state, + relative_pointer_state, + pointer_constraints_state, + output_nodes: Vec::new(), + pending_windows: Vec::new(), + foreign_toplevel_list, + xdg_decoration_state, + xdg_activation_state, + primary_selection_state, + popups, + gles_renderer, + pixman_renderer, + gbm_device: gbm_device_raw, + settings: RustCaptureSettings { + width, + height, + ..RustCaptureSettings::default() + }, + cursor_callback_set: false, + cursor_tx: wayland::cursor::spawn_cursor_worker( + cursor_size, + RustCaptureSettings::default().cursor_size_cap, + ), + clipboard_callback: None, + pending_clipboard_read: None, + current_selection_mime: None, + last_log_time: Instant::now(), + start_time: Instant::now(), + clock: Clock::new(), + use_gpu, + cursor_helper: Cursor::load(cursor_size), + keymap_policy: wayland::keymap::KeymapPolicy::empty(), + host: None, + current_cursor_icon: None, + cursor_buffer: None, + render_cursor_on_framebuffer: false, + render_node_path, + auto_gpu_selected, + pending_screenshot: None, + command_rx: None, + }; + // Seed the keymap policy with the seat's initial keymap so overlay binds splice onto + // the exact text clients received. + { + let initial_keymap = if let Some(kb) = state.seat.get_keyboard() { + kb.with_xkb_state(&mut state, |context| match context.xkb().lock() { + Ok(guard) => { + let keymap = unsafe { guard.keymap() }; + keymap.get_as_string(smithay::input::keyboard::xkb::KEYMAP_FORMAT_TEXT_V1) + } + Err(_) => String::new(), + }) + } else { + String::new() + }; + state.keymap_policy.rebuild_base(initial_keymap); + } - if state.recording_sink.is_some() && settings.output_mode == 0 { - eprintln!( - "[recording_sink] WARNING: recording_socket is set but output_mode is JPEG (0). \ - The recording socket requires a single H.264 stream. Please set output_mode=1 \ - on the Python CaptureSettings to produce a recordable output." - ); - } + let output = Output::new( + "HEADLESS-1".into(), + PhysicalProperties { + size: (width, height).into(), + subpixel: Subpixel::Unknown, + make: "Pixelflux".into(), + model: "Virtual".into(), + serial_number: "001".into(), + }, + ); + output.change_current_state( + Some(OutputMode { + size: (width, height).into(), + refresh: 60_000, + }), + Some(Transform::Normal), + Some(OutputScale::Fractional(1.0)), + Some((0, 0).into()), + ); + output.set_preferred(OutputMode { + size: (width, height).into(), + refresh: 60_000, + }); + state.space.map_output(&output, (0, 0)); + let global = output.create_global::(&dh); + let damage_tracker = OutputDamageTracker::from_output(&output); + state.output_nodes.push(wayland::frontend::OutputNode { + id: 0, + output, + global, + pos: (0, 0), + damage_tracker, + frame_buffer: vec![0u8; (width.max(0) as usize) * (height.max(0) as usize) * 4], + offscreen_buffer, + overlay_state: OverlayState::default(), + capture: None, + }); - let watermark_output_scale = state - .outputs - .first() - .map(|o| o.current_scale().fractional_scale()) - .unwrap_or(1.0); - state - .overlay_state - .load_watermark(&settings.watermark_path, watermark_output_scale); - state.callback = Some(cb); - state.is_capturing = true; - WAYLAND_CAPTURE_ALIVE.store(true, Ordering::Release); - state.render_cursor_on_framebuffer = settings.capture_cursor; - state.settings = settings.clone(); - state.encode_stats.frames.store(0, Ordering::Relaxed); - state.encode_stats.stripes.store(0, Ordering::Relaxed); - state.last_log_time = Instant::now(); - state.frame_counter = 0; - state.pending_force_idr = false; - state.vaapi_state = StripeState::default(); - if state.cursor_callback.is_some() { - if let Some(icon) = state.current_cursor_icon.clone() { - state.send_cursor_image(&icon); - } - } - if let Some(p) = state.encode_pool.take() { p.shutdown(); } - let prior_readback_encoder = state - .encode_join - .take() - .and_then(|j| j.join().ok()) - .flatten(); - if let Some(tx) = state.deliver_tx.take() { drop(tx); } - if let Some(j) = state.deliver_join.take() { let _ = j.join(); } - state.pending_hw_delivery = None; - state.pending_hw_damage = false; - if let Some(cb) = state.callback.take() { - let (tx, rx) = std::sync::mpsc::sync_channel::>(1); - let join = thread::spawn(move || { - crate::boost_thread_priority(-10); - while let Ok(stripes) = rx.recv() { - if PY_SHUTDOWN.load(Ordering::Relaxed) { continue; } - Python::attach(|py| { - for s in stripes { - match Py::new(py, StripeFrame::new_owned_meta( - s.data, s.data_type, s.stripe_y_start, - s.stripe_height, s.frame_id, - )) { - Ok(f) => { if let Err(e) = cb.call1(py, (f,)) { e.print(py); } } - Err(e) => eprintln!("[wayland] frame alloc error: {e:?}"), - } - } - }); - } - }); - state.deliver_tx = Some(tx); - state.deliver_join = Some(join); - } + /// Apply every queued control command in FIFO order. Sends wake the loop through the + /// separate wake channel, and the render tick ALSO drains before starting its work, so + /// queued input is applied ahead of a long render/encode instead of waiting it out. + fn drain_thread_commands(state: &mut AppState) { + let Some(rx) = state.command_rx.take() else { return }; + while let Ok(cmd) = rx.try_recv() { + handle_thread_command(state, cmd); + } + state.command_rx = Some(rx); + } - if state.video_encoder.is_none() { - if let Some(ref deliver_tx) = state.deliver_tx { - let pool = Arc::new(WlFramePool::new( - WL_POOL_SURFACES, - (settings.width.max(0) as usize) * (settings.height.max(0) as usize) * 4, - )); - state.pool_last_render = vec![0; WL_POOL_SURFACES]; - state.render_seq = 0; - // u64::MAX marks every slot stale so each one is read back - // before its first publish, whatever the damage says. - state.pool_content_gen = vec![u64::MAX; WL_POOL_SURFACES]; - state.content_gen = 0; - let c = &state.encode_controls; - c.bitrate_kbps.store(settings.video_bitrate_kbps, Ordering::Relaxed); - c.vbv_mult_milli.store( - (settings.video_vbv_multiplier * 1000.0).round() as i32, - Ordering::Relaxed, - ); - c.fps_milli.store( - (settings.target_fps.max(1.0) * 1000.0) as u64, - Ordering::Relaxed, - ); - c.rate_dirty.store(false, Ordering::Relaxed); - c.force_idr.store(false, Ordering::Relaxed); - // Also drop any leftover LiveTunables snapshot: UpdateTunables - // sets it even when no encode thread is consuming (zero-copy - // mode, or after StopCapture), and without this reset the new - // encode thread's first frame would apply that stale snapshot - // OVER the fresh StartCapture settings just established. - c.tunables_dirty.store(false, Ordering::Relaxed); - *c.tunables.lock().unwrap() = None; - let cfg = WlEncodeConfig { - settings: settings.clone(), - use_gpu: state.use_gpu, - try_gpu: gpu_intent && (!state.use_gpu || different_gpu), - prior: prior_readback_encoder, - recording_sink: state.recording_sink.clone(), - deliver_tx: deliver_tx.clone(), - controls: state.encode_controls.clone(), - stats: state.encode_stats.clone(), - }; - let pool2 = pool.clone(); - state.encode_join = Some( - thread::Builder::new() - .name("wl-encode".into()) - .spawn(move || wayland_encode_loop(&pool2, cfg)) - .expect("failed to spawn wl-encode thread"), - ); - state.encode_pool = Some(pool); - } - } else { - state.encode_stats.n_stripes.store(1, Ordering::Relaxed); - *state.encode_stats.desc.lock().unwrap() = - encoder_desc(&settings, state.video_encoder.as_ref(), false, true); - log_stream_settings(&settings, 1, state.video_encoder.as_ref(), false); - } - // Force the keyframe on the now-active path (as RequestIdr does), - // unconditionally: the damage tracker and offscreen buffer stay - // warm across StopCapture/StartCapture, so a restarted capture on - // a static screen otherwise produces no damage, no first frame, - // and no IDR in either the striped or the zero-copy path (a - // stop+start encoder switch froze until the next screen damage). - if state.encode_pool.is_some() { - state.encode_controls.force_idr.store(true, Ordering::Relaxed); - } else { - state.pending_force_idr = true; - } - state.needs_full_render = true; + fn handle_thread_command(state: &mut AppState, cmd: ThreadCommand) { + match cmd { + ThreadCommand::StartCapture { display_id, callback, settings } => { + start_capture_on_display(state, display_id, callback, settings); + } + ThreadCommand::StopCapture { display_id } => { + // Cursor and clipboard callbacks deliberately SURVIVE StopCapture: + // captures cycle on client disconnects and setting restarts, and a + // copy or cursor change during that gap must still reach Python. + // PY_SHUTDOWN gates every use against a finalizing interpreter. + stop_capture_on_display(state, display_id); + } + ThreadCommand::CreateOutput { id, width, height, x, y, scale, reply } => { + let _ = reply.send(create_output_on(state, id, width, height, x, y, scale)); + } + ThreadCommand::DestroyOutput { id, reply } => { + let _ = reply.send(destroy_output_on(state, id)); + } + ThreadCommand::RepositionOutput { id, x, y, reply } => { + let _ = reply.send(reposition_output_on(state, id, x, y)); } - CalloopEvent::Msg(ThreadCommand::StopCapture) => { - println!("[Wayland] Capture loop stopped."); - state.is_capturing = false; - WAYLAND_CAPTURE_ALIVE.store(false, Ordering::Release); - state.callback = None; - state.cursor_callback = None; - state.clipboard_callback = None; - state.video_encoder = None; - state.vaapi_state = StripeState::default(); - if let Some(p) = state.encode_pool.take() { p.shutdown(); } - if let Some(j) = state.encode_join.take() { let _ = j.join(); } - state.recording_sink = None; - *state.encode_stats.desc.lock().unwrap() = String::new(); - if let Some(tx) = state.deliver_tx.take() { drop(tx); } - if let Some(j) = state.deliver_join.take() { let _ = j.join(); } - state.pending_hw_delivery = None; - state.pending_hw_damage = false; + ThreadCommand::ListOutputs { reply } => { + let list = state + .output_nodes + .iter() + .map(|n| { + let (w, h) = n + .output + .current_mode() + .map(|m| (m.size.w, m.size.h)) + .unwrap_or((0, 0)); + ( + n.id, + n.pos.0, + n.pos.1, + w, + h, + n.output.current_scale().fractional_scale(), + n.capture.is_some(), + ) + }) + .collect(); + let _ = reply.send(list); + } + ThreadCommand::MoveWindowToOutput { window_id, output_id, reply } => { + let window = state + .space + .elements() + .find(|w| { + wayland::frontend::window_meta(w) + .map(|m| m.id == window_id) + .unwrap_or(false) + }) + .cloned(); + let ok = match window { + Some(w) => state.place_window_on_output(&w, output_id), + None => false, + }; + let _ = reply.send(ok); } - CalloopEvent::Msg(ThreadCommand::SetClipboardCallback(cb)) => { + ThreadCommand::ListWindows { reply } => { + use smithay::wayland::shell::xdg::XdgToplevelSurfaceData; + let mut list = Vec::new(); + for window in state.space.elements() { + let Some(meta) = wayland::frontend::window_meta(window) else { continue }; + let (title, app_id) = window + .toplevel() + .map(|tl| { + with_states(tl.wl_surface(), |states| { + states + .data_map + .get::() + .map(|d| { + let a = d.lock().unwrap(); + ( + a.title.clone().unwrap_or_default(), + a.app_id.clone().unwrap_or_default(), + ) + }) + .unwrap_or_default() + }) + }) + .unwrap_or_default(); + list.push(( + meta.id, + title, + app_id, + meta.output.load(Ordering::Relaxed), + meta.parked.load(Ordering::Relaxed), + )); + } + let _ = reply.send(list); + } + ThreadCommand::SetClipboardCallback(cb) => { state.clipboard_callback = Some(cb); + // Re-stage a read of the CURRENT selection so a copy made before this + // callback was (re)armed is delivered rather than lost; the post-dispatch + // drain performs the read (a compositor-owned selection is skipped there). + if let Some(mime) = state.current_selection_mime.clone() { + state.pending_clipboard_read = Some(mime); + } } - CalloopEvent::Msg(ThreadCommand::SetClipboard { mime, data }) => { + ThreadCommand::SetClipboard { mime, data } => { let mimes: Vec = if mime.starts_with("text/plain") { ["text/plain;charset=utf-8", "UTF8_STRING", "text/plain", "STRING", "TEXT"].iter().map(|s| s.to_string()).collect() } else { vec![mime.clone()] }; + let payload = std::sync::Arc::new((mime, data)); smithay::wayland::selection::data_device::set_data_device_selection( + &state.dh, + &state.seat.clone(), + mimes.clone(), + payload.clone(), + ); + // Middle-click parity with the X11 clipboard bridge: the same + // offer backs the primary selection too. + smithay::wayland::selection::primary_selection::set_primary_selection( &state.dh, &state.seat.clone(), mimes, - std::sync::Arc::new((mime, data)), + payload, ); + // The selection is compositor-owned now; a later SetClipboardCallback + // must not try to re-read a client source that no longer holds it. + state.current_selection_mime = None; } - CalloopEvent::Msg(ThreadCommand::SetCursorCallback(cb)) => { - state.cursor_callback = Some(cb); + ThreadCommand::SetCursorCallback(cb) => { + let _ = state.cursor_tx.send(CursorJob::SetCallback(cb)); + state.cursor_callback_set = true; if let Some(icon) = state.current_cursor_icon.clone() { state.send_cursor_image(&icon); + } else { + // No client has set a cursor yet. The render path treats + // None as the default theme cursor, so the first consumer + // must receive that same sprite instead of a blank pointer + // until the first client cursor event. + state.send_cursor_image(&CursorImageStatus::Named(Default::default())); + } + } + ThreadCommand::KeyboardKeys { events } => { + for (scancode, key_state_val) in events { + if let Some(host) = state.host.as_ref() { + host.key(scancode, key_state_val > 0); + continue; + } + let key_state = if key_state_val > 0 { + KeyState::Pressed + } else { + KeyState::Released + }; + let serial = next_serial(); + let time = wayland_time(); + if let Some(keyboard) = state.seat.get_keyboard() { + keyboard.input( + state, + Keycode::new(scancode), + key_state, + serial, + time, + |_, _, _| FilterResult::<()>::Forward, + ); + } } } - CalloopEvent::Msg(ThreadCommand::KeyboardKey { scancode, state: key_state_val }) => { + ThreadCommand::KeyboardKey { scancode, state: key_state_val } => { + if let Some(host) = state.host.as_ref() { + host.key(scancode, key_state_val > 0); + return; + } let key_state = if key_state_val > 0 { KeyState::Pressed } else { KeyState::Released }; let serial = next_serial(); let time = wayland_time(); @@ -2093,14 +3890,64 @@ fn run_wayland_thread( }); } } - CalloopEvent::Msg(ThreadCommand::SetKeymapString(text)) => { - if let Some(keyboard) = state.seat.get_keyboard() { - if let Err(e) = keyboard.set_keymap_from_string(state, text) { - eprintln!("[Wayland] keymap swap failed: {e:?}"); + ThreadCommand::SetKeymapString(text) => { + // rebuild_base rejects a string that will not compile without touching + // the policy, so the seat keymap survives a bad one either way. + if state.keymap_policy.rebuild_base(text) { + state.apply_keymap_policy(); + } else { + eprintln!("[Wayland] set_keymap_string: keymap failed to compile; keeping current keymap."); + } + } + ThreadCommand::SetXkbLayout { rules, model, layout, variant, options, reply } => { + match crate::wayland::keymap::compile_rmlvo(&rules, &model, &layout, &variant, &options) { + Some(text) => { + state.keymap_policy.rebuild_base(text); + state.apply_keymap_policy(); + let _ = reply.send(true); } + None => { + eprintln!("[Wayland] set_xkb_layout: RMLVO ({rules:?}, {model:?}, {layout:?}, {variant:?}, {options:?}) failed to compile."); + let _ = reply.send(false); + } + } + } + ThreadCommand::BindKeysyms { keysyms, reply } => { + let _ = reply.send(state.bind_keysyms(&keysyms)); + } + ThreadCommand::SetKeymapOverlay { binds } => { + match state.keymap_policy.overlay_text_for(&binds) { + Some(text) => state.apply_keymap_text(text), + None => eprintln!( + "[Wayland] set_keymap_overlay: no base keymap to splice onto." + ), } } - CalloopEvent::Msg(ThreadCommand::GetXkbKeymap { reply }) => { + ThreadCommand::GetKeyboardState { reply } => { + let (pressed, mods) = state + .seat + .get_keyboard() + .map(|kb| { + let pressed: Vec = + kb.pressed_keys().iter().map(|c| c.raw()).collect(); + let m = kb.modifier_state(); + let mask = (m.ctrl as u32) + | (m.shift as u32) << 1 + | (m.alt as u32) << 2 + | (m.logo as u32) << 3 + | (m.caps_lock as u32) << 4 + | (m.num_lock as u32) << 5 + | (m.iso_level3_shift as u32) << 6 + | (m.iso_level5_shift as u32) << 7; + (pressed, mask) + }) + .unwrap_or_default(); + let _ = reply.send((pressed, mods)); + } + ThreadCommand::Barrier { reply } => { + let _ = reply.send(()); + } + ThreadCommand::GetXkbKeymap { reply } => { let mut keymap_str = String::new(); if let Some(keyboard) = state.seat.get_keyboard() { keymap_str = keyboard.with_xkb_state(state, |context| { @@ -2117,35 +3964,48 @@ fn run_wayland_thread( } let _ = reply.send(keymap_str); } - CalloopEvent::Msg(ThreadCommand::PointerMotion { x, y }) => { + ThreadCommand::PointerMotion { x, y } => { + if let Some(host) = state.host.as_ref() { + host.pointer_motion_abs(x, y); + return; + } let serial = next_serial(); let time = wayland_time(); - let scale = state.settings.scale; - let logical_w = (state.settings.width as f64 / scale).floor(); - let logical_h = (state.settings.height as f64 / scale).floor(); - let logical_x = (x / scale).max(0.0).min(logical_w - 1.0); - let logical_y = (y / scale).max(0.0).min(logical_h - 1.0); + // (x, y) are physical union-layout coordinates: each output occupies + // the physical rectangle at its layout offset, and the point maps + // through the CONTAINING output's scale (clamped into the nearest + // output when outside all of them). + let p = state.layout_physical_to_logical(x, y); if let Some(pointer) = state.seat.get_pointer() { - let p = Point::::from((logical_x, logical_y)); - let mut under = None; - - if let Some(output) = state.outputs.first() { - let layer_map = layer_map_for_output(output); - let pointer_pos = p.to_i32_round(); - + // Layer surfaces live on the output under the point; their + // geometry is output-local, so hit-test with the local point and + // report the global location. + let layer_hit = |state: &AppState, layers: &[smithay::wayland::shell::wlr_layer::Layer]| { + let idx = state.node_idx_under(p)?; + let node = &state.output_nodes[idx]; + let origin = Point::::from(node.pos); + let local = (p - origin.to_f64()).to_i32_round(); + let layer_map = layer_map_for_output(&node.output); for layer in layer_map.layers().rev() { - let state_layer = layer.layer(); - if state_layer == smithay::wayland::shell::wlr_layer::Layer::Overlay || state_layer == smithay::wayland::shell::wlr_layer::Layer::Top { + if layers.contains(&layer.layer()) { if let Some(bbox) = layer_map.layer_geometry(layer) { - if bbox.contains(pointer_pos) { - under = Some((FocusTarget::LayerSurface(layer.clone()), bbox.loc.to_f64())); - break; + if bbox.contains(local) { + return Some(( + FocusTarget::LayerSurface(layer.clone()), + (bbox.loc + origin).to_f64(), + )); } } } } - } + None + }; + + let mut under = layer_hit(state, &[ + smithay::wayland::shell::wlr_layer::Layer::Overlay, + smithay::wayland::shell::wlr_layer::Layer::Top, + ]); if under.is_none() { under = state.space.element_under(p).map(|(window, loc)| { @@ -2154,44 +4014,26 @@ fn run_wayland_thread( } if under.is_none() { - if let Some(output) = state.outputs.first() { - let layer_map = layer_map_for_output(output); - let pointer_pos = p.to_i32_round(); - - for layer in layer_map.layers().rev() { - let state_layer = layer.layer(); - if state_layer == smithay::wayland::shell::wlr_layer::Layer::Bottom || state_layer == smithay::wayland::shell::wlr_layer::Layer::Background { - if let Some(bbox) = layer_map.layer_geometry(layer) { - if bbox.contains(pointer_pos) { - under = Some((FocusTarget::LayerSurface(layer.clone()), bbox.loc.to_f64())); - break; - } - } - } - } - } + under = layer_hit(state, &[ + smithay::wayland::shell::wlr_layer::Layer::Bottom, + smithay::wayland::shell::wlr_layer::Layer::Background, + ]); } pointer.motion(state, under, &MotionEvent { location: p, serial, time }); pointer.frame(state); } } - CalloopEvent::Msg(ThreadCommand::PointerRelativeMotion { dx, dy }) => { + ThreadCommand::PointerRelativeMotion { dx, dy } => { let utime = wayland_utime(); let time = wayland_time(); let serial = next_serial(); if let Some(pointer) = state.seat.get_pointer() { let current_pos = pointer.current_location(); - - let scale = state.settings.scale; - let max_w = (state.settings.width as f64 / scale).floor() - 1.0; - let max_h = (state.settings.height as f64 / scale).floor() - 1.0; - - let new_x = (current_pos.x + dx).max(0.0).min(max_w); - let new_y = (current_pos.y + dy).max(0.0).min(max_h); - - let new_pos = Point::::from((new_x, new_y)); + let new_pos = state.clamp_logical( + (current_pos.x + dx, current_pos.y + dy).into(), + ); let under = state.space.element_under(new_pos).map(|(window, loc)| { (FocusTarget::Window(window.clone()), loc.to_f64()) @@ -2217,7 +4059,11 @@ fn run_wayland_thread( pointer.frame(state); } } - CalloopEvent::Msg(ThreadCommand::PointerButton { btn, state: btn_state_val }) => { + ThreadCommand::PointerButton { btn, state: btn_state_val } => { + if let Some(host) = state.host.as_ref() { + host.pointer_button(btn, btn_state_val > 0); + return; + } let serial = next_serial(); let time = wayland_time(); let button_state = if btn_state_val > 0 { smithay::backend::input::ButtonState::Pressed } else { smithay::backend::input::ButtonState::Released }; @@ -2239,7 +4085,11 @@ fn run_wayland_thread( pointer.frame(state); } } - CalloopEvent::Msg(ThreadCommand::PointerAxis { x, y }) => { + ThreadCommand::PointerAxis { x, y } => { + if let Some(host) = state.host.as_ref() { + host.pointer_axis(x, y); + return; + } let time = wayland_time(); if let Some(pointer) = state.seat.get_pointer() { @@ -2265,73 +4115,135 @@ fn run_wayland_thread( } } } - CalloopEvent::Msg(ThreadCommand::UpdateCursorConfig { render_on_framebuffer }) => { + ThreadCommand::UpdateCursorConfig { render_on_framebuffer } => { state.render_cursor_on_framebuffer = render_on_framebuffer; } - CalloopEvent::Msg(ThreadCommand::RequestIdr) => { - if state.encode_pool.is_some() { - state.encode_controls.force_idr.store(true, Ordering::Relaxed); + ThreadCommand::SetCursorSize { size, reply } => { + if size <= 0 { + let _ = reply.send(false); } else { - state.pending_force_idr = true; + state.cursor_helper = Cursor::load(size); + let _ = state.cursor_tx.send(CursorJob::SetSize(size)); + // The burned-in cursor changed size; force a repaint everywhere so + // a static screen doesn't keep showing the old sprite. + for node in state.output_nodes.iter_mut() { + if let Some(cap) = node.capture.as_mut() { + cap.needs_full_render = true; + } + } + let _ = reply.send(true); } - state.needs_full_render = true; } - CalloopEvent::Msg(ThreadCommand::UpdateRate { bitrate_kbps, vbv_multiplier, fps }) => { - if let Some(b) = bitrate_kbps { state.settings.video_bitrate_kbps = b; } - if let Some(v) = vbv_multiplier { state.settings.video_vbv_multiplier = v; } - if let Some(f) = fps { if f > 0.0 { state.settings.target_fps = f; } } - if let Some(GpuEncoder::Nvenc(enc)) = state.video_encoder.as_mut() { - enc.reconfigure_rate(&state.settings); + ThreadCommand::RequestIdr { display_id } => { + if let Some(idx) = state.node_idx_for_id(display_id) { + if let Some(cap) = state.output_nodes[idx].capture.as_mut() { + cap.request_idr(); + } } - if let Some(GpuEncoder::Vaapi(enc)) = state.video_encoder.as_mut() { - enc.reconfigure_rate(&state.settings); + } + ThreadCommand::UpdateRate { display_id, bitrate_kbps, vbv_multiplier, fps } => { + if let Some(idx) = state.node_idx_for_id(display_id) { + if let Some(cap) = state.output_nodes[idx].capture.as_mut() { + if let Some(b) = bitrate_kbps { cap.settings.video_bitrate_kbps = b; } + if let Some(v) = vbv_multiplier { cap.settings.video_vbv_multiplier = v; } + if let Some(f) = fps { if f > 0.0 { cap.settings.target_fps = f; } } + if let Some(GpuEncoder::Nvenc(enc)) = cap.video_encoder.as_mut() { + enc.reconfigure_rate(&cap.settings); + } + if let Some(GpuEncoder::Vaapi(enc)) = cap.video_encoder.as_mut() { + enc.reconfigure_rate(&cap.settings); + } + let c = &cap.encode_controls; + c.bitrate_kbps.store(cap.settings.video_bitrate_kbps, Ordering::Relaxed); + c.vbv_mult_milli.store( + (cap.settings.video_vbv_multiplier * 1000.0).round() as i32, + Ordering::Relaxed, + ); + c.fps_milli.store( + (cap.settings.target_fps.max(1.0) * 1000.0) as u64, + Ordering::Relaxed, + ); + c.rate_dirty.store(true, Ordering::Release); + if display_id == 0 { + state.settings.video_bitrate_kbps = cap.settings.video_bitrate_kbps; + state.settings.video_vbv_multiplier = cap.settings.video_vbv_multiplier; + state.settings.target_fps = cap.settings.target_fps; + } + } } - let c = &state.encode_controls; - c.bitrate_kbps.store(state.settings.video_bitrate_kbps, Ordering::Relaxed); - c.vbv_mult_milli.store( - (state.settings.video_vbv_multiplier * 1000.0).round() as i32, - Ordering::Relaxed, - ); - c.fps_milli.store( - (state.settings.target_fps.max(1.0) * 1000.0) as u64, - Ordering::Relaxed, - ); - c.rate_dirty.store(true, Ordering::Release); } - CalloopEvent::Msg(ThreadCommand::UpdateTunables(t)) => { - t.apply_to(&mut state.settings); + ThreadCommand::UpdateTunables { display_id, tunables: t } => { state.render_cursor_on_framebuffer = t.capture_cursor; - *state.encode_controls.tunables.lock().unwrap() = Some(t); - state.encode_controls.tunables_dirty.store(true, Ordering::Release); + let _ = state.cursor_tx.send(CursorJob::SetSizeCap(t.cursor_size_cap)); + if display_id == 0 { + t.apply_to(&mut state.settings); + } + if let Some(idx) = state.node_idx_for_id(display_id) { + if let Some(cap) = state.output_nodes[idx].capture.as_mut() { + t.apply_to(&mut cap.settings); + *cap.encode_controls.tunables.lock().unwrap() = Some(t); + cap.encode_controls.tunables_dirty.store(true, Ordering::Release); + } + } } - CalloopEvent::Msg(ThreadCommand::CuScreenshot { resp }) => { - state.pending_screenshot = Some(resp); + ThreadCommand::CuScreenshot { display_id, resp } => { + if state.node_idx_for_id(display_id).is_some() { + state.pending_screenshot = Some((display_id, resp)); + } else { + let _ = resp.send(Err(format!("Unknown display: {display_id}"))); + } } - CalloopEvent::Msg(ThreadCommand::CuCursorPosition { resp }) => { + ThreadCommand::CuCursorPosition { resp } => { let pos = state.seat.get_pointer() .map(|p| p.current_location()) .unwrap_or_else(|| (0.0f64, 0.0f64).into()); - let scale = state.settings.scale; - let fb_x = pos.x * scale; - let fb_y = pos.y * scale; - let _ = resp.send((fb_x, fb_y)); + let _ = resp.send(state.layout_logical_to_physical(pos)); } - CalloopEvent::Msg(ThreadCommand::CuGetInfo { resp }) => { - let _ = resp.send(( - state.settings.width, - state.settings.height, - state.settings.scale, - )); + ThreadCommand::CuGetInfo { display_id, resp } => { + let info = state + .node_idx_for_id(display_id) + .map(|idx| { + let node = &state.output_nodes[idx]; + match node.capture.as_ref() { + Some(c) => (c.settings.width, c.settings.height, c.settings.scale), + None => node + .output + .current_mode() + .map(|m| { + ( + m.size.w, + m.size.h, + node.output.current_scale().fractional_scale(), + ) + }) + .unwrap_or((0, 0, 0.0)), + } + }) + .unwrap_or((0, 0, 0.0)); + let _ = resp.send(info); } - CalloopEvent::Closed => {} } + } + + state.command_rx = Some(command_rx); + event_loop + .handle() + .insert_source(wake_rx, |_, _, state| { + drain_thread_commands(state); }) .unwrap(); - let source = ListeningSocketSource::new_auto().unwrap(); + let source = match ListeningSocketSource::new_auto() { + Ok(s) => s, + Err(e) => { + eprintln!("[Wayland] compositor thread aborting: could not bind a wayland-N socket (XDG_RUNTIME_DIR unset/full?): {e}"); + return; + } + }; let socket_name = source.socket_name().to_string_lossy().into_owned(); println!("[Wayland] Socket listening on: {:?}", socket_name); std::env::set_var("WAYLAND_DISPLAY", &socket_name); + publish_socket_name(&socket_name); event_loop .handle() @@ -2349,572 +4261,111 @@ fn run_wayland_thread( let mut is_memory_throttling = false; event_loop .handle() - .insert_source(timer, move |_, _, state| { - let loop_start_time = Instant::now(); - state.space.refresh(); - - let current_rss = get_process_rss_bytes(); - let shm_usage = get_shm_usage_bytes(); - let memory_threshold = calculate_memory_threshold(state.settings.width, state.settings.height); - - if current_rss > memory_threshold || shm_usage > (4 * 1024 * 1024 * 1024) { - if !is_memory_throttling { - is_memory_throttling = true; - } - } else if is_memory_throttling - && current_rss < (memory_threshold as f64 * 0.75) as usize && shm_usage < (3 * 1024 * 1024 * 1024) { - is_memory_throttling = false; - } - - let now = Instant::now(); - let elapsed = now.duration_since(state.last_log_time).as_secs_f64(); - if elapsed >= 1.0 { - let frames = state.encode_stats.frames.swap(0, Ordering::Relaxed); - let stripes = state.encode_stats.stripes.swap(0, Ordering::Relaxed); - if state.is_capturing && state.settings.debug_logging { - let actual_fps = frames as f64 / elapsed; - let stripes_per_sec = stripes as f64 / elapsed; - let mode_str = state.encode_stats.desc.lock().unwrap().clone(); - let n_stripes = state.encode_stats.n_stripes.load(Ordering::Relaxed); - - let rss_mb = current_rss / 1024 / 1024; - let shm_mb = shm_usage / 1024 / 1024; - let throttle_warn = if is_memory_throttling { " [THROTTLED]" } else { "" }; - - println!("Res: {}x{} Mode: {} Stripes: {} EncFPS: {:.2} EncStripes/s: {:.2} Mem: {}MB SHM: {}MB{}", - state.settings.width, state.settings.height, mode_str, n_stripes, actual_fps, stripes_per_sec, rss_mb, shm_mb, throttle_warn); - } - state.last_log_time = now; - } - - if !state.is_capturing && state.pending_screenshot.is_none() { - return TimeoutAction::ToDuration(Duration::from_millis(16)); - } - - let requested_idr = state.pending_force_idr; - - let mut pool_slot: Option<(usize, Vec)> = None; - if let Some(ref pool) = state.encode_pool { - if !is_memory_throttling { - pool_slot = pool.try_begin(); - if pool_slot.is_none() { - return TimeoutAction::ToDuration(Duration::from_millis(1)); - } - } - } - - state.overlay_state.update_position( - state.settings.width, - state.settings.height, - state.settings.watermark_location_enum, - ); - - let mut render_success = false; - let mut render_sync = None; - let mut damage_rects: Vec> = Vec::new(); - let width = state.settings.width; - let height = state.settings.height; - - if let Some(output) = state.outputs.first().cloned() { - let render_age = if state.overlay_state.is_animated() || state.needs_full_render { 0 } else { 1 }; - if state.use_gpu { - if let Some(renderer) = state.gles_renderer.as_mut() { - if let Some((_bo, dmabuf)) = state.offscreen_buffer.as_mut() { - match renderer.bind(dmabuf) { - Ok(mut frame) => { - let mut elements: Vec>> = Vec::new(); - - if state.render_cursor_on_framebuffer { - if let Some(pointer) = state.seat.get_pointer() { - let pos = pointer.current_location(); - let output_scale_val = output.current_scale().fractional_scale(); - let scale = Scale::from(output_scale_val); - - if let Some(CursorImageStatus::Named(icon)) = &state.current_cursor_icon { - let name = wayland::frontend::cursor_icon_to_str(icon); - let time = Duration::from_millis(state.clock.now().as_millis() as u64); - if let Some(image) = state.cursor_helper.get_image_by_name(name, output_scale_val.round() as u32, time) { - if let Some(elem) = state.overlay_state.get_cursor_element(renderer, image, pos.to_i32_round()) { - elements.push(CompositionElements::Cursor(elem)); - } - } - } else if let Some(CursorImageStatus::Surface(surface)) = &state.current_cursor_icon { - let phys_pos = pos.to_physical(scale); - let elem_result = with_states(surface, |states| { - WaylandSurfaceRenderElement::from_surface(renderer, surface, states, phys_pos, 1.0, smithay::backend::renderer::element::Kind::Cursor) - }); - if let Ok(Some(cursor_elem)) = elem_result { - elements.push(CompositionElements::Surface(cursor_elem)); - } - } else if state.current_cursor_icon.is_none() { - let time = Duration::from_millis(state.clock.now().as_millis() as u64); - let image = state.cursor_helper.get_image(output_scale_val.round() as u32, time); - if let Some(elem) = state.overlay_state.get_cursor_element(renderer, image, pos.to_i32_round()) { - elements.push(CompositionElements::Cursor(elem)); - } - } - } - } - - if let Some(elem) = state.overlay_state.get_watermark_element(renderer) { - elements.push(CompositionElements::Cursor(elem)); - } - - { - let layer_map = layer_map_for_output(&output); - - let draw_layer = |renderer: &mut GlesRenderer, elements: &mut Vec>>, target_layer: smithay::wayland::shell::wlr_layer::Layer| { - for surface in layer_map.layers().rev() { - let current_layer = surface.layer(); - if current_layer == target_layer { - if let Some(geo) = layer_map.layer_geometry(surface) { - let elem = smithay::wayland::compositor::with_states(surface.wl_surface(), |states| { - WaylandSurfaceRenderElement::from_surface( - renderer, surface.wl_surface(), states, - geo.loc.to_physical_precise_round(1), 1.0, - smithay::backend::renderer::element::Kind::Unspecified - ) - }); - if let Ok(Some(e)) = elem { - elements.push(CompositionElements::Surface(e)); - } - } - } - } - }; - - draw_layer(renderer, &mut elements, smithay::wayland::shell::wlr_layer::Layer::Overlay); - draw_layer(renderer, &mut elements, smithay::wayland::shell::wlr_layer::Layer::Top); - } - - for window in state.space.elements_for_output(&output).collect::>().into_iter().rev() { - let window_loc = state.space.element_location(window).unwrap_or_default(); - - if let Some(surface) = window.wl_surface() { - let popups = PopupManager::popups_for_surface(&surface); - for (popup, location) in popups { - let popup_surface = popup.wl_surface(); - let popup_pos = window_loc + location; - let elem = smithay::wayland::compositor::with_states(popup_surface, |states| { - WaylandSurfaceRenderElement::from_surface( - renderer, - popup_surface, - states, - popup_pos.to_physical_precise_round(1), - 1.0, - smithay::backend::renderer::element::Kind::Unspecified - ) - }); - if let Ok(Some(e)) = elem { - elements.push(CompositionElements::Surface(e)); - } - } - } - - elements.extend(window.render_elements(renderer, window_loc.to_physical_precise_round(1), Scale::from(1.0), 1.0).into_iter().map(CompositionElements::Space)); - } - - { - let layer_map = layer_map_for_output(&output); - - let draw_layer = |renderer: &mut GlesRenderer, elements: &mut Vec>>, target_layer: smithay::wayland::shell::wlr_layer::Layer| { - for surface in layer_map.layers() { - let current_layer = surface.layer(); - if current_layer == target_layer { - if let Some(geo) = layer_map.layer_geometry(surface) { - let elem = smithay::wayland::compositor::with_states(surface.wl_surface(), |states| { - WaylandSurfaceRenderElement::from_surface( - renderer, surface.wl_surface(), states, - geo.loc.to_physical_precise_round(1), 1.0, - smithay::backend::renderer::element::Kind::Unspecified - ) - }); - if let Ok(Some(e)) = elem { - elements.push(CompositionElements::Surface(e)); - } - } - } - } - }; - - draw_layer(renderer, &mut elements, smithay::wayland::shell::wlr_layer::Layer::Bottom); - draw_layer(renderer, &mut elements, smithay::wayland::shell::wlr_layer::Layer::Background); - } - match damage_tracker.render_output(renderer, &mut frame, render_age, &elements, [0.1, 0.1, 0.1, 1.0]) { - Ok(result) => { - render_success = true; - if let Some(damage) = result.damage { - damage_rects = damage.clone(); - } - render_sync = Some(result.sync); - state.needs_full_render = false; - }, - Err(e) => eprintln!("Render error: {:?}", e) - } - if !damage_rects.is_empty() { - state.content_gen += 1; - } - if let Some((id, ref mut buf)) = pool_slot { - // No-damage ticks skip the readback, so a pooled - // buffer can lag the offscreen target whenever the - // encoder held the other slot across a tick. Publishing - // a lagging buffer would let a paint-over / burst / - // recovery send re-encode pre-damage pixels; one - // catch-up readback keeps every published buffer - // current while idle readbacks stay skipped. - if render_success && state.pool_content_gen[id] != state.content_gen { - let _ = renderer.with_context(|gl| unsafe { - gl.ReadPixels( - 0, - 0, - width, - height, - smithay::backend::renderer::gles::ffi::RGBA, - smithay::backend::renderer::gles::ffi::UNSIGNED_BYTE, - buf.as_mut_ptr() as *mut std::ffi::c_void, - ); - }); - state.pool_content_gen[id] = state.content_gen; - } - } else if state.pending_screenshot.is_some() { - let _ = renderer.with_context(|gl| unsafe { - gl.ReadPixels( - 0, - 0, - width, - height, - smithay::backend::renderer::gles::ffi::RGBA, - smithay::backend::renderer::gles::ffi::UNSIGNED_BYTE, - state.frame_buffer.as_mut_ptr() as *mut std::ffi::c_void, - ); - }); - } - }, - Err(e) => eprintln!("Failed to bind buffer: {:?}", e) - } - } - } - } else { - if let Some(renderer) = state.pixman_renderer.as_mut() { - let (ptr, buf_age) = match pool_slot { - Some((id, ref mut buf)) => { - let age = if state.pool_last_render[id] == 0 { - 0 - } else { - (state.render_seq + 1 - state.pool_last_render[id]) as usize - }; - (buf.as_mut_ptr() as *mut u32, age) - } - None => (state.frame_buffer.as_mut_ptr() as *mut u32, 0), - }; - let mut image = unsafe { - pixman::Image::from_raw_mut(pixman::FormatCode::A8R8G8B8, width as usize, height as usize, ptr, (width as usize) * 4, false).expect("Failed to create pixman image") - }; - match renderer.bind(&mut image) { - Ok(mut frame) => { - let mut elements: Vec>> = Vec::new(); - - if state.render_cursor_on_framebuffer { - if let Some(pointer) = state.seat.get_pointer() { - let pos = pointer.current_location(); - let output_scale_val = output.current_scale().fractional_scale(); - let scale = Scale::from(output_scale_val); - - if let Some(CursorImageStatus::Named(icon)) = &state.current_cursor_icon { - let name = wayland::frontend::cursor_icon_to_str(icon); - let time = Duration::from_millis(state.clock.now().as_millis() as u64); - if let Some(image) = state.cursor_helper.get_image_by_name(name, output_scale_val.round() as u32, time) { - if let Some(elem) = state.overlay_state.get_cursor_element(renderer, image, pos.to_i32_round()) { - elements.push(CompositionElements::Cursor(elem)); - } - } - } else if let Some(CursorImageStatus::Surface(surface)) = &state.current_cursor_icon { - let phys_pos = pos.to_physical(scale); - let elem_result = with_states(surface, |states| { - WaylandSurfaceRenderElement::from_surface(renderer, surface, states, phys_pos, 1.0, smithay::backend::renderer::element::Kind::Cursor) - }); - if let Ok(Some(cursor_elem)) = elem_result { - elements.push(CompositionElements::Surface(cursor_elem)); - } - } else if state.current_cursor_icon.is_none() { - let time = Duration::from_millis(state.clock.now().as_millis() as u64); - let image = state.cursor_helper.get_image(output_scale_val.round() as u32, time); - if let Some(elem) = state.overlay_state.get_cursor_element(renderer, image, pos.to_i32_round()) { - elements.push(CompositionElements::Cursor(elem)); - } - } - } - } - - if let Some(elem) = state.overlay_state.get_watermark_element(renderer) { - elements.push(CompositionElements::Cursor(elem)); - } - - { - let layer_map = layer_map_for_output(&output); - - let draw_layer = |renderer: &mut PixmanRenderer, elements: &mut Vec>>, target_layer: smithay::wayland::shell::wlr_layer::Layer| { - for surface in layer_map.layers() { - let current_layer = surface.layer(); - if current_layer == target_layer { - if let Some(geo) = layer_map.layer_geometry(surface) { - let elem = smithay::wayland::compositor::with_states(surface.wl_surface(), |states| { - WaylandSurfaceRenderElement::from_surface( - renderer, surface.wl_surface(), states, - geo.loc.to_physical_precise_round(1), 1.0, - smithay::backend::renderer::element::Kind::Unspecified - ) - }); - if let Ok(Some(e)) = elem { - elements.push(CompositionElements::Surface(e)); - } - } - } - } - }; - - draw_layer(renderer, &mut elements, smithay::wayland::shell::wlr_layer::Layer::Overlay); - draw_layer(renderer, &mut elements, smithay::wayland::shell::wlr_layer::Layer::Top); - } - - for window in state.space.elements_for_output(&output).collect::>().into_iter().rev() { - let loc = state.space.element_location(window).unwrap_or_default(); - - if let Some(surface) = window.wl_surface() { - let popups = PopupManager::popups_for_surface(&surface); - for (popup, location) in popups { - let popup_surface = popup.wl_surface(); { - let popup_pos = loc + location; - let elem = smithay::wayland::compositor::with_states(popup_surface, |states| { - WaylandSurfaceRenderElement::from_surface( - renderer, - popup_surface, - states, - popup_pos.to_physical_precise_round(1), - 1.0, - smithay::backend::renderer::element::Kind::Unspecified - ) - }); - if let Ok(Some(e)) = elem { - elements.push(CompositionElements::Surface(e)); - } - } - } - } - - elements.extend(window.render_elements(renderer, loc.to_physical_precise_round(1), Scale::from(1.0), 1.0).into_iter().map(CompositionElements::Space)); - } - - { - let layer_map = layer_map_for_output(&output); - - let draw_layer = |renderer: &mut PixmanRenderer, elements: &mut Vec>>, target_layer: smithay::wayland::shell::wlr_layer::Layer| { - for surface in layer_map.layers() { - let current_layer = surface.layer(); - if current_layer == target_layer { - if let Some(geo) = layer_map.layer_geometry(surface) { - let elem = smithay::wayland::compositor::with_states(surface.wl_surface(), |states| { - WaylandSurfaceRenderElement::from_surface( - renderer, surface.wl_surface(), states, - geo.loc.to_physical_precise_round(1), 1.0, - smithay::backend::renderer::element::Kind::Unspecified - ) - }); - if let Ok(Some(e)) = elem { - elements.push(CompositionElements::Surface(e)); - } - } - } - } - }; - - draw_layer(renderer, &mut elements, smithay::wayland::shell::wlr_layer::Layer::Bottom); - draw_layer(renderer, &mut elements, smithay::wayland::shell::wlr_layer::Layer::Background); - } - - let render_age = if state.overlay_state.is_animated() || state.needs_full_render { 0 } else { buf_age }; - match damage_tracker.render_output(renderer, &mut frame, render_age, &elements, [0.1, 0.1, 0.1, 1.0]) { - Ok(result) => { - render_success = true; - state.needs_full_render = false; - if let Some(damage) = result.damage { damage_rects = damage.clone(); } - }, - Err(e) => eprintln!("Render error: {:?}", e) - } - state.render_seq += 1; - if render_success { - if let Some((id, _)) = pool_slot { - state.pool_last_render[id] = state.render_seq; - } - } - }, - Err(e) => eprintln!("Failed to bind pixman image: {:?}", e) - } - } - } - - if render_success { - let time = state.clock.now(); - for window in state.space.elements() { - window.send_frame(&output, time, Some(Duration::ZERO), |_, _| Some(output.clone())); - } - - if is_memory_throttling { - if state.encode_pool.is_none() { - state.frame_counter = state.frame_counter.wrapping_add(1); - } - } else if state.encode_pool.is_some() { - if state.pending_screenshot.is_some() { - if let Some((_, ref buf)) = pool_slot { - let n = buf.len().min(state.frame_buffer.len()); - state.frame_buffer[..n].copy_from_slice(&buf[..n]); - } - } - if let Some((id, buf)) = pool_slot.take() { - let is_animated = state.overlay_state.is_animated(); - state.encode_pool.as_ref().unwrap().publish(WlFrame { - id, - buf, - frame_id: state.frame_counter, - damage: std::mem::take(&mut damage_rects), - is_animated, - }); - state.frame_counter = state.frame_counter.wrapping_add(1); - } - } else if let Some(ref mut encoder) = state.video_encoder { - // Deliver the parked frame (if any) first, WITHOUT blocking: - // this runs on the calloop thread, and a blocking send would - // freeze input/command/Wayland dispatch for as long as the - // Python consumer stalls (the readback path offloads its - // sends to the wl-encode thread for exactly this reason). - // While a frame stays parked, no new frame is encoded — an - // encoded frame joins the H.264 reference chain and can - // never be dropped — and the tick's damage is latched so - // the pause never loses a change. - let slot_free = match state.pending_hw_delivery.take() { - None => true, - Some(pending) => match state.deliver_tx.as_ref() { - None => true, - Some(tx) => match tx.try_send(pending) { - Ok(()) => true, - Err(std::sync::mpsc::TrySendError::Full(p)) => { - state.pending_hw_delivery = Some(p); - false - } - Err(std::sync::mpsc::TrySendError::Disconnected(_)) => true, - }, - }, - }; - if !slot_free { - if !damage_rects.is_empty() { - state.pending_hw_damage = true; - } - } else { - let is_animated = state.overlay_state.is_animated(); - let had_damage = !damage_rects.is_empty() - || std::mem::take(&mut state.pending_hw_damage); - let decision = crate::pipeline::decide_hw_fullframe( - &mut state.vaapi_state, - &state.settings, - state.frame_counter, - had_damage, - is_animated, - requested_idr, - ); - let send_frame = decision.send; - let force_idr = decision.force_idr; - let target_qp = decision.target_qp; + .insert_source(timer, move |_, _, state| { + // Apply queued commands (input above all) BEFORE the render/encode work: a + // command that raced the timer wakeup would otherwise wait out the whole tick. + drain_thread_commands(state); + let loop_start_time = Instant::now(); + state.space.refresh(); - if send_frame { - if let Some(sync) = render_sync.take() { - let _ = sync.wait(); - } - let force_idr_for_recording = state - .recording_sink - .as_ref() - .map(|s| s.should_force_idr()) - .unwrap_or(false); - let force_idr = force_idr || force_idr_for_recording; - let result = match encoder { - GpuEncoder::Nvenc(enc) => { - if let Some((_, ref dmabuf)) = state.offscreen_buffer { - enc.encode(dmabuf, state.frame_counter as u64, target_qp, force_idr) - } else { - Err("NVENC ZeroCopy requires offscreen buffer (GPU context)".to_string()) - } - }, - GpuEncoder::Vaapi(enc) => { - if let Some((_, ref dmabuf)) = state.offscreen_buffer { - enc.encode_dmabuf(dmabuf, state.frame_counter as u64, target_qp, force_idr) - } else { - Err("Vaapi ZeroCopy requires offscreen buffer (GPU context)".to_string()) - } - } - }; + let (current_rss, shm_usage) = get_cached_mem_usage(); + // The threshold follows the largest active capture (fallback: primary settings). + let (max_w, max_h) = state + .output_nodes + .iter() + .filter_map(|n| n.capture.as_ref().map(|c| (c.settings.width, c.settings.height))) + .fold((state.settings.width, state.settings.height), |acc, (w, h)| { + (acc.0.max(w), acc.1.max(h)) + }); + let memory_threshold = calculate_memory_threshold(max_w, max_h); - if let Ok(data) = result { - if !data.is_empty() { - state.encode_stats.frames.fetch_add(1, Ordering::Relaxed); - state.encode_stats.stripes.fetch_add(1, Ordering::Relaxed); - if let Some(ref tx) = state.deliver_tx { - let stripes = vec![EncodedStripe { - data, data_type: 2, stripe_y_start: 0, - stripe_height: height, frame_id: state.frame_counter as i32, - }]; - if let Some(ref socket) = state.recording_sink { - for stripe in &stripes { - let _ = socket.write_encoded_frame(stripe); - } - } - // Non-blocking: a full slot parks the frame - // (delivered ahead of any new encode above). - match tx.try_send(stripes) { - Ok(()) => {} - Err(std::sync::mpsc::TrySendError::Full(s)) => { - state.pending_hw_delivery = Some(s); - } - Err(std::sync::mpsc::TrySendError::Disconnected(_)) => {} - } - } - } - } else if let Err(e) = result { - eprintln!("HW Encode Error: {}", e); - } - } - state.pending_force_idr = false; - state.frame_counter = state.frame_counter.wrapping_add(1); - } + if current_rss > memory_threshold || shm_usage > (4 * 1024 * 1024 * 1024) { + if !is_memory_throttling { + is_memory_throttling = true; + } + } else if is_memory_throttling + && current_rss < (memory_threshold as f64 * 0.75) as usize && shm_usage < (3 * 1024 * 1024 * 1024) { + is_memory_throttling = false; + } + + let now = Instant::now(); + let elapsed = now.duration_since(state.last_log_time).as_secs_f64(); + if elapsed >= 1.0 { + for node in &state.output_nodes { + let Some(cap) = node.capture.as_ref() else { continue }; + let frames = cap.encode_stats.frames.swap(0, Ordering::Relaxed); + let stripes = cap.encode_stats.stripes.swap(0, Ordering::Relaxed); + if cap.settings.debug_logging { + let actual_fps = frames as f64 / elapsed; + let stripes_per_sec = stripes as f64 / elapsed; + let mode_str = cap.encode_stats.desc.lock().unwrap().clone(); + let n_stripes = cap.encode_stats.n_stripes.load(Ordering::Relaxed); + + let rss_mb = current_rss / 1024 / 1024; + let shm_mb = shm_usage / 1024 / 1024; + let throttle_warn = if is_memory_throttling { " [THROTTLED]" } else { "" }; + + println!("Display: {} Res: {}x{} Mode: {} Stripes: {} EncFPS: {:.2} EncStripes/s: {:.2} Mem: {}MB SHM: {}MB{}", + node.id, cap.settings.width, cap.settings.height, mode_str, n_stripes, actual_fps, stripes_per_sec, rss_mb, shm_mb, throttle_warn); } - if let Some(resp) = state.pending_screenshot.take() { - if !state.frame_buffer.is_empty() { - let w = state.settings.width as u32; - let h = state.settings.height as u32; - let png = if state.use_gpu { - crate::computer_use::encode_png_rgba(&state.frame_buffer, w, h) - } else { - let mut rgba = state.frame_buffer.clone(); - for px in rgba.chunks_exact_mut(4) { - px.swap(0, 2); - } - crate::computer_use::encode_png_rgba(&rgba, w, h) - }; - match png { - Ok(data) => { let _ = resp.send(data); } - Err(e) => { let _ = resp.send(Vec::new()); eprintln!("[ComputerUse] PNG encode error: {}", e); } - } - } else { - let _ = resp.send(Vec::new()); - } + } + state.last_log_time = now; + } + + let any_capturing = state.output_nodes.iter().any(|n| n.capture.is_some()); + if !any_capturing && state.pending_screenshot.is_none() { + // No render/encode work, but committed clients still need their + // frame callbacks: a vsynced client (FIFO Vulkan present, games, a + // nested compositor's own clients) otherwise blocks in its swap + // until a viewer attaches — apps appear frozen whenever nobody is + // watching. + let time = state.clock.now(); + for node in &state.output_nodes { + for window in state + .space + .elements_for_output(&node.output) + .cloned() + .collect::>() + { + window.send_frame(&node.output, time, Some(Duration::ZERO), |_, _| { + Some(node.output.clone()) + }); } } + return TimeoutAction::ToDuration(Duration::from_millis(16)); } - if let Some((id, buf)) = pool_slot.take() { - if let Some(ref pool) = state.encode_pool { - pool.cancel(id, buf); + + // Render every output that needs it. The nodes are taken out of the state so + // each per-output render can borrow the shared renderer/space alongside its + // own damage tracker and buffers. + let mut nodes = std::mem::take(&mut state.output_nodes); + let mut any_pool_busy = false; + for node in nodes.iter_mut() { + if render_node_tick(state, node, is_memory_throttling) { + any_pool_busy = true; } } + state.output_nodes = nodes; + + if any_pool_busy { + return TimeoutAction::ToDuration(Duration::from_millis(1)); + } let work_elapsed = loop_start_time.elapsed(); - let fps = (if is_memory_throttling { 5.0 } else { state.settings.target_fps }).max(1.0); + let max_fps = state + .output_nodes + .iter() + .filter_map(|n| n.capture.as_ref().map(|c| c.settings.target_fps)) + .fold(0.0f64, f64::max); + let raw_fps = if is_memory_throttling { + 5.0 + } else if max_fps > 0.0 { + max_fps + } else { + state.settings.target_fps + }; + // Settings are sanitized at the Python boundary; this final guard keeps a + // non-finite value from ever reaching Duration::from_secs_f64 (panics on NaN). + let fps = if raw_fps.is_finite() && raw_fps > 0.0 { raw_fps.min(MAX_FPS) } else { DEFAULT_FPS }; let target_frame_duration = Duration::from_secs_f64(1.0 / fps); let wait_duration = target_frame_duration.saturating_sub(work_elapsed); let final_wait = if wait_duration.as_millis() < 1 { Duration::from_millis(1) } else { wait_duration }; @@ -2925,27 +4376,22 @@ fn run_wayland_thread( event_loop .handle() .insert_source(Generic::new(display, Interest::READ, Mode::Level), |_, display, _state| { - unsafe { display.get_mut().dispatch_clients(_state).unwrap(); } + // A single misbehaving client must not take the compositor (and every other + // session) down with it. + if let Err(e) = unsafe { display.get_mut().dispatch_clients(_state) } { + eprintln!("[Wayland] client dispatch error: {e:?}"); + } Ok(PostAction::Continue) }) .unwrap(); - if let Ok(port_str) = std::env::var("PIXELFLUX_CU") { - if let Ok(port) = port_str.parse::() { - println!("[ComputerUse] Starting server on port {} (PIXELFLUX_CU={})", port, port); - let cu_tx = command_tx.clone(); - thread::spawn(move || { - crate::computer_use::run_cu_server(cu_tx, port); - }); - } else { - println!("[ComputerUse] Invalid PIXELFLUX_CU value: '{}' - expected a port number", port_str); - } - } + crate::computer_use::register_wayland_backend(command_tx.clone()); + crate::computer_use::spawn_cu_from_env(); - event_loop.run(None, &mut state, |state| { + let _ = event_loop.run(None, &mut state, |state| { state.process_pending_clipboard_read(); - state.dh.flush_clients().unwrap(); - }).unwrap(); + let _ = state.dh.flush_clients(); + }); } /// Zero-copy encoded-frame handoff to Python. Owns the encoded `Vec` and @@ -2954,7 +4400,7 @@ fn run_wayland_thread( /// four stripe-metadata ints as Python attributes. #[pyclass] struct StripeFrame { - data: Vec, + data: Arc>, #[pyo3(get, set)] data_type: i32, #[pyo3(get, set)] @@ -2966,10 +4412,10 @@ struct StripeFrame { } impl StripeFrame { - /// Hot-path constructor: MOVES the encoded buffer in (no copy) and carries stripe + /// Hot-path constructor: shares the encoder's buffer by `Arc` (no copy) and carries stripe /// metadata as attributes, so the consumer can read it without parsing a header /// (required for omit_stripe_headers). - fn new_owned_meta(data: Vec, data_type: i32, stripe_y_start: i32, stripe_height: i32, frame_id: i32) -> Self { + fn new_owned_meta(data: Arc>, data_type: i32, stripe_y_start: i32, stripe_height: i32, frame_id: i32) -> Self { Self { data, data_type, stripe_y_start, stripe_height, frame_id } } } @@ -2981,7 +4427,7 @@ impl StripeFrame { #[new] #[pyo3(signature = (data, data_type = 0, stripe_y_start = 0, stripe_height = 0, frame_id = 0))] fn new(data: Vec, data_type: i32, stripe_y_start: i32, stripe_height: i32, frame_id: i32) -> Self { - Self { data, data_type, stripe_y_start, stripe_height, frame_id } + Self { data: Arc::new(data), data_type, stripe_y_start, stripe_height, frame_id } } fn __len__(&self) -> usize { @@ -3022,6 +4468,18 @@ impl StripeFrame { #[pyclass] struct WaylandBackend { tx: smithay::reexports::calloop::channel::Sender, + /// Wakes the calloop after each command send: the command channel itself is drained in + /// place by the compositor thread (render tick and wake handler), not registered as its + /// own source, so input never waits behind an in-flight render tick's timer wakeup. + wake_tx: smithay::reexports::calloop::channel::Sender<()>, +} + +impl WaylandBackend { + fn send(&self, cmd: ThreadCommand) -> Result<(), String> { + self.tx.send(cmd).map_err(|e| e.to_string())?; + let _ = self.wake_tx.send(()); + Ok(()) + } } #[pymethods] @@ -3040,71 +4498,200 @@ impl WaylandBackend { cursor_size: i32, ) -> Self { let (tx, rx) = smithay::reexports::calloop::channel::channel(); + let (wake_tx, wake_rx) = smithay::reexports::calloop::channel::channel(); let cu_tx = tx.clone(); thread::spawn(move || { crate::boost_thread_priority(-15); - run_wayland_thread(rx, cu_tx, width, height, dri_node, auto_gpu_selected, cursor_size); + run_wayland_thread(WaylandThreadConfig { + command_rx: rx, + wake_rx, + command_tx: cu_tx, + initial_width: width, + initial_height: height, + explicit_dri_node: dri_node, + auto_gpu_selected, + cursor_size, + }); }); - WaylandBackend { tx } + WaylandBackend { tx, wake_tx } } - /// Begin a capture with the given frame callback and settings. + /// Begin a capture with the given frame callback and settings. The target display is + /// the settings' `display_id` attribute (absent = 0, the primary); each display id runs + /// at most one capture, independent of every other display's. /// /// Issuing the start also clears the interpreter-teardown gate: starting from Python proves the /// interpreter is live again after a manual atexit sweep. fn start_capture(&self, callback: Py, settings: &Bound<'_, PyAny>) -> PyResult<()> { let rust_settings = extract_settings(settings)?; + let display_id = read_display_id(settings); PY_SHUTDOWN.store(false, Ordering::Relaxed); - self.tx - .send(ThreadCommand::StartCapture(callback, rust_settings)) + self.send(ThreadCommand::StartCapture { display_id, callback: Some(callback), settings: rust_settings }) .map_err(|e| PyErr::new::(format!("Failed to send start command: {}", e)))?; Ok(()) } - fn stop_capture(&self) -> PyResult<()> { - self.tx - .send(ThreadCommand::StopCapture) + /// Stop the capture bound to `display_id` (default: the primary display). + #[pyo3(signature = (display_id = 0))] + fn stop_capture(&self, display_id: u32) -> PyResult<()> { + self.send(ThreadCommand::StopCapture { display_id }) .map_err(|e| PyErr::new::(format!("Failed to send stop command: {}", e)))?; Ok(()) } + /// Create an additional output (`WxH` physical pixels at fractional `scale`) mapped into + /// the layout at offset `(x, y)`; `id` is the display key used by every per-display API. + /// False when the id is taken, the geometry/scale is invalid, the rectangle overlaps a + /// live output, or the GPU render target cannot be allocated. Capture reconfigures + /// (`start_capture` on an existing output) resize without this validation, so a + /// multi-step relayout must stay overlap-free at every step by caller ordering. + // The parameter list is the Python signature; grouping it would change the ABI. + #[allow(clippy::too_many_arguments)] + #[pyo3(signature = (id, width, height, x = 0, y = 0, scale = 1.0))] + fn create_output( + &self, + py: Python<'_>, + id: u32, + width: i32, + height: i32, + x: i32, + y: i32, + scale: f64, + ) -> PyResult { + let (reply_tx, reply_rx) = std::sync::mpsc::channel::(); + self.send(ThreadCommand::CreateOutput { id, width, height, x, y, scale, reply: reply_tx }) + .map_err(|e| PyErr::new::(format!("Failed to create output: {}", e)))?; + Ok(py + .detach(move || reply_rx.recv_timeout(Duration::from_secs(2))) + .unwrap_or(false)) + } + + /// Destroy a secondary output: its capture ends cleanly and its windows relocate to the + /// primary output. False for the primary (id 0) or an unknown id. + fn destroy_output(&self, py: Python<'_>, id: u32) -> PyResult { + let (reply_tx, reply_rx) = std::sync::mpsc::channel::(); + self.send(ThreadCommand::DestroyOutput { id, reply: reply_tx }) + .map_err(|e| PyErr::new::(format!("Failed to destroy output: {}", e)))?; + Ok(py + .detach(move || reply_rx.recv_timeout(Duration::from_secs(2))) + .unwrap_or(false)) + } + + /// Move an existing output (the primary, id 0, included) to layout offset `(x, y)`; + /// its windows, absolute input injection, and cursor compositing follow. False for an + /// unknown id or a destination overlapping a live output. Capture reconfigures + /// (`start_capture` on an existing output) resize without this validation, so a + /// multi-step relayout must stay overlap-free at every step by caller ordering. + fn reposition_output(&self, py: Python<'_>, id: u32, x: i32, y: i32) -> PyResult { + let (reply_tx, reply_rx) = std::sync::mpsc::channel::(); + self.send(ThreadCommand::RepositionOutput { id, x, y, reply: reply_tx }) + .map_err(|e| PyErr::new::(format!("Failed to reposition output: {}", e)))?; + Ok(py + .detach(move || reply_rx.recv_timeout(Duration::from_secs(2))) + .unwrap_or(false)) + } + + /// Every live output as `(id, x, y, width, height, scale, capturing)` — width/height in + /// physical pixels, `(x, y)` the layout offset. + fn list_outputs(&self, py: Python<'_>) -> PyResult> { + let (reply_tx, reply_rx) = std::sync::mpsc::channel(); + self.send(ThreadCommand::ListOutputs { reply: reply_tx }) + .map_err(|e| PyErr::new::(format!("Failed to list outputs: {}", e)))?; + Ok(py + .detach(move || reply_rx.recv_timeout(Duration::from_secs(2))) + .unwrap_or_default()) + } + + /// Move the window with the given id onto output `output_id`, fullscreened at that + /// output's logical size. False for an unknown window or output id. + fn move_window_to_output(&self, py: Python<'_>, window_id: u32, output_id: u32) -> PyResult { + let (reply_tx, reply_rx) = std::sync::mpsc::channel::(); + self.send(ThreadCommand::MoveWindowToOutput { window_id, output_id, reply: reply_tx }) + .map_err(|e| PyErr::new::(format!("Failed to move window: {}", e)))?; + Ok(py + .detach(move || reply_rx.recv_timeout(Duration::from_secs(2))) + .unwrap_or(false)) + } + + /// Every mapped window as `(window_id, title, app_id, output_id, waiting)`. A waiting + /// window is tagged for that output but mapped clear of every one of them, holding its + /// size until an output exists for it — a nested session's spare screens. + fn list_windows(&self, py: Python<'_>) -> PyResult> { + let (reply_tx, reply_rx) = std::sync::mpsc::channel(); + self.send(ThreadCommand::ListWindows { reply: reply_tx }) + .map_err(|e| PyErr::new::(format!("Failed to list windows: {}", e)))?; + Ok(py + .detach(move || reply_rx.recv_timeout(Duration::from_secs(2))) + .unwrap_or_default()) + } + fn set_cursor_callback(&self, callback: Py) -> PyResult<()> { - self.tx - .send(ThreadCommand::SetCursorCallback(callback)) + self.send(ThreadCommand::SetCursorCallback(callback)) .map_err(|e| PyErr::new::(format!("Failed to set cursor callback: {}", e)))?; Ok(()) } + /// Recreate the cursor theme at `size` pixels — no restart: subsequent named-cursor + /// callbacks and the burned-in cursor overlay render at the new size. False for a + /// non-positive size. + fn set_cursor_size(&self, py: Python<'_>, size: i32) -> PyResult { + let (reply_tx, reply_rx) = std::sync::mpsc::channel::(); + self.send(ThreadCommand::SetCursorSize { size, reply: reply_tx }) + .map_err(|e| PyErr::new::(format!("Failed to set cursor size: {}", e)))?; + Ok(py + .detach(move || reply_rx.recv_timeout(Duration::from_secs(2))) + .unwrap_or(false)) + } + /// cb(mime: str, data: bytes) fires when a client app copies to the clipboard. fn set_clipboard_callback(&self, callback: Py) -> PyResult<()> { - self.tx - .send(ThreadCommand::SetClipboardCallback(callback)) + self.send(ThreadCommand::SetClipboardCallback(callback)) .map_err(|e| PyErr::new::(format!("Failed to set clipboard callback: {}", e)))?; Ok(()) } /// Compositor-side clipboard offer: serve `data` as `mime` to pasting clients. fn set_clipboard(&self, mime: String, data: Vec) -> PyResult<()> { - self.tx - .send(ThreadCommand::SetClipboard { mime, data }) + self.send(ThreadCommand::SetClipboard { mime, data }) .map_err(|e| PyErr::new::(format!("Failed to set clipboard: {}", e)))?; Ok(()) } fn inject_key(&self, scancode: u32, state: u32) -> PyResult<()> { - self.tx - .send(ThreadCommand::KeyboardKey { scancode, state }) + self.send(ThreadCommand::KeyboardKey { scancode, state }) .map_err(|e| PyErr::new::(format!("Failed to inject key: {}", e)))?; Ok(()) } + /// Inject an ordered run of `(keycode, state)` events as one message, so a paste + /// costs one channel send and one wake instead of one per event. + fn inject_keys(&self, events: Vec<(u32, u32)>) -> PyResult<()> { + if events.is_empty() { + return Ok(()); + } + self.send(ThreadCommand::KeyboardKeys { events }) + .map_err(|e| PyErr::new::(format!("Failed to inject keys: {}", e)))?; + Ok(()) + } + /// Swap the seat keyboard's xkb keymap (XKB_KEYMAP_FORMAT_TEXT_V1 text). The caller /// owns keysym-to-keycode policy: define keycodes here, then press them via /// `inject_key`. Ordered with key events on the one compositor channel. + /// Bind explicit `(keycode, keysym)` pairs onto the current base keymap in ONE swap. + /// + /// The caller owns the assignment — which keysym goes to which keycode, and when to + /// recycle one — because that tracks layouts and user reports. This end only assembles + /// the xkb text and delivers it, reusing the installed base rather than recompiling a + /// re-supplied one. False when no base keymap is installed yet. + fn set_keymap_overlay(&self, binds: Vec<(u32, u32)>) -> PyResult<()> { + self.send(ThreadCommand::SetKeymapOverlay { binds }) + .map_err(|e| PyErr::new::(format!("Failed to set overlay: {}", e)))?; + Ok(()) + } + fn set_keymap_string(&self, text: String) -> PyResult<()> { - self.tx - .send(ThreadCommand::SetKeymapString(text)) + self.send(ThreadCommand::SetKeymapString(text)) .map_err(|e| PyErr::new::(format!("Failed to set keymap: {}", e)))?; Ok(()) } @@ -3117,8 +4704,7 @@ impl WaylandBackend { /// and an empty string is returned when the keymap cannot be read in time. fn get_xkb_keymap_string(&self, py: Python<'_>) -> PyResult { let (reply_tx, reply_rx) = std::sync::mpsc::channel::(); - self.tx - .send(ThreadCommand::GetXkbKeymap { reply: reply_tx }) + self.send(ThreadCommand::GetXkbKeymap { reply: reply_tx }) .map_err(|e| PyErr::new::(format!("Failed to request keymap: {}", e)))?; let result = py.detach(move || reply_rx.recv_timeout(Duration::from_secs(2))); match result { @@ -3128,36 +4714,31 @@ impl WaylandBackend { } fn inject_mouse_move(&self, x: f64, y: f64) -> PyResult<()> { - self.tx - .send(ThreadCommand::PointerMotion { x, y }) + self.send(ThreadCommand::PointerMotion { x, y }) .map_err(|e| PyErr::new::(format!("Failed to inject motion: {}", e)))?; Ok(()) } fn inject_relative_mouse_move(&self, dx: f64, dy: f64) -> PyResult<()> { - self.tx - .send(ThreadCommand::PointerRelativeMotion { dx, dy }) + self.send(ThreadCommand::PointerRelativeMotion { dx, dy }) .map_err(|e| PyErr::new::(format!("Failed to inject relative motion: {}", e)))?; Ok(()) } fn inject_mouse_button(&self, btn: u32, state: u32) -> PyResult<()> { - self.tx - .send(ThreadCommand::PointerButton { btn, state }) + self.send(ThreadCommand::PointerButton { btn, state }) .map_err(|e| PyErr::new::(format!("Failed to inject button: {}", e)))?; Ok(()) } fn inject_mouse_scroll(&self, x: f64, y: f64) -> PyResult<()> { - self.tx - .send(ThreadCommand::PointerAxis { x, y }) + self.send(ThreadCommand::PointerAxis { x, y }) .map_err(|e| PyErr::new::(format!("Failed to inject axis: {}", e)))?; Ok(()) } fn set_cursor_rendering(&self, enabled: bool) -> PyResult<()> { - self.tx - .send(ThreadCommand::UpdateCursorConfig { render_on_framebuffer: enabled }) + self.send(ThreadCommand::UpdateCursorConfig { render_on_framebuffer: enabled }) .map_err(|e| PyErr::new::(format!("Failed to set cursor config: {}", e)))?; Ok(()) } @@ -3166,31 +4747,87 @@ impl WaylandBackend { /// or a decoder reset can resume immediately. With the default infinite GOP this /// is the only recovery path, so every consumer that can lose decoder state must /// call it. No-op cost on the JPEG/software path (keyframes are N/A). - fn request_idr_frame(&self) -> PyResult<()> { - self.tx - .send(ThreadCommand::RequestIdr) + #[pyo3(signature = (display_id = 0))] + fn request_idr_frame(&self, display_id: u32) -> PyResult<()> { + self.send(ThreadCommand::RequestIdr { display_id }) .map_err(|e| PyErr::new::(format!("Failed to request IDR: {}", e)))?; Ok(()) } - /// Apply a live bitrate (kbps) / VBV (kb) / framerate change to the running capture. - fn update_rate(&self, bitrate_kbps: Option, vbv_multiplier: Option, fps: Option) -> PyResult<()> { - self.tx - .send(ThreadCommand::UpdateRate { bitrate_kbps, vbv_multiplier, fps }) + /// Apply a live bitrate (kbps) / VBV (kb) / framerate change to the given display's + /// running capture. + #[pyo3(signature = (bitrate_kbps = None, vbv_multiplier = None, fps = None, display_id = 0))] + fn update_rate(&self, bitrate_kbps: Option, vbv_multiplier: Option, fps: Option, display_id: u32) -> PyResult<()> { + self.send(ThreadCommand::UpdateRate { display_id, bitrate_kbps, vbv_multiplier, fps }) .map_err(|e| PyErr::new::(format!("Failed to update rate: {}", e)))?; Ok(()) } + + /// Set the seat's BASE xkb layout from RMLVO names at runtime (empty strings select the + /// xkbcommon defaults). Returns whether the layout compiled and was applied; overlay binds + /// rebuild on top with their keycodes unchanged. + #[pyo3(signature = (layout, variant = String::new(), options = String::new(), model = String::new(), rules = String::new()))] + fn set_xkb_layout( + &self, + py: Python<'_>, + layout: String, + variant: String, + options: String, + model: String, + rules: String, + ) -> PyResult { + let (reply_tx, reply_rx) = std::sync::mpsc::channel::(); + self.send(ThreadCommand::SetXkbLayout { rules, model, layout, variant, options, reply: reply_tx }) + .map_err(|e| PyErr::new::(format!("Failed to set layout: {}", e)))?; + Ok(py + .detach(move || reply_rx.recv_timeout(Duration::from_secs(2))) + .unwrap_or(false)) + } + + /// Debug/verification readback of the seat keyboard: `(pressed_keycodes, modifier_mask)` + /// with mask bits 1 ctrl, 2 shift, 4 alt, 8 logo, 16 caps, 32 num, 64 altgr, 128 level5. + fn get_keyboard_state(&self, py: Python<'_>) -> PyResult<(Vec, u32)> { + let (reply_tx, reply_rx) = std::sync::mpsc::channel::<(Vec, u32)>(); + self.send(ThreadCommand::GetKeyboardState { reply: reply_tx }) + .map_err(|e| PyErr::new::(format!("Failed to read keyboard state: {}", e)))?; + Ok(py + .detach(move || reply_rx.recv_timeout(Duration::from_secs(2))) + .unwrap_or_default()) + } + + /// The capture geometry actually live on the given display: `(width, height, scale)` in + /// physical pixels. Reflects any degrade a `start_capture` performed (H.264 even-masking, + /// GBM allocation failure keeping the previous mode), and the command channel is FIFO, so + /// calling this after `start_capture` returns what that start realized. + #[pyo3(signature = (display_id = 0))] + fn get_realized_geometry(&self, py: Python<'_>, display_id: u32) -> PyResult<(i32, i32, f64)> { + let (reply_tx, reply_rx) = std::sync::mpsc::channel::<(i32, i32, f64)>(); + self.send(ThreadCommand::CuGetInfo { display_id, resp: reply_tx }) + .map_err(|e| PyErr::new::(format!("Failed to read geometry: {}", e)))?; + Ok(py + .detach(move || reply_rx.recv_timeout(Duration::from_secs(2))) + .unwrap_or((0, 0, 0.0))) + } } impl WaylandBackend { - fn update_tunables(&self, t: LiveTunables) -> PyResult<()> { - self.tx - .send(ThreadCommand::UpdateTunables(t)) + fn update_tunables(&self, display_id: u32, t: LiveTunables) -> PyResult<()> { + self.send(ThreadCommand::UpdateTunables { display_id, tunables: t }) .map_err(|e| PyErr::new::(format!("Failed to update tunables: {}", e)))?; Ok(()) } } +/// The optional `display_id` attribute on a settings object (absent/invalid = 0, the +/// primary display). +fn read_display_id(settings: &Bound<'_, PyAny>) -> u32 { + settings + .getattr("display_id") + .ok() + .and_then(|v| v.extract::().ok()) + .unwrap_or(0) +} + use std::sync::atomic::{AtomicBool, AtomicI32, AtomicU32, AtomicU64, Ordering}; use std::sync::{Condvar, Mutex, OnceLock}; @@ -3210,7 +4847,7 @@ fn stripe_frame_from_buffer( stripe_height: i32, frame_id: i32, ) -> StripeFrame { - StripeFrame::new_owned_meta(data, data_type, stripe_y_start, stripe_height, frame_id) + StripeFrame::new_owned_meta(Arc::new(data), data_type, stripe_y_start, stripe_height, frame_id) } /// Capture configuration read by `start_capture` (each field by attribute name via @@ -3218,6 +4855,8 @@ fn stripe_frame_from_buffer( /// can stash extra attributes not listed here. #[pyclass(dict)] struct CaptureSettings { + /// Wayland display key this capture binds to (0 = primary output); ignored on X11. + #[pyo3(get, set)] display_id: u32, #[pyo3(get, set)] capture_width: i32, #[pyo3(get, set)] capture_height: i32, #[pyo3(get, set)] scale: f64, @@ -3263,6 +4902,8 @@ struct CaptureSettings { #[pyo3(get, set)] use_wayland: Py, /// H.264 recording tap: a Unix socket path to bind, or empty for none. #[pyo3(get, set)] recording_socket: Py, + /// Wayland display of an EXTERNAL compositor to capture (host-capture mode). + #[pyo3(get, set)] wayland_host_display: Py, /// Compositor cursor-theme size in pixels; <=0 keeps the theme default (24). #[pyo3(get, set)] cursor_size: i32, /// Longest cursor edge the X11 out-of-band cursor callback delivers; larger @@ -3275,6 +4916,7 @@ impl CaptureSettings { #[new] fn new(py: Python<'_>) -> Self { Self { + display_id: 0, capture_width: 1920, capture_height: 1080, scale: 1.0, capture_x: 0, capture_y: 0, target_fps: 60.0, jpeg_quality: 85, paint_over_jpeg_quality: 95, use_paint_over_quality: false, paint_over_trigger_frames: 10, @@ -3289,13 +4931,42 @@ impl CaptureSettings { auto_adjust_screen_capture_size: false, omit_stripe_headers: false, encode_node_path: py.None(), render_node_path: py.None(), auto_gpu: py.None(), use_wayland: py.None(), - recording_socket: py.None(), cursor_size: -1, cursor_size_cap: 32, + recording_socket: py.None(), wayland_host_display: py.None(), + cursor_size: -1, cursor_size_cap: 32, } } } /// Process-wide Wayland backend: input and capture share ONE compositor (constructed lazily). static WAYLAND_BACKEND: OnceLock>>> = OnceLock::new(); +/// The compositor's auto-picked socket name (e.g. "wayland-1"), published by the compositor +/// thread once its listening socket exists. `ListeningSocketSource::new_auto` binds the first +/// FREE wayland-N, which need not match any configured index — consumers must read the real +/// name from here instead of assuming one. +static WAYLAND_SOCKET_NAME: Mutex> = Mutex::new(None); +static WAYLAND_SOCKET_CV: Condvar = Condvar::new(); + +fn publish_socket_name(name: &str) { + *WAYLAND_SOCKET_NAME.lock().unwrap() = Some(name.to_string()); + WAYLAND_SOCKET_CV.notify_all(); +} + +/// Wait (bounded) for the compositor thread to publish its socket name. +fn wait_socket_name(timeout: Duration) -> Option { + let deadline = Instant::now() + timeout; + let mut g = WAYLAND_SOCKET_NAME.lock().unwrap(); + loop { + if let Some(name) = g.as_ref() { + return Some(name.clone()); + } + let now = Instant::now(); + if now >= deadline { + return None; + } + let (gg, _) = WAYLAND_SOCKET_CV.wait_timeout(g, deadline - now).unwrap(); + g = gg; + } +} /// Cursor callback registered before the backend exists (selkies registers it pre-start); /// applied when the backend is created, which is deferred to capture start so the real /// render node (not a placeholder) reaches the compositor. @@ -3304,16 +4975,25 @@ static PENDING_CURSOR_CALLBACK: Mutex>> = Mutex::new(None); /// threads must never attach to a finalizing interpreter (aborts the process pre-3.13). /// Cleared by a fresh capture start (only a live interpreter can start one). pub(crate) static PY_SHUTDOWN: AtomicBool = AtomicBool::new(false); -/// ScreenCapture id that owns the active shared Wayland capture (0 = none). Only the owner may -/// stop it, so an input-only or stale instance can't tear down a live capture. -static WAYLAND_OWNER: AtomicU64 = AtomicU64::new(0); -/// Real Wayland capture liveness (StartCapture -> true, StopCapture -> false), mirrored -/// from AppState.is_capturing so the Python-facing is_capturing() reports whether the -/// pipeline is actually running, not merely which ScreenCapture owns the backend. -static WAYLAND_CAPTURE_ALIVE: std::sync::atomic::AtomicBool = - std::sync::atomic::AtomicBool::new(false); -/// Hands each `ScreenCapture` a unique, monotonic id — the token `WAYLAND_OWNER` compares to -/// decide which instance is allowed to stop the one shared compositor capture. Starts at 1 so 0 can +/// Per-display capture ownership: display id -> the ScreenCapture id that owns that +/// display's capture. Only the owner may stop it, so an input-only or stale instance can't +/// tear down a live capture. +static WAYLAND_OWNERS: OnceLock>> = OnceLock::new(); + +fn wayland_owners() -> &'static Mutex> { + WAYLAND_OWNERS.get_or_init(|| Mutex::new(std::collections::HashMap::new())) +} + +/// Display ids whose capture pipeline is actually running (StartCapture inserts, +/// StopCapture/DestroyOutput remove), so the Python-facing is_capturing() reports pipeline +/// liveness, not merely which ScreenCapture owns the backend. +static WAYLAND_ALIVE_DISPLAYS: OnceLock>> = OnceLock::new(); + +fn wayland_alive() -> &'static Mutex> { + WAYLAND_ALIVE_DISPLAYS.get_or_init(|| Mutex::new(std::collections::HashSet::new())) +} +/// Hands each `ScreenCapture` a unique, monotonic id — the token `WAYLAND_OWNERS` compares to +/// decide which instance is allowed to stop a display's capture. Starts at 1 so 0 can /// mean "no owner". static NEXT_CAPTURE_ID: AtomicU64 = AtomicU64::new(1); /// Registry of every live X11 capture's `Controls`, so the atexit sweep can flag them all to @@ -3357,22 +5037,23 @@ pub(crate) fn boost_thread_priority(nice: libc::c_int) { /// Forward a live rate change to the shared Wayland backend (no-op if none is running). fn wayland_update_rate( py: Python<'_>, + display_id: u32, bitrate_kbps: Option, vbv_multiplier: Option, fps: Option, ) { if let Some(slot) = WAYLAND_BACKEND.get() { if let Some(be) = slot.lock().unwrap().as_ref() { - let _ = be.bind(py).borrow().update_rate(bitrate_kbps, vbv_multiplier, fps); + let _ = be.bind(py).borrow().update_rate(bitrate_kbps, vbv_multiplier, fps, display_id); } } } /// Forward live per-frame tunables to the shared Wayland backend (no-op if none is running). -fn wayland_update_tunables(py: Python<'_>, t: LiveTunables) { +fn wayland_update_tunables(py: Python<'_>, display_id: u32, t: LiveTunables) { if let Some(slot) = WAYLAND_BACKEND.get() { if let Some(be) = slot.lock().unwrap().as_ref() { - let _ = be.bind(py).borrow().update_tunables(t); + let _ = be.bind(py).borrow().update_tunables(display_id, t); } } } @@ -3457,6 +5138,14 @@ fn want_wayland(settings: &Bound<'_, PyAny>) -> bool { struct ScState { /// 0 = idle, 1 = X11, 2 = Wayland. backend: u8, + /// This capture holds one reference on the shared X11 cursor monitor. Set in + /// the same locked section as `backend = 1` and TAKEN in the same locked + /// section a stop reads the backend, so acquire/release pair exactly per + /// capture — inferring the reference from `backend` alone would let a stop + /// that interleaves with a start release a reference not yet taken (leaking + /// the monitor once the acquire lands). The GIL happens to serialize that + /// window today; the pairing must not depend on it. + cursor_ref: bool, controls: Option>, handle: Option>, cap_thread_id: Option, @@ -3477,6 +5166,8 @@ struct ScState { /// self-joining. deliver_handle: Option>, deliver_thread_id: Option, + /// The Wayland display id this instance's capture is bound to (backend == 2). + wl_display: u32, } /// Unified capture handle exposed to Python. Drives the X11 capture directly or delegates to the @@ -3499,7 +5190,7 @@ impl ScreenCapture { /// GIL across the joins would deadlock. A re-entrant stop arriving on the capture, encode, or /// deliver thread cannot join itself, so it detaches and lets the threads exit on the stop flag. fn stop_internal(&self, py: Python<'_>) -> PyResult<()> { - let (handle, deliver_handle, same_thread, backend, controls) = { + let (handle, deliver_handle, same_thread, backend, controls, wl_display, cursor_ref) = { let mut st = self.inner.lock().unwrap(); if let Some(c) = &st.controls { c.stop.store(true, Ordering::Relaxed); @@ -3519,27 +5210,37 @@ impl ScreenCapture { let handle = st.handle.take(); let deliver_handle = st.deliver_handle.take(); let backend = st.backend; + let cursor_ref = std::mem::take(&mut st.cursor_ref); + let wl_display = st.wl_display; st.backend = 0; st.cap_thread_id = None; st.encode_thread_id = None; st.encode_tid_rx = None; st.deliver_thread_id = None; - (handle, deliver_handle, same, backend, controls) + st.wl_display = 0; + (handle, deliver_handle, same, backend, controls, wl_display, cursor_ref) }; if let Some(c) = &controls { live_x11().lock().unwrap().retain(|x| !Arc::ptr_eq(x, c)); } - if backend == 1 { + if cursor_ref { crate::x11::cursor::release(py); } if backend == 2 { - if WAYLAND_OWNER - .compare_exchange(self.id, 0, Ordering::AcqRel, Ordering::Relaxed) - .is_ok() - { + let did = wl_display; + let owned = { + let mut owners = wayland_owners().lock().unwrap(); + if owners.get(&did) == Some(&self.id) { + owners.remove(&did); + true + } else { + false + } + }; + if owned { if let Some(slot) = WAYLAND_BACKEND.get() { if let Some(be) = slot.lock().unwrap().as_ref() { - let _ = be.bind(py).borrow().stop_capture(); + let _ = be.bind(py).borrow().stop_capture(did); } } } @@ -3574,6 +5275,7 @@ impl ScreenCapture { id: NEXT_CAPTURE_ID.fetch_add(1, Ordering::Relaxed), inner: Mutex::new(ScState { backend: 0, + cursor_ref: false, controls: None, handle: None, cap_thread_id: None, @@ -3581,6 +5283,7 @@ impl ScreenCapture { encode_tid_rx: None, deliver_handle: None, deliver_thread_id: None, + wl_display: 0, }), } } @@ -3602,10 +5305,14 @@ impl ScreenCapture { callback: Py, settings: &Bound<'_, PyAny>, ) -> PyResult<()> { + let display_id = read_display_id(settings); let live_wayland_restart = want_wayland(settings) - && self.inner.lock().unwrap().backend == 2 - && WAYLAND_OWNER.load(Ordering::Relaxed) == self.id - && WAYLAND_CAPTURE_ALIVE.load(Ordering::Acquire); + && { + let st = self.inner.lock().unwrap(); + st.backend == 2 && st.wl_display == display_id + } + && wayland_owners().lock().unwrap().get(&display_id) == Some(&self.id) + && wayland_alive().lock().unwrap().contains(&display_id); if !live_wayland_restart { self.stop_internal(py)?; } @@ -3637,11 +5344,19 @@ impl ScreenCapture { cursor_size, )?; be.bind(py).borrow().start_capture(callback, settings)?; - WAYLAND_OWNER.store(self.id, Ordering::Relaxed); - self.inner.lock().unwrap().backend = 2; + wayland_owners().lock().unwrap().insert(display_id, self.id); + { + let mut st = self.inner.lock().unwrap(); + st.backend = 2; + st.wl_display = display_id; + } return Ok(()); } + // A fresh start proves the interpreter is alive: clear the teardown flag the atexit + // sweep may have set (the Wayland start path already does the same), or the delivery + // thread below would drop every frame. + PY_SHUTDOWN.store(false, Ordering::Relaxed); let mut rs = rs; if rs.encode_node_index < -1 { let auto_gpu = settings @@ -3767,8 +5482,12 @@ impl ScreenCapture { None } }; + // Take the monitor reference BEFORE publishing the capture: a stop that + // observes this capture must always find the reference it is to release. + crate::x11::cursor::acquire(cursor_cap); let mut st = self.inner.lock().unwrap(); st.backend = 1; + st.cursor_ref = true; st.controls = Some(controls); st.handle = Some(handle); st.cap_thread_id = tid; @@ -3777,7 +5496,6 @@ impl ScreenCapture { st.deliver_handle = Some(deliver_handle); st.deliver_thread_id = Some(deliver_thread_id); drop(st); - crate::x11::cursor::acquire(cursor_cap); Ok(()) } @@ -3786,9 +5504,9 @@ impl ScreenCapture { } fn request_idr_frame(&self, py: Python<'_>) -> PyResult<()> { - let (backend, controls) = { + let (backend, controls, did) = { let st = self.inner.lock().unwrap(); - (st.backend, st.controls.clone()) + (st.backend, st.controls.clone(), st.wl_display) }; match backend { 1 => { @@ -3799,7 +5517,7 @@ impl ScreenCapture { 2 => { if let Some(slot) = WAYLAND_BACKEND.get() { if let Some(be) = slot.lock().unwrap().as_ref() { - let _ = be.bind(py).borrow().request_idr_frame(); + let _ = be.bind(py).borrow().request_idr_frame(did); } } } @@ -3813,9 +5531,9 @@ impl ScreenCapture { /// On the X11 path the dirty flag is Release-published after the payload store, so the encode /// thread's Acquire read can never observe the flag set against a stale bitrate. fn update_video_bitrate(&self, py: Python<'_>, kbps: i32) -> PyResult<()> { - let (backend, controls) = { + let (backend, controls, did) = { let st = self.inner.lock().unwrap(); - (st.backend, st.controls.clone()) + (st.backend, st.controls.clone(), st.wl_display) }; match backend { 1 => { @@ -3824,16 +5542,16 @@ impl ScreenCapture { c.rate_dirty.store(true, Ordering::Release); } } - 2 => wayland_update_rate(py, Some(kbps), None, None), + 2 => wayland_update_rate(py, did, Some(kbps), None, None), _ => {} } Ok(()) } fn update_framerate(&self, py: Python<'_>, fps: f64) -> PyResult<()> { - let (backend, controls) = { + let (backend, controls, did) = { let st = self.inner.lock().unwrap(); - (st.backend, st.controls.clone()) + (st.backend, st.controls.clone(), st.wl_display) }; match backend { 1 => { @@ -3842,7 +5560,7 @@ impl ScreenCapture { c.rate_dirty.store(true, Ordering::Release); } } - 2 => wayland_update_rate(py, None, None, Some(fps)), + 2 => wayland_update_rate(py, did, None, None, Some(fps)), _ => {} } Ok(()) @@ -3850,9 +5568,9 @@ impl ScreenCapture { /// Live CBR VBV change, as a multiple of one frame's bit budget (<= 0 = policy default). fn update_vbv_multiplier(&self, py: Python<'_>, multiplier: f64) -> PyResult<()> { - let (backend, controls) = { + let (backend, controls, did) = { let st = self.inner.lock().unwrap(); - (st.backend, st.controls.clone()) + (st.backend, st.controls.clone(), st.wl_display) }; match backend { 1 => { @@ -3862,7 +5580,7 @@ impl ScreenCapture { c.rate_dirty.store(true, Ordering::Release); } } - 2 => wayland_update_rate(py, None, Some(multiplier), None), + 2 => wayland_update_rate(py, did, None, Some(multiplier), None), _ => {} } Ok(()) @@ -3874,9 +5592,9 @@ impl ScreenCapture { fn update_tunables(&self, py: Python<'_>, settings: &Bound<'_, PyAny>) -> PyResult<()> { let rs = extract_settings(settings)?; let t = LiveTunables::from_settings(&rs); - let (backend, controls) = { + let (backend, controls, did) = { let st = self.inner.lock().unwrap(); - (st.backend, st.controls.clone()) + (st.backend, st.controls.clone(), st.wl_display) }; match backend { 1 => { @@ -3887,7 +5605,7 @@ impl ScreenCapture { } crate::x11::cursor::set_size_cap(rs.cursor_size_cap); } - 2 => wayland_update_tunables(py, t), + 2 => wayland_update_tunables(py, did, t), _ => {} } Ok(()) @@ -3924,8 +5642,8 @@ impl ScreenCapture { .as_ref() .map(|c| !c.stop.load(Ordering::Relaxed)) .unwrap_or(false), - 2 => WAYLAND_OWNER.load(Ordering::Relaxed) == self.id - && WAYLAND_CAPTURE_ALIVE.load(Ordering::Acquire), + 2 => wayland_owners().lock().unwrap().get(&st.wl_display) == Some(&self.id) + && wayland_alive().lock().unwrap().contains(&st.wl_display), _ => false, } } @@ -3933,9 +5651,102 @@ impl ScreenCapture { fn inject_key(&self, py: Python<'_>, scancode: u32, state: u32) -> PyResult<()> { wayland_backend_running(py).map_or(Ok(()), |be| be.bind(py).borrow().inject_key(scancode, state)) } + /// Inject an ordered run of `(keycode, state)` events in one message. + fn inject_keys(&self, py: Python<'_>, events: Vec<(u32, u32)>) -> PyResult<()> { + wayland_backend_running(py).map_or(Ok(()), |be| be.bind(py).borrow().inject_keys(events)) + } fn set_keymap_string(&self, py: Python<'_>, text: String) -> PyResult<()> { wayland_backend_running(py).map_or(Ok(()), |be| be.bind(py).borrow().set_keymap_string(text)) } + /// Bind explicit `(keycode, keysym)` pairs onto the current base keymap in one swap; + /// false when no backend runs or no base keymap is installed. See the backend method + /// for the split of responsibility. + fn set_keymap_overlay(&self, py: Python<'_>, binds: Vec<(u32, u32)>) -> PyResult<()> { + wayland_backend_running(py) + .map_or(Ok(()), |be| be.bind(py).borrow().set_keymap_overlay(binds)) + } + /// Compositor apps run under (a nested labwc/kwin session pixelflux captures), + /// the target for Computer-Use text injection; selkies resolves it and hands it + /// over. Empty clears it. Stored process-wide since the CU server is per-process. + fn set_app_wayland_display(&self, display: String) { + crate::computer_use::set_app_wayland_display( + if display.is_empty() { None } else { Some(display) }, + ); + } + /// Type `text` through `display`'s zwp_virtual_keyboard_manager_v1 as a one-shot + /// client: selkies' text-injection path, targeting whichever compositor the apps + /// live under (the nested session's, or pixelflux's own in a direct session). + /// Blocking; releases the GIL for the duration. Raises + /// [`VirtualKeyboardUnavailable`] when the compositor lacks the protocol. + fn type_text_wayland(&self, py: Python<'_>, display: String, text: String) -> PyResult<()> { + py.detach(move || { + let path = crate::wayland::wlclient::socket_path(&display) + .ok_or_else(|| "XDG_RUNTIME_DIR is unset".to_string())?; + crate::wayland::vkclient::type_text_to(&path, &text) + }) + .map_err(|e: String| { + if e.contains("zwp_virtual_keyboard_manager_v1") { + VirtualKeyboardUnavailable::new_err(e) + } else { + pyo3::exceptions::PyRuntimeError::new_err(e) + } + }) + } + /// Mimes the app compositor's current selection offers (empty = nothing copied). + fn clipboard_types_app(&self, py: Python<'_>, display: String) -> PyResult> { + py.detach(move || { + crate::wayland::dcclient::list_types(&app_socket_path(&display)?) + }) + .map_err(pyo3::exceptions::PyRuntimeError::new_err) + } + /// The app compositor selection's payload for `mime`, or None when nothing is + /// copied or the selection does not offer that mime. + fn clipboard_read_app( + &self, + py: Python<'_>, + display: String, + mime: String, + ) -> PyResult>> { + let data = py + .detach(move || crate::wayland::dcclient::read(&app_socket_path(&display)?, &mime)) + .map_err(pyo3::exceptions::PyRuntimeError::new_err)?; + Ok(data.map(|d| pyo3::types::PyBytes::new(py, &d).unbind())) + } + /// Take the app compositor's selection, serving `entries` (mime, bytes) to + /// every paster from a background thread until another client copies. + fn clipboard_write_app( + &self, + py: Python<'_>, + display: String, + entries: Vec<(String, Vec)>, + ) -> PyResult<()> { + py.detach(move || crate::wayland::dcclient::write(&app_socket_path(&display)?, entries)) + .map_err(pyo3::exceptions::PyRuntimeError::new_err) + } + /// Drop the app compositor's selection. + fn clipboard_clear_app(&self, py: Python<'_>, display: String) -> PyResult<()> { + py.detach(move || crate::wayland::dcclient::clear(&app_socket_path(&display)?)) + .map_err(pyo3::exceptions::PyRuntimeError::new_err) + } + /// Invoke `callback(mimes: list[str])` from a background thread on every + /// selection change in the app compositor (including the one current at call + /// time). A second watch for the same display replaces the first. + fn clipboard_watch_app( + &self, + py: Python<'_>, + display: String, + callback: Py, + ) -> PyResult<()> { + py.detach(move || crate::wayland::dcclient::watch(&app_socket_path(&display)?, callback)) + .map_err(pyo3::exceptions::PyRuntimeError::new_err) + } + /// Stop the selection watch for `display` (no-op without one). + fn clipboard_unwatch_app(&self, py: Python<'_>, display: String) { + let _ = py.detach(move || { + crate::wayland::dcclient::unwatch(&app_socket_path(&display)?); + Ok::<(), String>(()) + }); + } fn inject_mouse_move(&self, py: Python<'_>, x: f64, y: f64) -> PyResult<()> { wayland_backend_running(py).map_or(Ok(()), |be| be.bind(py).borrow().inject_mouse_move(x, y)) } @@ -4004,6 +5815,88 @@ impl ScreenCapture { )), } } + /// Set the seat's BASE xkb layout from RMLVO names; false when no backend runs or the + /// layout fails to compile. + #[pyo3(signature = (layout, variant = String::new(), options = String::new(), model = String::new(), rules = String::new()))] + fn set_xkb_layout( + &self, + py: Python<'_>, + layout: String, + variant: String, + options: String, + model: String, + rules: String, + ) -> PyResult { + wayland_backend_running(py).map_or(Ok(false), |be| { + be.bind(py).borrow().set_xkb_layout(py, layout, variant, options, model, rules) + }) + } + /// Seat keyboard readback: `(pressed_keycodes, modifier_mask)`; empty when no backend runs. + fn get_keyboard_state(&self, py: Python<'_>) -> PyResult<(Vec, u32)> { + wayland_backend_running(py) + .map_or(Ok((Vec::new(), 0)), |be| be.bind(py).borrow().get_keyboard_state(py)) + } + /// The capture geometry actually live on the given display `(width, height, scale)`; + /// `(0, 0, 0.0)` when no backend runs. + #[pyo3(signature = (display_id = 0))] + fn get_realized_geometry(&self, py: Python<'_>, display_id: u32) -> PyResult<(i32, i32, f64)> { + wayland_backend_running(py) + .map_or(Ok((0, 0, 0.0)), |be| be.bind(py).borrow().get_realized_geometry(py, display_id)) + } + /// Create an additional Wayland output (see `WaylandBackend.create_output`); false when + /// no backend runs. + // The parameter list is the Python signature; grouping it would change the ABI. + #[allow(clippy::too_many_arguments)] + #[pyo3(signature = (id, width, height, x = 0, y = 0, scale = 1.0))] + fn create_output( + &self, + py: Python<'_>, + id: u32, + width: i32, + height: i32, + x: i32, + y: i32, + scale: f64, + ) -> PyResult { + wayland_backend_running(py).map_or(Ok(false), |be| { + be.bind(py).borrow().create_output(py, id, width, height, x, y, scale) + }) + } + /// Destroy a secondary Wayland output; false when no backend runs. + fn destroy_output(&self, py: Python<'_>, id: u32) -> PyResult { + wayland_backend_running(py) + .map_or(Ok(false), |be| be.bind(py).borrow().destroy_output(py, id)) + } + /// Move a Wayland output (the primary included) to layout offset `(x, y)`; false when + /// no backend runs or the id is unknown. + fn reposition_output(&self, py: Python<'_>, id: u32, x: i32, y: i32) -> PyResult { + wayland_backend_running(py) + .map_or(Ok(false), |be| be.bind(py).borrow().reposition_output(py, id, x, y)) + } + /// Recreate the Wayland cursor theme at `size` pixels (named-cursor callbacks and the + /// burned-in overlay); false when no backend runs or the size is non-positive. + fn set_cursor_size(&self, py: Python<'_>, size: i32) -> PyResult { + wayland_backend_running(py) + .map_or(Ok(false), |be| be.bind(py).borrow().set_cursor_size(py, size)) + } + /// Every live Wayland output as `(id, x, y, width, height, scale, capturing)`; empty + /// when no backend runs. + fn list_outputs(&self, py: Python<'_>) -> PyResult> { + wayland_backend_running(py) + .map_or(Ok(Vec::new()), |be| be.bind(py).borrow().list_outputs(py)) + } + /// Move a window onto an output (fullscreened there); false when no backend runs. + fn move_window_to_output(&self, py: Python<'_>, window_id: u32, output_id: u32) -> PyResult { + wayland_backend_running(py).map_or(Ok(false), |be| { + be.bind(py).borrow().move_window_to_output(py, window_id, output_id) + }) + } + /// Every mapped window as `(window_id, title, app_id, output_id, waiting)`; empty when + /// no backend runs. + fn list_windows(&self, py: Python<'_>) -> PyResult> { + wayland_backend_running(py) + .map_or(Ok(Vec::new()), |be| be.bind(py).borrow().list_windows(py)) + } } /// Best-effort teardown: flag the capture thread to exit without joining. @@ -4013,11 +5906,123 @@ impl ScreenCapture { /// atexit sweep. impl Drop for ScreenCapture { fn drop(&mut self) { - if let Ok(st) = self.inner.lock() { + if let Ok(mut st) = self.inner.lock() { if let Some(c) = &st.controls { c.stop.store(true, Ordering::Relaxed); } + // A Wayland capture is owned by this instance: release it, or a GC'd handle + // would leave the compositor encoding and delivering forever. + if st.backend == 2 && !crate::PY_SHUTDOWN.load(Ordering::Relaxed) { + let did = st.wl_display; + let owned = { + let mut owners = wayland_owners().lock().unwrap(); + if owners.get(&did) == Some(&self.id) { + owners.remove(&did); + true + } else { + false + } + }; + if owned { + if let Some(slot) = WAYLAND_BACKEND.get() { + if let Some(be) = slot.lock().unwrap().as_ref() { + Python::attach(|py| { + let _ = be.bind(py).borrow().stop_capture(did); + }); + } + } + } + } + // Pair the cursor-monitor acquire from start_capture: a dropped + // handle that never got stop_capture would pin the refcount and its + // monitor thread forever. + if std::mem::take(&mut st.cursor_ref) && !crate::PY_SHUTDOWN.load(Ordering::Relaxed) { + Python::attach(crate::x11::cursor::release); + } + } + } +} + +/// Build a Python dict from a recorder status snapshot (one shape for status and stop). +fn recording_status_dict(py: Python<'_>, s: &crate::recorder::RecordingStatus) -> PyResult> { + let d = pyo3::types::PyDict::new(py); + d.set_item("active", s.active)?; + d.set_item("path", &s.path)?; + d.set_item("backend", s.backend)?; + d.set_item("mode", s.mode)?; + d.set_item("frames", s.frames)?; + d.set_item("sync_frames", s.sync_frames)?; + d.set_item("dropped", s.dropped)?; + d.set_item("skipped_non_h264", s.skipped_non_h264)?; + d.set_item("bytes", s.bytes)?; + d.set_item("duration_s", s.duration_s)?; + d.set_item("width", s.width)?; + d.set_item("height", s.height)?; + d.set_item("error", s.error.as_deref())?; + Ok(d.into_any().unbind()) +} + +/// Start the built-in MP4 recorder. Works with no capture and no client running: the +/// recorder owns an independent capture (X11 root, or a Wayland output of the in-process +/// compositor) and taps a live streaming session instead of restarting it. `settings` is an +/// optional `CaptureSettings` for a recorder-owned capture (H.264 only; `display_id` +/// selects the Wayland output); when omitted, `PIXELFLUX_RECORD_*` environment variables +/// and full-screen defaults apply. +#[pyfunction] +#[pyo3(signature = (path, settings = None))] +fn start_recording( + py: Python<'_>, + path: String, + settings: Option<&Bound<'_, PyAny>>, +) -> PyResult> { + let mut opts = crate::recorder::RecordOptions::from_env(path); + if let Some(s) = settings { + let rs = extract_settings(s)?; + if rs.output_mode != 1 { + return Err(PyErr::new::( + "recording requires H.264 capture settings (output_mode=1); JPEG cannot be recorded", + )); } + opts.display_id = read_display_id(s); + if let Some(explicit) = s + .getattr("use_wayland") + .ok() + .and_then(|v| v.extract::().ok()) + { + opts.backend = Some(if explicit { + crate::recorder::PreferredBackend::Wayland + } else { + crate::recorder::PreferredBackend::X11 + }); + } + // Explicit settings are authoritative over the env knobs they subsume. + opts.fps = 0.0; + opts.bitrate_kbps = 0; + opts.capture = Some(rs); + } + let status = py + .detach(|| crate::recorder::start(opts)) + .map_err(PyErr::new::)?; + recording_status_dict(py, &status) +} + +/// Stop the active recording, finalize the MP4, and return the final status dict. Raises +/// when no recording is active or nothing recordable was captured. +#[pyfunction] +fn stop_recording(py: Python<'_>) -> PyResult> { + let status = py + .detach(crate::recorder::stop) + .map_err(PyErr::new::)?; + recording_status_dict(py, &status) +} + +/// Status of the live recording (or the last finished one); `None` if this process has +/// never recorded. +#[pyfunction] +fn recording_status(py: Python<'_>) -> PyResult> { + match crate::recorder::status() { + Some(s) => recording_status_dict(py, &s), + None => Ok(py.None()), } } @@ -4025,7 +6030,9 @@ impl Drop for ScreenCapture { /// connect (sets WAYLAND_DISPLAY for children of this process). Idempotent; the running /// backend keeps its node/dimensions on later calls. `render_node` is an explicit /// /dev/dri/renderD* path; `auto_gpu` is a truthy string or vendor/driver token; empty -/// values mean software rendering. +/// values mean software rendering. Returns the compositor's actual socket name (the +/// auto-picked `wayland-N`, which need not match any configured index); empty string only +/// if the socket did not come up in time. #[pyfunction] #[pyo3(signature = (width = 0, height = 0, render_node = String::new(), auto_gpu = String::new(), cursor_size = -1))] fn ensure_wayland_display( @@ -4035,9 +6042,19 @@ fn ensure_wayland_display( render_node: String, auto_gpu: String, cursor_size: i32, -) -> PyResult<()> { - ensure_wayland_backend(py, width, height, render_node, auto_gpu, String::new(), cursor_size) - .map(|_| ()) +) -> PyResult { + ensure_wayland_backend(py, width, height, render_node, auto_gpu, String::new(), cursor_size)?; + Ok(py + .detach(|| wait_socket_name(Duration::from_secs(5))) + .unwrap_or_default()) +} + +/// The running compositor's Wayland socket name (e.g. "wayland-1"), or None when no +/// compositor thread has been started (this never creates one). +#[pyfunction] +fn get_wayland_display_name(py: Python<'_>) -> Option { + wayland_backend_running(py)?; + py.detach(|| wait_socket_name(Duration::from_secs(2))) } /// Stop every live capture (registered with atexit) before interpreter finalization. @@ -4051,8 +6068,12 @@ fn ensure_wayland_display( #[pyfunction] fn _stop_all_captures(py: Python<'_>) { PY_SHUTDOWN.store(true, Ordering::Relaxed); + // Finalize any active recording first so its last buffered MP4 sample is flushed and + // its own capture (if any) is stopped through the normal path. + py.detach(crate::recorder::finalize_on_exit); *PENDING_CURSOR_CALLBACK.lock().unwrap() = None; crate::x11::cursor::shutdown(); + crate::wayland::dcclient::unwatch_all(); if let Some(slot) = LIVE_X11.get() { for c in slot.lock().unwrap().iter() { c.stop.store(true, Ordering::Relaxed); @@ -4060,16 +6081,52 @@ fn _stop_all_captures(py: Python<'_>) { } if let Some(slot) = WAYLAND_BACKEND.get() { if let Some(be) = slot.lock().unwrap().as_ref() { - let _ = be.bind(py).borrow().stop_capture(); + let be = be.bind(py).borrow(); + let mut displays: Vec = + wayland_alive().lock().unwrap().iter().copied().collect(); + if !displays.contains(&0) { + displays.push(0); + } + for did in displays { + let _ = be.stop_capture(did); + } + // Wait (bounded) until the calloop finished processing the stops: dropping a + // hardware encoder session (NVENC/CUDA) mid-process-exit segfaults, so the + // interpreter must not finalize while that teardown is still running. + let (ack_tx, ack_rx) = std::sync::mpsc::channel::<()>(); + if be.send(ThreadCommand::Barrier { reply: ack_tx }).is_ok() { + let _ = py.detach(move || ack_rx.recv_timeout(Duration::from_secs(2))); + } } } - WAYLAND_OWNER.store(0, Ordering::Relaxed); - WAYLAND_CAPTURE_ALIVE.store(false, Ordering::Relaxed); + wayland_owners().lock().unwrap().clear(); + wayland_alive().lock().unwrap().clear(); py.detach(|| std::thread::sleep(Duration::from_millis(50))); } /// The `pixelflux` Python module: registers the exported classes and functions, and hooks /// `_stop_all_captures` into `atexit` so every live capture is stopped before interpreter shutdown. +/// Socket path for a Wayland display name, with the ABI methods' error shape. +fn app_socket_path(display: &str) -> Result { + crate::wayland::wlclient::socket_path(display) + .ok_or_else(|| "XDG_RUNTIME_DIR is unset".to_string()) +} + +pyo3::create_exception!( + pixelflux, + VirtualKeyboardUnavailable, + pyo3::exceptions::PyRuntimeError, + "The target compositor does not advertise zwp_virtual_keyboard_manager_v1." +); + +/// Start the Computer-Use HTTP server: a bare port listens on all interfaces, +/// `host:port` scopes it. Idempotent; the PIXELFLUX_CU env var remains the +/// standalone fallback. +#[pyfunction] +fn start_computer_use(bind: String) { + crate::computer_use::start_cu_server(&bind); +} + #[pymodule] fn pixelflux(m: &Bound<'_, PyModule>) -> PyResult<()> { m.add_class::()?; @@ -4080,15 +6137,86 @@ fn pixelflux(m: &Bound<'_, PyModule>) -> PyResult<()> { m.add("X11_CURSOR_CALLBACK", true)?; m.add_function(wrap_pyfunction!(stripe_frame_from_buffer, m)?)?; m.add_function(wrap_pyfunction!(ensure_wayland_display, m)?)?; + m.add_function(wrap_pyfunction!(get_wayland_display_name, m)?)?; + m.add_function(wrap_pyfunction!(start_recording, m)?)?; + m.add_function(wrap_pyfunction!(stop_recording, m)?)?; + m.add_function(wrap_pyfunction!(recording_status, m)?)?; + m.add_function(wrap_pyfunction!(start_computer_use, m)?)?; + m.add( + "VirtualKeyboardUnavailable", + m.py().get_type::(), + )?; m.add_function(wrap_pyfunction!(_stop_all_captures, m)?)?; if let Ok(atexit) = m.py().import("atexit") { let _ = atexit.call_method1("register", (m.getattr("_stop_all_captures")?,)); } + // Standalone CU entry point: with PIXELFLUX_CU set the server binds at import, serving + // X11 (via DISPLAY) until a Wayland compositor registers itself as the backend. + crate::computer_use::spawn_cu_from_env(); + // PIXELFLUX_RECORD=: start recording from process start (X11 immediately, or as + // soon as the in-process Wayland compositor comes up). + crate::recorder::autostart_from_env(); Ok(()) } +#[cfg(test)] +mod output_overlap_tests { + //! Invariants of the output-placement intersection predicate: strict interior + //! intersection (touching edges never overlap), containment and identity overlap, + //! empty/negative rectangles never overlap, and extreme coordinates do not wrap. + use super::rects_overlap; + + #[test] + fn disjoint_rects_do_not_overlap() { + assert!(!rects_overlap((0, 0, 100, 100), (200, 0, 100, 100))); + assert!(!rects_overlap((0, 0, 100, 100), (0, 200, 100, 100))); + } + + #[test] + fn touching_edges_do_not_overlap() { + // Right edge of a meets left edge of b, and bottom meets top. + assert!(!rects_overlap((0, 0, 100, 100), (100, 0, 100, 100))); + assert!(!rects_overlap((0, 0, 100, 100), (0, 100, 100, 100))); + // Corner touch only. + assert!(!rects_overlap((0, 0, 100, 100), (100, 100, 50, 50))); + } + + #[test] + fn one_pixel_intrusion_overlaps() { + assert!(rects_overlap((0, 0, 100, 100), (99, 0, 100, 100))); + assert!(rects_overlap((0, 0, 100, 100), (0, 99, 100, 100))); + } + + #[test] + fn containment_and_identity_overlap() { + assert!(rects_overlap((0, 0, 100, 100), (25, 25, 10, 10))); + assert!(rects_overlap((25, 25, 10, 10), (0, 0, 100, 100))); + assert!(rects_overlap((5, 5, 50, 50), (5, 5, 50, 50))); + } + + #[test] + fn empty_or_negative_rects_never_overlap() { + assert!(!rects_overlap((10, 10, 0, 50), (0, 0, 100, 100))); + assert!(!rects_overlap((10, 10, 50, 0), (0, 0, 100, 100))); + assert!(!rects_overlap((10, 10, -5, 5), (0, 0, 100, 100))); + assert!(!rects_overlap((0, 0, 100, 100), (10, 10, 0, 0))); + } + + #[test] + fn negative_origins_overlap_correctly() { + assert!(rects_overlap((-50, -50, 100, 100), (0, 0, 100, 100))); + assert!(!rects_overlap((-100, -100, 100, 100), (0, 0, 100, 100))); + } + + #[test] + fn extreme_coordinates_do_not_wrap() { + assert!(!rects_overlap((i32::MAX - 10, 0, 10, 10), (i32::MIN, 0, 10, 10))); + assert!(rects_overlap((i32::MAX - 10, 0, 10, 10), (i32::MAX - 5, 0, 10, 10))); + } +} + #[cfg(test)] mod wl_frame_pool_tests { //! Invariants under test: try_begin is non-blocking and hands out a buffer ONLY while the diff --git a/pixelflux/src/pipeline.rs b/pixelflux/src/pipeline.rs index 6c504c5..b682fbc 100644 --- a/pixelflux/src/pipeline.rs +++ b/pixelflux/src/pipeline.rs @@ -14,7 +14,6 @@ use crate::encoders::nvenc::NvencEncoder; use crate::encoders::oh264::Openh264Encoder; use crate::encoders::software::{encode_cpu, EncodedStripe, StripeState}; use crate::encoders::vaapi::VaapiEncoder; -use crate::recording_sink::RecordingSink; use crate::RustCaptureSettings; use std::sync::Arc; @@ -156,6 +155,31 @@ enum X11Encoder { Openh264(Openh264Encoder), } +/// Software H.264 encoder used when hardware encoders are unavailable or the CPU is forced. +/// With GPL components this is the striped x264 path (its state lives per-stripe inside +/// `encode_cpu`, so no persistent object exists here); without them (the crate built with the +/// `gpl` feature disabled) the BSD-licensed OpenH264 full-frame encoder is substituted instead. +#[cfg(feature = "gpl")] +fn software_h264_encoder(settings: &RustCaptureSettings) -> X11Encoder { + let _ = settings; + X11Encoder::None +} + +#[cfg(not(feature = "gpl"))] +fn software_h264_encoder(settings: &RustCaptureSettings) -> X11Encoder { + if settings.output_mode != 1 { + return X11Encoder::None; + } + println!("[x11] pixelflux was built without GPL components (libx264 disabled); substituting the BSD-licensed OpenH264 software encoder."); + match Openh264Encoder::new(settings) { + Some(e) => X11Encoder::Openh264(e), + None => { + eprintln!("[x11] OpenH264 init failed; no software H.264 encoder is available in this GPL-free build."); + X11Encoder::None + } + } +} + /// Everything the X11 host-ARGB path has to remember between frames. /// /// Unlike the Wayland backend, X11 capture has no compositor to report what changed, so this @@ -164,7 +188,8 @@ enum X11Encoder { /// content frame-to-frame. Full-frame H.264 runs through `decide_hw_fullframe`; striped JPEG/x264 /// runs through `encode_cpu` with `hash_damage=true`. /// -/// Recording sink fan-out is handled at the delivery layer. +/// Recording fan-out (socket sink and MP4 recorder) is handled at the delivery layer; a +/// consumer needing a keyframe goes through [`X11Pipeline::request_idr`] like everyone else. pub struct X11Pipeline { settings: RustCaptureSettings, stripes: Vec, @@ -172,7 +197,6 @@ pub struct X11Pipeline { hw_state: StripeState, frame_counter: u16, pending_force_idr: bool, - recording_sink: Option>, } impl X11Pipeline { @@ -182,22 +206,26 @@ impl X11Pipeline { /// /// * `settings` - Capture configuration. The `output_mode`, `use_openh264`, `use_cpu`, /// `encode_node_index`, and `video_fullcolor` fields drive encoder selection. - /// * `recording_sink` - Optional Unix-socket H.264 fan-out. Owned by the caller; not - /// rebound on auto-adjust resizes to avoid tearing down the socket listener. /// /// # Encoder selection /// /// 1. **OpenH264** — when `use_openh264` is set (explicit opt-in, full-frame software). /// 2. **NVENC** — on an NVIDIA driver (or no detectable GPU, since the attempt is cheap). - /// 3. **VA-API** — on any other GPU driver (except 4:4:4 full-color, which falls to x264). - /// 4. **Software** (`X11Encoder::None`) — on any hardware init failure or explicit software. - pub fn new(settings: RustCaptureSettings, recording_sink: Option>) -> Self { + /// 3. **VA-API** — on any other GPU driver (except 4:4:4 full-color, which falls back to the + /// software path). + /// 4. **Software** — `software_h264_encoder`: `X11Encoder::None`, meaning the striped x264 + /// inside `encode_cpu`, in a GPL build; the full-frame OpenH264 encoder when the `gpl` + /// feature is disabled. + pub fn new(settings: RustCaptureSettings) -> Self { let hw = if settings.output_mode == 1 && settings.use_openh264 { println!("[x11] OpenH264 software encoder selected."); match Openh264Encoder::new(&settings) { Some(e) => X11Encoder::Openh264(e), None => { + #[cfg(feature = "gpl")] eprintln!("[x11] OpenH264 init failed; falling back to software x264"); + #[cfg(not(feature = "gpl"))] + eprintln!("[x11] OpenH264 init failed; no software H.264 encoder is available in this GPL-free build."); X11Encoder::None } } @@ -208,7 +236,7 @@ impl X11Pipeline { if !encode_driver.is_empty() && !encode_driver.contains("nvidia") { if settings.video_fullcolor { println!("[x11] 4:4:4 full-color requested. VAAPI does not support this profile reliably. Falling back to CPU."); - X11Encoder::None + software_h264_encoder(&settings) } else { println!("[x11] Initializing Unified VAAPI Encoder..."); match VaapiEncoder::new_host(&settings) { @@ -218,7 +246,7 @@ impl X11Pipeline { } Err(err) => { eprintln!("[x11] Failed to init VAAPI: {err}. Falling back to CPU."); - X11Encoder::None + software_h264_encoder(&settings) } } } @@ -231,7 +259,7 @@ impl X11Pipeline { } Err(err) => { eprintln!("[x11] Failed to init NVENC: {err}. Falling back to CPU."); - X11Encoder::None + software_h264_encoder(&settings) } } } @@ -239,7 +267,7 @@ impl X11Pipeline { if settings.output_mode == 1 { println!("[x11] No GPU Encoder available -> Using CPU Software Encoding."); } - X11Encoder::None + software_h264_encoder(&settings) }; Self { settings, @@ -248,7 +276,6 @@ impl X11Pipeline { hw_state: StripeState::default(), frame_counter: 0, pending_force_idr: false, - recording_sink, } } @@ -360,8 +387,7 @@ impl X11Pipeline { ); if d.send { let fc = self.frame_counter as u64; - let force_idr = d.force_idr - || self.recording_sink.as_ref().map(|s| s.should_force_idr()).unwrap_or(false); + let force_idr = d.force_idr; let res = match &mut self.hw { X11Encoder::Nvenc(enc) => { enc.encode_cpu_argb(argb, stride, fc, d.target_qp, force_idr) @@ -377,7 +403,7 @@ impl X11Pipeline { match res { Ok(data) if !data.is_empty() => { vec![EncodedStripe { - data, + data: Arc::new(data), data_type: 2, stripe_y_start: 0, stripe_height: height, @@ -416,7 +442,10 @@ impl X11Pipeline { ) }; - self.pending_force_idr = false; + // An unserved request stays armed: on an infinite GOP an IDR lost to an encode + // error or skip would never self-heal, leaving a joining consumer with an + // undecodable stream. + self.pending_force_idr = requested && out.is_empty(); self.frame_counter = self.frame_counter.wrapping_add(1); out } @@ -441,7 +470,7 @@ mod tests { use_paint_over_quality: false, ..Default::default() }; - let mut p = X11Pipeline::new(s, None); + let mut p = X11Pipeline::new(s); let stride = 128 * 4; let frame_a = vec![10u8; stride * 128]; let mut frame_b = frame_a.clone(); @@ -474,7 +503,7 @@ mod tests { target_fps: 60.0, ..Default::default() }; - let mut p = X11Pipeline::new(s, None); + let mut p = X11Pipeline::new(s); let stride = 128 * 4; let frame = vec![10u8; stride * 128]; assert!(!p.process(&frame, stride).is_empty(), "first frame emits"); diff --git a/pixelflux/src/recorder/mod.rs b/pixelflux/src/recorder/mod.rs new file mode 100644 index 0000000..8b22581 --- /dev/null +++ b/pixelflux/src/recorder/mod.rs @@ -0,0 +1,747 @@ +/* + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at https://mozilla.org/MPL/2.0/. + */ + +//! Built-in MP4 recorder: an independent pipeline that captures the desktop to a +//! ready-to-play fragmented MP4 without needing any connected client. +//! +//! One implementation serves three control surfaces — the Python module functions +//! (`start_recording` / `stop_recording` / `recording_status`), the `PIXELFLUX_RECORD*` +//! environment variables (record from process start), and the `record_start` / +//! `record_stop` / `record_status` endpoints on the Computer Use HTTP server — so their +//! behavior cannot drift. +//! +//! The recorder is a consumer of the existing capture machinery, never a hook inside an +//! encoder: +//! +//! * **X11**: it always runs its own [`crate::x11::run_capture`] instance against the root +//! window (a second capture of the same root is independent of any streaming session), with +//! the encoded frames delivered straight into the recorder's queue. +//! * **Wayland**: the compositor lives in this process and allows one capture per output, so +//! the recorder attaches at the delivery layer. When the output is already being captured +//! for a streaming client it taps that stream (and paces recovery keyframes through the +//! standard `RequestIdr` command); when the output is idle it starts its own capture with no +//! Python callback and taps the identical delivery point. +//! +//! Frames cross into the recorder through one bounded queue, mirroring the Unix-socket sink's +//! isolation contract: the delivery thread only clones an `Arc` and `try_send`s, an overflowing +//! queue drops frames (never blocks the pipeline), and all muxing happens on the recorder's own +//! writer thread. Timestamps are wall-clock, so the damage-driven, variable-rate frame flow +//! lands at its true times in the MP4. + +pub mod mp4; + +use std::sync::atomic::{AtomicBool, AtomicU32, AtomicU64, Ordering}; +use std::sync::{mpsc, Arc, Mutex}; +use std::thread; +use std::time::{Duration, Instant}; + +use crossbeam_channel::{bounded, Receiver, Sender, TrySendError}; + +use crate::encoders::software::EncodedStripe; +use crate::RustCaptureSettings; +use crate::ThreadCommand; + +/// Recorder queue bound, matching the socket sink's stalled-consumer policy: a writer that +/// falls this far behind loses frames instead of growing memory or blocking the encoder. +const QUEUE_CAP: usize = 256; + +/// A queued frame: `Arc`-shared encoded payload, byte offset where the Annex-B stream +/// starts (past the wire header when present), and the wall-clock capture time. +type TapFrame = (Arc>, usize, u64); + +/// Which capture system feeds the recording. +#[derive(Clone, Copy, PartialEq, Eq, Debug)] +pub enum PreferredBackend { + X11, + Wayland, +} + +/// Resolved recording parameters. Environment variables provide the defaults; explicit +/// Python/REST arguments override them. +#[derive(Clone)] +pub struct RecordOptions { + pub path: String, + pub display_id: u32, + /// Capture fps cap for a recorder-owned capture (`<= 0` = default 30). An attached + /// recording follows the live session's rate. + pub fps: f64, + /// Bitrate override in kbps for a recorder-owned capture (`<= 0` = settings default). + pub bitrate_kbps: i32, + /// Recovery-keyframe cadence in seconds (`<= 0` = default 2.0). Recorder-owned captures + /// carry it in their settings (scheduled IDRs); attached recordings pace standard + /// request-IDR commands at this interval. + pub keyframe_interval_s: f64, + pub backend: Option, + /// Full capture settings for a recorder-owned capture; `None` derives them (full root / + /// current output geometry, H.264 full-frame). + pub capture: Option, +} + +impl RecordOptions { + /// Options seeded entirely from `PIXELFLUX_RECORD_*` environment variables. + pub fn from_env(path: String) -> Self { + let f = |k: &str| std::env::var(k).ok().and_then(|v| v.parse::().ok()); + let backend = match std::env::var("PIXELFLUX_RECORD_BACKEND").ok().as_deref() { + Some("x11") => Some(PreferredBackend::X11), + Some("wayland") => Some(PreferredBackend::Wayland), + _ => None, + }; + Self { + path, + display_id: f("PIXELFLUX_RECORD_DISPLAY").map(|v| v as u32).unwrap_or(0), + fps: f("PIXELFLUX_RECORD_FPS").unwrap_or(0.0), + bitrate_kbps: f("PIXELFLUX_RECORD_BITRATE").map(|v| v as i32).unwrap_or(0), + keyframe_interval_s: f("PIXELFLUX_RECORD_KEYFRAME_S").unwrap_or(0.0), + backend, + capture: None, + } + } + + fn effective_fps(&self) -> f64 { + if self.fps > 0.0 { self.fps } else { 30.0 } + } + + fn effective_keyframe_s(&self) -> f64 { + if self.keyframe_interval_s > 0.0 { self.keyframe_interval_s } else { 2.0 } + } +} + +/// Live counters shared between the feeding side (delivery threads), the writer thread and +/// the status surfaces. +struct RecShared { + start: Instant, + enqueued: AtomicU64, + dropped: AtomicU64, + skipped_non_h264: AtomicU64, + muxed: AtomicU64, + sync_frames: AtomicU64, + bytes: AtomicU64, + width: AtomicU32, + height: AtomicU32, + last_idr_req_us: AtomicU64, + error: Mutex>, +} + +impl RecShared { + fn new() -> Arc { + Arc::new(Self { + start: Instant::now(), + enqueued: AtomicU64::new(0), + dropped: AtomicU64::new(0), + skipped_non_h264: AtomicU64::new(0), + muxed: AtomicU64::new(0), + sync_frames: AtomicU64::new(0), + bytes: AtomicU64::new(0), + width: AtomicU32::new(0), + height: AtomicU32::new(0), + last_idr_req_us: AtomicU64::new(0), + error: Mutex::new(None), + }) + } + + fn set_error(&self, msg: String) { + let mut g = self.error.lock().unwrap(); + if g.is_none() { + eprintln!("[recorder] {msg}"); + *g = Some(msg); + } + } +} + +/// Point-in-time view of the recorder, identical across the Python, env and REST surfaces. +#[derive(Clone, Debug)] +pub struct RecordingStatus { + pub active: bool, + pub path: String, + pub backend: &'static str, + pub mode: &'static str, + pub frames: u64, + pub sync_frames: u64, + pub dropped: u64, + pub skipped_non_h264: u64, + pub bytes: u64, + pub duration_s: f64, + pub width: u32, + pub height: u32, + pub error: Option, +} + +/// The wayland-side tap handle consulted by [`wayland_tap`]; cloneable so the delivery +/// thread can drop the registry lock before doing any work. +#[derive(Clone)] +struct WlTap { + display_id: u32, + tx: Sender, + shared: Arc, + /// Standard request-IDR path for attached recordings (`None` when the recorder owns the + /// capture and scheduled keyframes ride in its settings). + idr: Option, + keyframe_interval_us: u64, +} + +/// Sends `ThreadCommand::RequestIdr` for one display over the compositor's command channel. +#[derive(Clone)] +struct IdrRequester { + tx: smithay::reexports::calloop::channel::Sender, + display_id: u32, +} + +impl IdrRequester { + fn request(&self) { + let _ = self.tx.send(ThreadCommand::RequestIdr { display_id: self.display_id }); + } +} + +/// How the active recording is fed and what must be torn down on stop. +enum RecordingMode { + /// Recorder-owned X11 capture: its controls and capture thread. + X11Own { + controls: Arc, + join: thread::JoinHandle<()>, + }, + /// Recorder-owned Wayland capture on `display_id` (stopped unless a streaming client has + /// since taken ownership of the display). + WaylandOwn { display_id: u32 }, + /// Tap on a streaming client's live Wayland capture; nothing to stop. + WaylandAttached, +} + +struct ActiveRecording { + path: String, + backend: &'static str, + mode_name: &'static str, + mode: RecordingMode, + tx: Sender, + shared: Arc, + writer: thread::JoinHandle<()>, +} + +static ACTIVE: Mutex> = Mutex::new(None); +static LAST_FINISHED: Mutex> = Mutex::new(None); +/// Fast idle guard for [`wayland_tap`]: the delivery threads pay one relaxed atomic load +/// per frame while no recording is armed. +static WL_TAP_ARMED: AtomicBool = AtomicBool::new(false); +static WL_TAP: Mutex> = Mutex::new(None); + +/// Delivery-layer feed from the Wayland pipelines (readback encode loop and zero-copy +/// tick). Near-zero cost when idle; while recording it clones an `Arc` into the bounded +/// queue and never blocks. +pub(crate) fn wayland_tap(display_id: u32, stripes: &[EncodedStripe]) { + if !WL_TAP_ARMED.load(Ordering::Relaxed) { + return; + } + let Some(tap) = WL_TAP.lock().unwrap().clone() else { return }; + if tap.display_id != display_id { + return; + } + offer_frame(&tap.shared, &tap.tx, stripes); + if let Some(ref idr) = tap.idr { + pace_idr_requests(&tap.shared, idr, tap.keyframe_interval_us); + } +} + +/// Issue a standard request-IDR when the cadence interval has elapsed. Compare-and-swap on +/// the last-request stamp so concurrent delivery threads emit one request per interval. +fn pace_idr_requests(shared: &RecShared, idr: &IdrRequester, interval_us: u64) { + if interval_us == 0 { + return; + } + let now = shared.start.elapsed().as_micros() as u64; + let last = shared.last_idr_req_us.load(Ordering::Relaxed); + if now.saturating_sub(last) >= interval_us + && shared + .last_idr_req_us + .compare_exchange(last, now, Ordering::Relaxed, Ordering::Relaxed) + .is_ok() + { + idr.request(); + } +} + +/// Enqueue one delivered frame for muxing. Only a single full-frame H.264 stripe is +/// recordable; JPEG or striped output is counted and skipped so the surfaces can report a +/// clear "nothing recordable" error instead of writing a corrupt file. +fn offer_frame(shared: &RecShared, tx: &Sender, stripes: &[EncodedStripe]) { + if stripes.is_empty() { + return; + } + if stripes.len() != 1 || stripes[0].data_type != 2 || stripes[0].stripe_y_start != 0 { + shared.skipped_non_h264.fetch_add(1, Ordering::Relaxed); + return; + } + let s = &stripes[0]; + if s.data.is_empty() { + return; + } + let offset = if s.data.len() >= 10 && s.data[0] == 0x04 { 10 } else { 0 }; + if s.data.len() == offset { + return; + } + let pts_us = shared.start.elapsed().as_micros() as u64; + match tx.try_send((s.data.clone(), offset, pts_us)) { + Ok(()) => { + shared.enqueued.fetch_add(1, Ordering::Relaxed); + } + Err(TrySendError::Full(_)) => { + shared.dropped.fetch_add(1, Ordering::Relaxed); + } + Err(TrySendError::Disconnected(_)) => {} + } +} + +/// Writer-thread body: drain the queue, convert Annex-B to AVCC, and mux. Output starts at +/// the first IDR with parameter sets; earlier frames are discarded. Runs until every sender +/// is dropped (stop) or a write error occurs, then finalizes the file and publishes the +/// final status. +fn writer_thread( + rx: Receiver, + file: std::fs::File, + path: String, + backend: &'static str, + mode_name: &'static str, + shared: Arc, +) { + let mut writer = mp4::FragmentWriter::new(std::io::BufWriter::new(file)); + let mut builder = mp4::H264SampleBuilder::new(); + + 'recv: for (buf, offset, pts_us) in rx.iter() { + let Some(sample) = builder.build_sample(&buf[offset..]) else { continue }; + if !writer.init_written() { + if !sample.sync || !builder.have_parameter_sets() { + continue; + } + let Some(cfg) = builder.track_config() else { + shared.set_error("failed to parse H.264 SPS for MP4 init".to_string()); + break 'recv; + }; + shared.width.store(cfg.width, Ordering::Relaxed); + shared.height.store(cfg.height, Ordering::Relaxed); + if let Err(e) = writer.write_init(&cfg) { + shared.set_error(format!("MP4 init write failed: {e}")); + break 'recv; + } + } + if let Err(e) = writer.push_sample(sample.data, sample.sync, pts_us) { + shared.set_error(format!("MP4 write failed: {e}")); + break 'recv; + } + let st = writer.stats(); + shared.muxed.store(st.samples + 1, Ordering::Relaxed); // +1: one sample is buffered + shared.sync_frames.store(st.sync_samples, Ordering::Relaxed); + shared.bytes.store(st.bytes, Ordering::Relaxed); + } + + let final_stats = match writer.finish() { + Ok(st) => st, + Err(e) => { + shared.set_error(format!("MP4 finalize failed: {e}")); + mp4::Mp4Stats::default() + } + }; + if final_stats.samples == 0 { + shared.set_error( + "no recordable H.264 frames received (the session must produce a single \ + full-frame H.264 stream; JPEG and striped modes cannot be recorded)" + .to_string(), + ); + let _ = std::fs::remove_file(&path); + } + let status = RecordingStatus { + active: false, + path, + backend, + mode: mode_name, + frames: final_stats.samples, + sync_frames: final_stats.sync_samples, + dropped: shared.dropped.load(Ordering::Relaxed), + skipped_non_h264: shared.skipped_non_h264.load(Ordering::Relaxed), + bytes: final_stats.bytes, + duration_s: final_stats.duration_us as f64 / 1e6, + width: shared.width.load(Ordering::Relaxed), + height: shared.height.load(Ordering::Relaxed), + error: shared.error.lock().unwrap().clone(), + }; + println!( + "[recorder] finished {}: {} frames ({} sync), {:.2}s, {} bytes", + status.path, status.frames, status.sync_frames, status.duration_s, status.bytes + ); + *LAST_FINISHED.lock().unwrap() = Some(status); +} + +/// Capture settings for a recorder-owned capture: explicit settings when given (validated +/// H.264), otherwise derived defaults, always forced to the one recordable shape (full-frame +/// H.264, no socket rebind, scheduled recovery keyframes). +fn own_capture_settings(opts: &RecordOptions) -> Result { + let mut s = match &opts.capture { + Some(explicit) => { + if explicit.output_mode != 1 { + return Err( + "recording requires H.264 capture settings (output_mode=1); JPEG cannot be recorded" + .to_string(), + ); + } + explicit.clone() + } + None => RustCaptureSettings { + width: 0, + height: 0, + output_mode: 1, + capture_cursor: true, + target_fps: opts.effective_fps(), + ..Default::default() + }, + }; + s.video_fullframe = true; + s.recording_socket = String::new(); + if opts.fps > 0.0 { + s.target_fps = opts.fps; + } + if opts.bitrate_kbps > 0 { + s.video_bitrate_kbps = opts.bitrate_kbps; + } + if s.keyframe_interval_s <= 0.0 { + s.keyframe_interval_s = opts.effective_keyframe_s(); + } + Ok(s) +} + +/// Start a recording. Exactly one may be active per process; returns the initial status. +pub fn start(opts: RecordOptions) -> Result { + let mut guard = ACTIVE.lock().unwrap(); + if guard.is_some() { + return Err("a recording is already active".to_string()); + } + if opts.path.is_empty() { + return Err("recording path is empty".to_string()); + } + if let Some(cap) = &opts.capture { + if cap.output_mode != 1 { + return Err( + "recording requires H.264 capture settings (output_mode=1); JPEG cannot be recorded" + .to_string(), + ); + } + } + + let wl_tx = crate::computer_use::wayland_command_sender(); + let backend = match opts.backend { + Some(PreferredBackend::X11) => PreferredBackend::X11, + Some(PreferredBackend::Wayland) => { + if wl_tx.is_none() { + return Err("no Wayland compositor is running in this process".to_string()); + } + PreferredBackend::Wayland + } + None => { + if wl_tx.is_some() { + PreferredBackend::Wayland + } else if std::env::var("DISPLAY").map(|v| !v.is_empty()).unwrap_or(false) { + PreferredBackend::X11 + } else { + return Err( + "no capture backend available: no in-process Wayland compositor and DISPLAY is unset" + .to_string(), + ); + } + } + }; + + let file = std::fs::File::create(&opts.path) + .map_err(|e| format!("cannot create {}: {e}", opts.path))?; + let shared = RecShared::new(); + let (tx, rx) = bounded::(QUEUE_CAP); + + let (mode, mode_name, backend_name) = match backend { + PreferredBackend::Wayland => { + let cmd_tx = wl_tx.unwrap(); + let display_id = opts.display_id; + let attached = crate::wayland_alive().lock().unwrap().contains(&display_id); + if attached { + *WL_TAP.lock().unwrap() = Some(WlTap { + display_id, + tx: tx.clone(), + shared: shared.clone(), + idr: Some(IdrRequester { tx: cmd_tx.clone(), display_id }), + keyframe_interval_us: (opts.effective_keyframe_s() * 1e6) as u64, + }); + WL_TAP_ARMED.store(true, Ordering::Relaxed); + // The stream is mid-GOP: a standard request-IDR gives the file its first + // decodable frame promptly. + let _ = cmd_tx.send(ThreadCommand::RequestIdr { display_id }); + (RecordingMode::WaylandAttached, "attached", "wayland") + } else { + let mut settings = own_capture_settings(&opts)?; + if settings.width <= 0 || settings.height <= 0 { + let (reply_tx, reply_rx) = mpsc::channel(); + cmd_tx + .send(ThreadCommand::ListOutputs { reply: reply_tx }) + .map_err(|_| "wayland compositor is not accepting commands".to_string())?; + let outputs = reply_rx + .recv_timeout(Duration::from_secs(5)) + .map_err(|_| "wayland compositor did not report its outputs".to_string())?; + let out = outputs + .iter() + .find(|o| o.0 == display_id) + .ok_or_else(|| format!("no wayland output with display id {display_id}"))?; + settings.width = out.3; + settings.height = out.4; + settings.scale = out.5; + } + *WL_TAP.lock().unwrap() = Some(WlTap { + display_id, + tx: tx.clone(), + shared: shared.clone(), + idr: None, + keyframe_interval_us: 0, + }); + WL_TAP_ARMED.store(true, Ordering::Relaxed); + cmd_tx + .send(ThreadCommand::StartCapture { display_id, callback: None, settings }) + .map_err(|_| { + disarm_tap(); + "wayland compositor is not accepting commands".to_string() + })?; + // StartCapture has no reply; liveness shows up in the alive set. + let deadline = Instant::now() + Duration::from_secs(3); + loop { + if crate::wayland_alive().lock().unwrap().contains(&display_id) { + break; + } + if Instant::now() >= deadline { + disarm_tap(); + return Err(format!( + "wayland capture on display {display_id} failed to start" + )); + } + thread::sleep(Duration::from_millis(20)); + } + (RecordingMode::WaylandOwn { display_id }, "own-capture", "wayland") + } + } + PreferredBackend::X11 => { + let settings = own_capture_settings(&opts)?; + let controls = Arc::new(crate::x11::Controls::new(&settings)); + crate::live_x11().lock().unwrap().push(controls.clone()); + let err_slot: Arc>> = Arc::new(Mutex::new(None)); + let err_slot2 = err_slot.clone(); + let (etid_tx, etid_rx) = mpsc::channel(); + let cap_controls = controls.clone(); + let feed_tx = tx.clone(); + let feed_shared = shared.clone(); + let join = thread::spawn(move || { + let on_frame = move |stripes: Vec| { + offer_frame(&feed_shared, &feed_tx, &stripes); + }; + if let Err(e) = crate::x11::run_capture(settings, cap_controls, etid_tx, on_frame) { + eprintln!("[recorder] x11 capture error: {e}"); + *err_slot2.lock().unwrap() = Some(e); + } + }); + match etid_rx.recv_timeout(Duration::from_secs(3)) { + Ok(_) => {} + Err(mpsc::RecvTimeoutError::Disconnected) => { + let _ = join.join(); + crate::live_x11().lock().unwrap().retain(|c| !Arc::ptr_eq(c, &controls)); + let msg = err_slot + .lock() + .unwrap() + .clone() + .unwrap_or_else(|| "X11 capture exited during start".to_string()); + return Err(msg); + } + // Slow X server setup: the capture is still coming up; proceed. + Err(mpsc::RecvTimeoutError::Timeout) => {} + } + (RecordingMode::X11Own { controls, join }, "own-capture", "x11") + } + }; + + let writer = { + let shared = shared.clone(); + let path = opts.path.clone(); + thread::Builder::new() + .name("pf-recorder".to_string()) + .spawn(move || writer_thread(rx, file, path, backend_name, mode_name, shared)) + .map_err(|e| format!("failed to spawn recorder writer thread: {e}"))? + }; + + println!( + "[recorder] recording to {} ({} {})", + opts.path, backend_name, mode_name + ); + let status = RecordingStatus { + active: true, + path: opts.path.clone(), + backend: backend_name, + mode: mode_name, + frames: 0, + sync_frames: 0, + dropped: 0, + skipped_non_h264: 0, + bytes: 0, + duration_s: 0.0, + width: 0, + height: 0, + error: None, + }; + *guard = Some(ActiveRecording { + path: opts.path, + backend: backend_name, + mode_name, + mode, + tx, + shared, + writer, + }); + Ok(status) +} + +fn disarm_tap() { + WL_TAP_ARMED.store(false, Ordering::Relaxed); + *WL_TAP.lock().unwrap() = None; +} + +/// Stop the active recording, finalize the MP4, and return the final status. Errors when no +/// recording is active or when nothing recordable was ever received. +pub fn stop() -> Result { + let active = ACTIVE + .lock() + .unwrap() + .take() + .ok_or_else(|| "no recording is active".to_string())?; + disarm_tap(); + match active.mode { + RecordingMode::X11Own { controls, join } => { + controls.stop.store(true, Ordering::Relaxed); + let _ = join.join(); + crate::live_x11().lock().unwrap().retain(|c| !Arc::ptr_eq(c, &controls)); + } + RecordingMode::WaylandOwn { display_id } => { + // A streaming client that reconfigured this display now owns it; the capture + // must survive the recorder's exit. + let client_owns = crate::wayland_owners().lock().unwrap().contains_key(&display_id); + if !client_owns { + if let Some(tx) = crate::computer_use::wayland_command_sender() { + let _ = tx.send(ThreadCommand::StopCapture { display_id }); + // The next start classifies attached-vs-own against wayland_alive: + // wait for the compositor to drain the stop, or an immediate restart + // attaches to the dying capture and records nothing. + let (ack_tx, ack_rx) = mpsc::channel(); + if tx.send(ThreadCommand::Barrier { reply: ack_tx }).is_ok() { + let _ = ack_rx.recv_timeout(Duration::from_secs(2)); + } + } + } + } + RecordingMode::WaylandAttached => {} + } + drop(active.tx); + let _ = active.writer.join(); + let finished = LAST_FINISHED.lock().unwrap().clone().ok_or_else(|| { + format!("recording of {} produced no final status", active.path) + })?; + match finished.error { + Some(ref e) => Err(e.clone()), + None => Ok(finished), + } +} + +/// The current status: the live recording when one is active, otherwise the last finished +/// one; `None` when the process has never recorded. +pub fn status() -> Option { + let guard = ACTIVE.lock().unwrap(); + if let Some(a) = guard.as_ref() { + return Some(RecordingStatus { + active: true, + path: a.path.clone(), + backend: a.backend, + mode: a.mode_name, + frames: a.shared.muxed.load(Ordering::Relaxed), + sync_frames: a.shared.sync_frames.load(Ordering::Relaxed), + dropped: a.shared.dropped.load(Ordering::Relaxed), + skipped_non_h264: a.shared.skipped_non_h264.load(Ordering::Relaxed), + bytes: a.shared.bytes.load(Ordering::Relaxed), + duration_s: a.shared.start.elapsed().as_secs_f64(), + width: a.shared.width.load(Ordering::Relaxed), + height: a.shared.height.load(Ordering::Relaxed), + error: a.shared.error.lock().unwrap().clone(), + }); + } + drop(guard); + LAST_FINISHED.lock().unwrap().clone() +} + +/// Finalize any active recording at interpreter shutdown so the MP4's last buffered sample +/// is flushed. Best-effort; the fragmented layout keeps even an unflushed file playable. +pub fn finalize_on_exit() { + if ACTIVE.lock().unwrap().is_some() { + let _ = stop(); + } +} + +/// `PIXELFLUX_RECORD=`: record from process start. X11 (via `DISPLAY`) starts +/// immediately; otherwise a background thread waits for the in-process Wayland compositor to +/// come up and then starts. Retries transient failures until the deadline. +pub fn autostart_from_env() { + use std::sync::OnceLock; + static STARTED: OnceLock<()> = OnceLock::new(); + let Ok(path) = std::env::var("PIXELFLUX_RECORD") else { return }; + if path.is_empty() { + return; + } + let mut first = false; + STARTED.get_or_init(|| first = true); + if !first { + return; + } + thread::spawn(move || { + let opts = RecordOptions::from_env(path); + let deadline = Instant::now() + Duration::from_secs(300); + let mut last_err = String::new(); + loop { + let wayland_up = crate::computer_use::wayland_command_sender().is_some(); + let x11_up = std::env::var("DISPLAY").map(|v| !v.is_empty()).unwrap_or(false); + let ready = match opts.backend { + Some(PreferredBackend::X11) => x11_up, + Some(PreferredBackend::Wayland) => wayland_up, + None => wayland_up || x11_up, + }; + if ready { + match start(opts.clone()) { + Ok(_) => return, + Err(e) => last_err = e, + } + } + if Instant::now() >= deadline { + eprintln!( + "[recorder] PIXELFLUX_RECORD autostart gave up: {}", + if last_err.is_empty() { "no capture backend appeared" } else { &last_err } + ); + return; + } + thread::sleep(Duration::from_millis(250)); + } + }); +} + +/// Serialize a status into the JSON shape shared by the REST endpoints. +pub fn status_to_json(s: &RecordingStatus) -> serde_json::Value { + serde_json::json!({ + "active": s.active, + "path": s.path, + "backend": s.backend, + "mode": s.mode, + "frames": s.frames, + "sync_frames": s.sync_frames, + "dropped": s.dropped, + "skipped_non_h264": s.skipped_non_h264, + "bytes": s.bytes, + "duration_s": s.duration_s, + "width": s.width, + "height": s.height, + "error": s.error, + }) +} diff --git a/pixelflux/src/recorder/mp4.rs b/pixelflux/src/recorder/mp4.rs new file mode 100644 index 0000000..f1a2297 --- /dev/null +++ b/pixelflux/src/recorder/mp4.rs @@ -0,0 +1,807 @@ +/* + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at https://mozilla.org/MPL/2.0/. + */ + +//! Pure-Rust fragmented-MP4 (fMP4) muxer for the built-in recorder. +//! +//! Hand-rolled rather than pulled in as a dependency for two reasons: `ffmpeg-sys-next`'s +//! `avformat` feature would add libavformat as a hard runtime dependency of every build, and the +//! pure-Rust mp4 crates only write moov-trailing progressive files, which lose everything on a +//! crash. Fragmented MP4 needs no trailer and no seeking — each `moof`+`mdat` pair is +//! self-contained — so a file truncated by a crash or SIGKILL stays playable up to the last +//! fragment, and the writer works on any `Write` sink. +//! +//! Timestamps are caller-supplied wall-clock microseconds (damage-driven capture emits sparse, +//! irregular frames), carried at a 90 kHz track timescale with one sample per fragment: `tfdt` +//! anchors every sample at its true capture time, so variable framerate needs no constant-rate +//! lie. The sample duration is only known once the NEXT frame arrives, so one sample is always +//! buffered and flushed a frame behind (a clean stop closes it with the median observed +//! duration); on SIGKILL at most that one buffered frame is lost — every fragment already +//! written remains playable. +//! +//! Codec support is deliberately split: [`annexb_to_sample`] and the H.264-specific parameter-set +//! capture live in [`H264SampleBuilder`], while the fragment/box writer below is codec-agnostic +//! (bytes + sync flag + timestamps + a ready-made `stsd` sample entry). HEVC or AV1 recording +//! later means a new sample builder emitting an `hvc1`/`av01` entry, not a new muxer. + +use std::io::Write; + +/// 90 kHz: the conventional H.264 track timescale, exactly representing common frame intervals. +const TIMESCALE: u32 = 90_000; + +/// Fallback duration for the final buffered sample when only one frame was ever written +/// (no observed inter-frame delta to take a median of): 1/30 s. +const DEFAULT_LAST_DURATION: u32 = TIMESCALE / 30; + +/// Split an Annex-B elementary stream into NAL payloads (start codes removed, emulation +/// prevention bytes kept — the RBSP layer is only unescaped where a parser needs it). +pub fn split_annexb(data: &[u8]) -> Vec<&[u8]> { + let mut nals = Vec::new(); + let mut i = 0usize; + let mut nal_start: Option = None; + while i + 2 < data.len() { + if data[i] == 0 && data[i + 1] == 0 && data[i + 2] == 1 { + let code_start = if i > 0 && data[i - 1] == 0 { i - 1 } else { i }; + if let Some(s) = nal_start { + if code_start > s { + nals.push(&data[s..code_start]); + } + } + i += 3; + nal_start = Some(i); + } else if data[i + 2] == 0 { + // A zero at i+2 can begin the next start code; only advance one byte. + i += 1; + } else { + i += 3; + } + } + if let Some(s) = nal_start { + if data.len() > s { + nals.push(&data[s..]); + } + } + nals +} + +/// Unescape an H.264 RBSP: drop the emulation-prevention byte from every `00 00 03` run. +fn unescape_rbsp(data: &[u8]) -> Vec { + let mut out = Vec::with_capacity(data.len()); + let mut zeros = 0u32; + for &b in data { + if zeros >= 2 && b == 3 { + zeros = 0; + continue; + } + if b == 0 { + zeros += 1; + } else { + zeros = 0; + } + out.push(b); + } + out +} + +/// MSB-first bit reader over an unescaped RBSP, with Exp-Golomb decode for SPS parsing. +struct BitReader<'a> { + data: &'a [u8], + pos: usize, +} + +impl<'a> BitReader<'a> { + fn new(data: &'a [u8]) -> Self { + Self { data, pos: 0 } + } + + fn bit(&mut self) -> Option { + let byte = *self.data.get(self.pos / 8)?; + let bit = (byte >> (7 - (self.pos % 8))) & 1; + self.pos += 1; + Some(bit as u32) + } + + fn bits(&mut self, n: u32) -> Option { + let mut v = 0u32; + for _ in 0..n { + v = (v << 1) | self.bit()?; + } + Some(v) + } + + /// ue(v): count leading zeros, then read that many bits after the marker one. + fn ue(&mut self) -> Option { + let mut zeros = 0u32; + while self.bit()? == 0 { + zeros += 1; + if zeros > 31 { + return None; + } + } + let rest = self.bits(zeros)?; + Some((1u32 << zeros) - 1 + rest) + } + + fn se(&mut self) -> Option { + let k = self.ue()? as i64; + Some(if k % 2 == 0 { -(k / 2) as i32 } else { ((k + 1) / 2) as i32 }) + } +} + +/// Coded frame dimensions parsed out of an H.264 SPS NAL (with its NAL header byte). +/// +/// Walks every field ahead of `pic_width_in_mbs_minus1` — including the high-profile +/// chroma/bit-depth/scaling-list block — and applies the frame-cropping rectangle with the +/// chroma-format-dependent crop units, so 4:2:0, 4:2:2 and 4:4:4 streams from any of the +/// project's encoders all report their true display size. +pub fn parse_sps_dimensions(sps_nal: &[u8]) -> Option<(u32, u32)> { + if sps_nal.len() < 4 || sps_nal[0] & 0x1f != 7 { + return None; + } + let profile_idc = sps_nal[1]; + let rbsp = unescape_rbsp(&sps_nal[4..]); + let mut r = BitReader::new(&rbsp); + r.ue()?; // seq_parameter_set_id + + let mut chroma_format_idc = 1u32; + if matches!( + profile_idc, + 100 | 110 | 122 | 244 | 44 | 83 | 86 | 118 | 128 | 138 | 139 | 134 | 135 + ) { + chroma_format_idc = r.ue()?; + if chroma_format_idc == 3 { + r.bit()?; // separate_colour_plane_flag + } + r.ue()?; // bit_depth_luma_minus8 + r.ue()?; // bit_depth_chroma_minus8 + r.bit()?; // qpprime_y_zero_transform_bypass_flag + if r.bit()? == 1 { + // seq_scaling_matrix_present_flag + let lists = if chroma_format_idc == 3 { 12 } else { 8 }; + for i in 0..lists { + if r.bit()? == 1 { + let size = if i < 6 { 16 } else { 64 }; + let mut next_scale = 8i32; + let mut last_scale = 8i32; + for _ in 0..size { + if next_scale != 0 { + let delta = r.se()?; + next_scale = (last_scale + delta + 256) % 256; + } + if next_scale != 0 { + last_scale = next_scale; + } + } + } + } + } + } + + r.ue()?; // log2_max_frame_num_minus4 + let pic_order_cnt_type = r.ue()?; + if pic_order_cnt_type == 0 { + r.ue()?; // log2_max_pic_order_cnt_lsb_minus4 + } else if pic_order_cnt_type == 1 { + r.bit()?; // delta_pic_order_always_zero_flag + r.se()?; // offset_for_non_ref_pic + r.se()?; // offset_for_top_to_bottom_field + let n = r.ue()?; + for _ in 0..n { + r.se()?; + } + } + r.ue()?; // max_num_ref_frames + r.bit()?; // gaps_in_frame_num_value_allowed_flag + let pic_width_in_mbs = r.ue()? + 1; + let pic_height_in_map_units = r.ue()? + 1; + let frame_mbs_only = r.bit()?; + if frame_mbs_only == 0 { + r.bit()?; // mb_adaptive_frame_field_flag + } + r.bit()?; // direct_8x8_inference_flag + + let (mut crop_l, mut crop_r, mut crop_t, mut crop_b) = (0u32, 0u32, 0u32, 0u32); + if r.bit()? == 1 { + crop_l = r.ue()?; + crop_r = r.ue()?; + crop_t = r.ue()?; + crop_b = r.ue()?; + } + + let (sub_w, sub_h) = match chroma_format_idc { + 0 | 3 => (1u32, 1u32), + 2 => (2, 1), + _ => (2, 2), + }; + let crop_unit_x = sub_w; + let crop_unit_y = sub_h * (2 - frame_mbs_only); + let width = pic_width_in_mbs * 16 - crop_unit_x * (crop_l + crop_r); + let height = (2 - frame_mbs_only) * pic_height_in_map_units * 16 - crop_unit_y * (crop_t + crop_b); + Some((width, height)) +} + +fn mk_box(fourcc: &[u8; 4], payload: &[u8]) -> Vec { + let mut b = Vec::with_capacity(8 + payload.len()); + b.extend_from_slice(&((8 + payload.len()) as u32).to_be_bytes()); + b.extend_from_slice(fourcc); + b.extend_from_slice(payload); + b +} + +fn mk_full_box(fourcc: &[u8; 4], version: u8, flags: u32, payload: &[u8]) -> Vec { + let mut p = Vec::with_capacity(4 + payload.len()); + p.push(version); + p.extend_from_slice(&flags.to_be_bytes()[1..]); + p.extend_from_slice(payload); + mk_box(fourcc, &p) +} + +const MATRIX_IDENTITY: [u8; 36] = { + let mut m = [0u8; 36]; + m[1] = 0x01; // 0x00010000 + m[17] = 0x01; + m[32] = 0x40; // 0x40000000 + m +}; + +/// A codec's contribution to the init segment: its `stsd` sample entry (with decoder +/// configuration record) plus the display dimensions for `tkhd`. +pub struct TrackConfig { + pub sample_entry: Vec, + pub width: u32, + pub height: u32, +} + +/// One buffered access unit awaiting its duration (known when the next one arrives). +struct PendingSample { + data: Vec, + sync: bool, + dts: u64, +} + +/// Aggregate counters reported when the writer finishes. +#[derive(Clone, Copy, Debug, Default)] +pub struct Mp4Stats { + pub samples: u64, + pub sync_samples: u64, + pub bytes: u64, + pub duration_us: u64, +} + +/// Codec-agnostic fMP4 fragment writer: `ftyp`+`moov` once, then one `moof`+`mdat` pair +/// per sample, each anchored at its wall-clock decode time via `tfdt`. +pub struct FragmentWriter { + out: W, + seq: u32, + wrote_init: bool, + pending: Option, + last_dts: Option, + /// Every flushed inter-frame duration, kept so the final buffered sample (whose + /// successor never arrives) can close on the MEDIAN — damage-driven capture makes the + /// last observed delta an outlier as often as not (a long static gap, a burst pair). + durations: Vec, + stats: Mp4Stats, +} + +impl FragmentWriter { + pub fn new(out: W) -> Self { + Self { + out, + seq: 0, + wrote_init: false, + pending: None, + last_dts: None, + durations: Vec::new(), + stats: Mp4Stats::default(), + } + } + + pub fn init_written(&self) -> bool { + self.wrote_init + } + + /// Counters for the fragments written so far (the buffered pending sample is not yet + /// included; `finish` folds it in). + pub fn stats(&self) -> Mp4Stats { + self.stats + } + + /// Write `ftyp` + `moov` (track 1, `mvex`/`trex` marking the movie fragmented). Must be + /// called once, before the first sample. + pub fn write_init(&mut self, cfg: &TrackConfig) -> std::io::Result<()> { + let mut ftyp_p = Vec::new(); + ftyp_p.extend_from_slice(b"isom"); + ftyp_p.extend_from_slice(&0x200u32.to_be_bytes()); + for brand in [b"isom", b"iso5", b"iso6", b"avc1", b"mp41"] { + ftyp_p.extend_from_slice(brand); + } + let ftyp = mk_box(b"ftyp", &ftyp_p); + + let mut mvhd_p = Vec::new(); + mvhd_p.extend_from_slice(&[0u8; 8]); // creation/modification time + mvhd_p.extend_from_slice(&TIMESCALE.to_be_bytes()); + mvhd_p.extend_from_slice(&0u32.to_be_bytes()); // duration: unknown (fragmented) + mvhd_p.extend_from_slice(&0x00010000u32.to_be_bytes()); // rate 1.0 + mvhd_p.extend_from_slice(&0x0100u16.to_be_bytes()); // volume 1.0 + mvhd_p.extend_from_slice(&[0u8; 10]); // reserved + mvhd_p.extend_from_slice(&MATRIX_IDENTITY); + mvhd_p.extend_from_slice(&[0u8; 24]); // pre_defined + mvhd_p.extend_from_slice(&2u32.to_be_bytes()); // next_track_ID + let mvhd = mk_full_box(b"mvhd", 0, 0, &mvhd_p); + + let mut tkhd_p = Vec::new(); + tkhd_p.extend_from_slice(&[0u8; 8]); + tkhd_p.extend_from_slice(&1u32.to_be_bytes()); // track_ID + tkhd_p.extend_from_slice(&[0u8; 4]); // reserved + tkhd_p.extend_from_slice(&0u32.to_be_bytes()); // duration + tkhd_p.extend_from_slice(&[0u8; 16]); // reserved, layer, group, volume, reserved + tkhd_p.extend_from_slice(&MATRIX_IDENTITY); + tkhd_p.extend_from_slice(&(cfg.width << 16).to_be_bytes()); + tkhd_p.extend_from_slice(&(cfg.height << 16).to_be_bytes()); + let tkhd = mk_full_box(b"tkhd", 0, 3, &tkhd_p); // enabled | in_movie + + let mut mdhd_p = Vec::new(); + mdhd_p.extend_from_slice(&[0u8; 8]); + mdhd_p.extend_from_slice(&TIMESCALE.to_be_bytes()); + mdhd_p.extend_from_slice(&0u32.to_be_bytes()); + mdhd_p.extend_from_slice(&0x55c4u16.to_be_bytes()); // language: und + mdhd_p.extend_from_slice(&[0u8; 2]); + let mdhd = mk_full_box(b"mdhd", 0, 0, &mdhd_p); + + let mut hdlr_p = Vec::new(); + hdlr_p.extend_from_slice(&[0u8; 4]); // pre_defined + hdlr_p.extend_from_slice(b"vide"); + hdlr_p.extend_from_slice(&[0u8; 12]); + hdlr_p.extend_from_slice(b"pixelflux\0"); + let hdlr = mk_full_box(b"hdlr", 0, 0, &hdlr_p); + + let vmhd = mk_full_box(b"vmhd", 0, 1, &[0u8; 8]); + let url = mk_full_box(b"url ", 0, 1, &[]); // self-contained + let mut dref_p = 1u32.to_be_bytes().to_vec(); + dref_p.extend_from_slice(&url); + let dref = mk_full_box(b"dref", 0, 0, &dref_p); + let dinf = mk_box(b"dinf", &dref); + + let mut stsd_p = 1u32.to_be_bytes().to_vec(); + stsd_p.extend_from_slice(&cfg.sample_entry); + let stsd = mk_full_box(b"stsd", 0, 0, &stsd_p); + let stts = mk_full_box(b"stts", 0, 0, &0u32.to_be_bytes()); + let stsc = mk_full_box(b"stsc", 0, 0, &0u32.to_be_bytes()); + let stsz = mk_full_box(b"stsz", 0, 0, &[0u8; 8]); + let stco = mk_full_box(b"stco", 0, 0, &0u32.to_be_bytes()); + let stbl = mk_box( + b"stbl", + &[stsd, stts, stsc, stsz, stco].concat(), + ); + + let minf = mk_box(b"minf", &[vmhd, dinf, stbl].concat()); + let mdia = mk_box(b"mdia", &[mdhd, hdlr, minf].concat()); + let trak = mk_box(b"trak", &[tkhd, mdia].concat()); + + let mut trex_p = Vec::new(); + trex_p.extend_from_slice(&1u32.to_be_bytes()); // track_ID + trex_p.extend_from_slice(&1u32.to_be_bytes()); // default_sample_description_index + trex_p.extend_from_slice(&[0u8; 12]); // default duration/size/flags + let trex = mk_full_box(b"trex", 0, 0, &trex_p); + let mvex = mk_box(b"mvex", &trex); + + let moov = mk_box(b"moov", &[mvhd, trak, mvex].concat()); + + self.out.write_all(&ftyp)?; + self.out.write_all(&moov)?; + self.stats.bytes += (ftyp.len() + moov.len()) as u64; + self.wrote_init = true; + Ok(()) + } + + /// Queue one sample at `pts_us` (wall-clock microseconds since recording start), + /// flushing the previously buffered sample with its now-known duration. Timestamps are + /// clamped strictly monotonic so a repeated or reordered clock can never emit a + /// zero/negative duration. + pub fn push_sample(&mut self, data: Vec, sync: bool, pts_us: u64) -> std::io::Result<()> { + let mut dts = pts_us * (TIMESCALE as u64 / 1000) / 1000; + if let Some(last) = self.last_dts { + if dts <= last { + dts = last + 1; + } + } + self.last_dts = Some(dts); + if let Some(prev) = self.pending.take() { + let duration = (dts - prev.dts).min(u32::MAX as u64) as u32; + self.durations.push(duration); + self.write_fragment(&prev, duration)?; + } + self.pending = Some(PendingSample { data, sync, dts }); + Ok(()) + } + + /// Median of the observed inter-frame durations (default 1/30 s when fewer than two + /// frames were pushed), used to close the final sample. + fn median_duration(&self) -> u32 { + if self.durations.is_empty() { + return DEFAULT_LAST_DURATION; + } + let mut sorted = self.durations.clone(); + sorted.sort_unstable(); + sorted[sorted.len() / 2] + } + + fn write_fragment(&mut self, s: &PendingSample, duration: u32) -> std::io::Result<()> { + self.seq += 1; + + let mut tfhd_p = Vec::new(); + tfhd_p.extend_from_slice(&1u32.to_be_bytes()); // track_ID + let tfhd = mk_full_box(b"tfhd", 0, 0x020000, &tfhd_p); // default-base-is-moof + + let mut tfdt_p = Vec::new(); + tfdt_p.extend_from_slice(&s.dts.to_be_bytes()); + let tfdt = mk_full_box(b"tfdt", 1, 0, &tfdt_p); + + // sample flags: sync = "depends on nothing"; non-sync also sets the non-sync bit. + let sample_flags: u32 = if s.sync { 0x0200_0000 } else { 0x0101_0000 }; + let mut trun_p = Vec::new(); + trun_p.extend_from_slice(&1u32.to_be_bytes()); // sample_count + trun_p.extend_from_slice(&0i32.to_be_bytes()); // data_offset placeholder + trun_p.extend_from_slice(&duration.to_be_bytes()); + trun_p.extend_from_slice(&(s.data.len() as u32).to_be_bytes()); + trun_p.extend_from_slice(&sample_flags.to_be_bytes()); + // data-offset | sample-duration | sample-size | sample-flags + let mut trun = mk_full_box(b"trun", 0, 0x000701, &trun_p); + + let traf_len = 8 + tfhd.len() + tfdt.len() + trun.len(); + let moof_len = 8 + 16 + traf_len; // mfhd is 16 bytes + // First sample byte sits just past the mdat header, relative to moof start. + let data_offset = (moof_len + 8) as i32; + let off_pos = trun.len() - trun_p.len() + 4; // into trun payload: after sample_count + trun[off_pos..off_pos + 4].copy_from_slice(&data_offset.to_be_bytes()); + + let mfhd = mk_full_box(b"mfhd", 0, 0, &self.seq.to_be_bytes()); + let traf = mk_box(b"traf", &[tfhd, tfdt, trun].concat()); + let moof = mk_box(b"moof", &[mfhd, traf].concat()); + debug_assert_eq!(moof.len(), moof_len); + + self.out.write_all(&moof)?; + self.out.write_all(&((8 + s.data.len()) as u32).to_be_bytes())?; + self.out.write_all(b"mdat")?; + self.out.write_all(&s.data)?; + self.out.flush()?; + + self.stats.samples += 1; + if s.sync { + self.stats.sync_samples += 1; + } + self.stats.bytes += (moof.len() + 8 + s.data.len()) as u64; + self.stats.duration_us = (s.dts + duration as u64) * 1000 / (TIMESCALE as u64 / 1000); + Ok(()) + } + + /// Flush the final buffered sample, closed with the MEDIAN observed inter-frame + /// duration (its successor never arrives), and return the aggregate counters. + pub fn finish(mut self) -> std::io::Result { + if let Some(prev) = self.pending.take() { + let d = self.median_duration(); + self.write_fragment(&prev, d)?; + } + self.out.flush()?; + Ok(self.stats) + } +} + +/// H.264-specific front end: captures SPS/PPS from the stream, gates output on the first +/// IDR, converts Annex-B access units to AVCC samples, and builds the `avc1` sample entry. +pub struct H264SampleBuilder { + sps: Option>, + pps: Option>, +} + +/// One converted access unit ready for the fragment writer. +pub struct BuiltSample { + pub data: Vec, + pub sync: bool, +} + +impl Default for H264SampleBuilder { + fn default() -> Self { + Self::new() + } +} + +impl H264SampleBuilder { + pub fn new() -> Self { + Self { sps: None, pps: None } + } + + pub fn have_parameter_sets(&self) -> bool { + self.sps.is_some() && self.pps.is_some() + } + + /// Convert one Annex-B access unit into a length-prefixed AVCC sample, harvesting + /// SPS/PPS on the way. Returns `None` for an AU with no slice data (e.g. bare parameter + /// sets). `sync` is true when the AU contains an IDR slice. + pub fn build_sample(&mut self, annexb: &[u8]) -> Option { + let nals = split_annexb(annexb); + let mut data = Vec::with_capacity(annexb.len() + 8); + let mut sync = false; + let mut has_slice = false; + for nal in nals { + if nal.is_empty() { + continue; + } + match nal[0] & 0x1f { + 7 => { + if self.sps.as_deref() != Some(nal) { + self.sps = Some(nal.to_vec()); + } + } + 8 => { + if self.pps.as_deref() != Some(nal) { + self.pps = Some(nal.to_vec()); + } + } + 5 => { + sync = true; + has_slice = true; + } + 1 => has_slice = true, + _ => {} + } + data.extend_from_slice(&(nal.len() as u32).to_be_bytes()); + data.extend_from_slice(nal); + } + if !has_slice { + return None; + } + Some(BuiltSample { data, sync }) + } + + /// Build the `avc1` sample entry + `avcC` record from the captured parameter sets, with + /// the display dimensions parsed from the SPS. + pub fn track_config(&self) -> Option { + let sps = self.sps.as_deref()?; + let pps = self.pps.as_deref()?; + let (width, height) = parse_sps_dimensions(sps)?; + + let mut avcc_p = vec![ + 1, // configurationVersion + sps[1], // AVCProfileIndication + sps[2], // profile_compatibility + sps[3], // AVCLevelIndication + 0xff, // lengthSizeMinusOne = 3 + 0xe1, // numOfSequenceParameterSets = 1 + ]; + avcc_p.extend_from_slice(&(sps.len() as u16).to_be_bytes()); + avcc_p.extend_from_slice(sps); + avcc_p.push(1); // numOfPictureParameterSets + avcc_p.extend_from_slice(&(pps.len() as u16).to_be_bytes()); + avcc_p.extend_from_slice(pps); + let avcc = mk_box(b"avcC", &avcc_p); + + let mut entry_p = Vec::new(); + entry_p.extend_from_slice(&[0u8; 6]); // reserved + entry_p.extend_from_slice(&1u16.to_be_bytes()); // data_reference_index + entry_p.extend_from_slice(&[0u8; 16]); // pre_defined / reserved + entry_p.extend_from_slice(&(width as u16).to_be_bytes()); + entry_p.extend_from_slice(&(height as u16).to_be_bytes()); + entry_p.extend_from_slice(&0x0048_0000u32.to_be_bytes()); // horizresolution 72 dpi + entry_p.extend_from_slice(&0x0048_0000u32.to_be_bytes()); // vertresolution + entry_p.extend_from_slice(&[0u8; 4]); // reserved + entry_p.extend_from_slice(&1u16.to_be_bytes()); // frame_count + entry_p.extend_from_slice(&[0u8; 32]); // compressorname + entry_p.extend_from_slice(&0x0018u16.to_be_bytes()); // depth 24 + entry_p.extend_from_slice(&(-1i16).to_be_bytes()); // pre_defined + entry_p.extend_from_slice(&avcc); + let entry = mk_box(b"avc1", &entry_p); + + Some(TrackConfig { sample_entry: entry, width, height }) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + // x264 SPS at 1284x722 (High 4:2:0: cropping in both axes on odd-macroblock dims). + const SPS_HIGH_1284X722: &str = "67640020acd9405105de788c0440000003004000000f03c60c6580"; + // x264 SPS at 640x360 (Constrained Baseline). + const SPS_BASE_640X360: &str = "6742c01ed900a02ff970110000030001000003003c0f162e48"; + // x264 SPS at 1920x1080 (High 4:4:4 Predictive: chroma_format_idc == 3 path). + const SPS_444_1920X1080: &str = "67f40028919b280f0044fc4e0220000003002000000781e30632c0"; + + fn hex(s: &str) -> Vec { + (0..s.len()) + .step_by(2) + .map(|i| u8::from_str_radix(&s[i..i + 2], 16).unwrap()) + .collect() + } + + /// Both start-code lengths delimit NALs; payloads come back exactly, with no framing. + #[test] + fn split_annexb_handles_mixed_start_codes() { + let mut data = Vec::new(); + data.extend_from_slice(&[0, 0, 0, 1, 0x67, 0xAA, 0xBB]); + data.extend_from_slice(&[0, 0, 1, 0x68, 0xCC]); + data.extend_from_slice(&[0, 0, 0, 1, 0x65, 0x00, 0x00, 0x03, 0x01, 0xDD]); + let nals = split_annexb(&data); + assert_eq!(nals.len(), 3); + assert_eq!(nals[0], &[0x67, 0xAA, 0xBB]); + assert_eq!(nals[1], &[0x68, 0xCC]); + assert_eq!(nals[2], &[0x65, 0x00, 0x00, 0x03, 0x01, 0xDD]); + } + + /// Dimensions from real x264 SPS across the profiles the project's encoders emit, + /// including the frame-cropping and 4:4:4 chroma paths. + #[test] + fn sps_dimensions_across_profiles() { + assert_eq!(parse_sps_dimensions(&hex(SPS_HIGH_1284X722)), Some((1284, 722))); + assert_eq!(parse_sps_dimensions(&hex(SPS_BASE_640X360)), Some((640, 360))); + assert_eq!(parse_sps_dimensions(&hex(SPS_444_1920X1080)), Some((1920, 1080))); + } + + /// Annex-B -> AVCC: every NAL is length-prefixed, IDR marks sync, parameter sets are + /// harvested, and a parameter-set-only AU yields no sample. + #[test] + fn annexb_to_avcc_sample() { + let sps = hex(SPS_BASE_640X360); + let mut au = Vec::new(); + au.extend_from_slice(&[0, 0, 0, 1]); + au.extend_from_slice(&sps); + au.extend_from_slice(&[0, 0, 0, 1, 0x68, 0xCE, 0x38, 0x80]); + au.extend_from_slice(&[0, 0, 0, 1, 0x65, 1, 2, 3, 4]); + + let mut b = H264SampleBuilder::new(); + assert!(b.build_sample(&[0u8, 0, 0, 1, 0x67, 0x42, 0xc0, 0x1e, 0xd9]).is_none()); + let s = b.build_sample(&au).expect("IDR AU builds a sample"); + assert!(s.sync); + assert!(b.have_parameter_sets()); + // Sample = 3 length-prefixed NALs, sizes preserved. + let mut off = 0usize; + let mut sizes = Vec::new(); + while off < s.data.len() { + let n = u32::from_be_bytes(s.data[off..off + 4].try_into().unwrap()) as usize; + sizes.push(n); + off += 4 + n; + } + assert_eq!(off, s.data.len()); + assert_eq!(sizes, vec![sps.len(), 4, 5]); + + let p = b.build_sample(&[0u8, 0, 0, 1, 0x41, 9, 9]).unwrap(); + assert!(!p.sync); + } + + /// Walk the top-level boxes of a finished two-sample stream: init once, then one + /// moof+mdat pair per sample, with sizes that exactly tile the buffer. + #[test] + fn fragment_stream_box_layout() { + let mut b = H264SampleBuilder::new(); + let mut au = Vec::new(); + au.extend_from_slice(&[0, 0, 0, 1]); + au.extend_from_slice(&hex(SPS_BASE_640X360)); + au.extend_from_slice(&[0, 0, 0, 1, 0x68, 0xCE, 0x38, 0x80]); + au.extend_from_slice(&[0, 0, 0, 1, 0x65, 1, 2, 3, 4]); + let s1 = b.build_sample(&au).unwrap(); + let s2 = b.build_sample(&[0u8, 0, 0, 1, 0x41, 5, 6, 7]).unwrap(); + + let mut buf = Vec::new(); + let mut w = FragmentWriter::new(&mut buf); + w.write_init(&b.track_config().unwrap()).unwrap(); + w.push_sample(s1.data, s1.sync, 0).unwrap(); + w.push_sample(s2.data, s2.sync, 33_000).unwrap(); + let stats = w.finish().unwrap(); + assert_eq!(stats.samples, 2); + assert_eq!(stats.sync_samples, 1); + assert_eq!(stats.bytes as usize, buf.len()); + + let mut kinds = Vec::new(); + let mut off = 0usize; + while off < buf.len() { + let size = u32::from_be_bytes(buf[off..off + 4].try_into().unwrap()) as usize; + kinds.push(buf[off + 4..off + 8].to_vec()); + assert!(size >= 8 && off + size <= buf.len()); + off += size; + } + assert_eq!(off, buf.len()); + let names: Vec<&str> = kinds.iter().map(|k| std::str::from_utf8(k).unwrap()).collect(); + assert_eq!(names, vec!["ftyp", "moov", "moof", "mdat", "moof", "mdat"]); + } + + /// Every trun sample_duration in stream order (the writer emits one sample per trun). + fn trun_durations(buf: &[u8]) -> Vec { + let mut out = Vec::new(); + let mut off = 0usize; + while off + 24 <= buf.len() { + if &buf[off + 4..off + 8] == b"trun" { + // [size][fourcc][ver+flags][sample_count][data_offset][duration] + out.push(u32::from_be_bytes(buf[off + 20..off + 24].try_into().unwrap())); + } + off += 1; + } + out + } + + /// Each flushed sample's duration is the pts delta to its successor, and the final + /// buffered sample closes with the MEDIAN observed duration — not the last delta, which + /// under damage-driven capture is an outlier as often as not. + #[test] + fn sample_durations_are_pts_deltas_with_median_tail() { + let mut buf = Vec::new(); + let mut w = FragmentWriter::new(&mut buf); + let cfg = { + let mut b = H264SampleBuilder::new(); + let mut au = vec![0, 0, 0, 1]; + au.extend_from_slice(&hex(SPS_BASE_640X360)); + au.extend_from_slice(&[0, 0, 0, 1, 0x68, 0xCE, 0x38, 0x80]); + au.extend_from_slice(&[0, 0, 0, 1, 0x65, 1]); + b.build_sample(&au); + b.track_config().unwrap() + }; + w.write_init(&cfg).unwrap(); + // 33 ms, 33 ms, then a 300 ms static gap before the final frame. + w.push_sample(vec![0, 0, 0, 1, 0x65], true, 0).unwrap(); + w.push_sample(vec![0, 0, 0, 1, 0x41], false, 33_000).unwrap(); + w.push_sample(vec![0, 0, 0, 1, 0x41], false, 66_000).unwrap(); + w.push_sample(vec![0, 0, 0, 1, 0x41], false, 366_000).unwrap(); + let stats = w.finish().unwrap(); + assert_eq!(stats.samples, 4); + // 90 kHz ticks: 33 ms = 2970. The tail closes at the median (2970), not 27000. + assert_eq!(trun_durations(&buf), vec![2970, 2970, 27_000, 2970]); + } + + /// A single-frame recording still closes with a sane nonzero duration. + #[test] + fn single_sample_uses_default_duration() { + let mut buf = Vec::new(); + let mut w = FragmentWriter::new(&mut buf); + let cfg = { + let mut b = H264SampleBuilder::new(); + let mut au = vec![0, 0, 0, 1]; + au.extend_from_slice(&hex(SPS_BASE_640X360)); + au.extend_from_slice(&[0, 0, 0, 1, 0x68, 0xCE, 0x38, 0x80]); + au.extend_from_slice(&[0, 0, 0, 1, 0x65, 1]); + b.build_sample(&au); + b.track_config().unwrap() + }; + w.write_init(&cfg).unwrap(); + w.push_sample(vec![0, 0, 0, 1, 0x65], true, 0).unwrap(); + let stats = w.finish().unwrap(); + assert_eq!(stats.samples, 1); + assert_eq!(trun_durations(&buf), vec![DEFAULT_LAST_DURATION]); + } + + /// Non-monotonic wall-clock input is clamped to strictly increasing decode times, so no + /// fragment can carry a zero or negative duration. + #[test] + fn pts_clamped_strictly_monotonic() { + let mut buf = Vec::new(); + let mut w = FragmentWriter::new(&mut buf); + let cfg = { + let mut b = H264SampleBuilder::new(); + let mut au = vec![0, 0, 0, 1]; + au.extend_from_slice(&hex(SPS_BASE_640X360)); + au.extend_from_slice(&[0, 0, 0, 1, 0x68, 0xCE, 0x38, 0x80]); + au.extend_from_slice(&[0, 0, 0, 1, 0x65, 1]); + b.build_sample(&au); + b.track_config().unwrap() + }; + w.write_init(&cfg).unwrap(); + w.push_sample(vec![0, 0, 0, 1, 0x65], true, 1000).unwrap(); + w.push_sample(vec![0, 0, 0, 1, 0x41], false, 1000).unwrap(); // repeated clock + w.push_sample(vec![0, 0, 0, 1, 0x41], false, 500).unwrap(); // clock went backwards + let stats = w.finish().unwrap(); + assert_eq!(stats.samples, 3); + + // Extract each tfdt baseMediaDecodeTime and check strict monotonicity. + let mut times = Vec::new(); + let mut off = 0usize; + while off + 8 <= buf.len() { + if &buf[off + 4..off + 8] == b"tfdt" { + let t = u64::from_be_bytes(buf[off + 12..off + 20].try_into().unwrap()); + times.push(t); + } + off += 1; + } + assert_eq!(times.len(), 3); + assert!(times.windows(2).all(|w| w[1] > w[0]), "tfdt times: {times:?}"); + } +} diff --git a/pixelflux/src/recording_sink.rs b/pixelflux/src/recording_sink.rs index b0fa05d..b4eb426 100644 --- a/pixelflux/src/recording_sink.rs +++ b/pixelflux/src/recording_sink.rs @@ -1,73 +1,77 @@ //! Unix-socket H.264 fan-out for external recording. //! -//! A background listener thread binds a Unix domain socket and accepts an arbitrary number of -//! concurrent clients. Each encoded H.264 stripe written through [`RecordingSink`] is broadcast -//! to every connected client as a plain Annex-B elementary stream — the 10-byte wire header -//! (tag, keyframe flag, frame number, y-start, width, height) is stripped so the output is -//! directly muxable. +//! Frames are intercepted at the delivery layer (not inside each encoder), so the tap works +//! uniformly for every full-frame encoder. The 10-byte pixelflux wire header is skipped so +//! consumers receive a plain Annex-B elementary stream that is directly muxable. //! -//! The sink is created once per capture session and shared (via `Arc`) between the encode -//! threads and the delivery layer. When a new client connects, [`RecordingSink::should_force_idr`] -//! returns `true` on the next call, signalling the encoder to emit an IDR so the fresh consumer -//! can start decoding from a clean reference frame. +//! The tap must never perturb the live viewer transport, and it never copies frame bytes: +//! stripe payloads are `Arc`-shared, so the encode thread only clones a handle into a bounded +//! per-client channel drained by a dedicated writer thread. A slow or stalled recorder blocks +//! nothing but itself and is dropped once its queue overflows. A newly connected client arms +//! [`RecordingSink::should_force_idr`] so the next encode emits an IDR it can decode from. use std::fs; use std::io::{ErrorKind, Write}; -use std::os::unix::net::{UnixListener, UnixStream}; +use std::os::unix::net::UnixListener; use std::sync::atomic::{AtomicBool, Ordering}; use std::sync::{Arc, Mutex}; use std::thread; use std::time::Duration; -/// Write timeout applied to each accepted client stream. Keeps a slow consumer from blocking -/// the encode thread; a stalled write simply drops that client. +use crossbeam_channel::{bounded, Sender, TrySendError}; + +use crate::encoders::software::EncodedStripe; + +/// Per-write timeout on a client stream; a stalled write surfaces as a soft error that +/// [`write_all_frame`] retries, keeping the writer thread responsive to teardown. const WRITE_TIMEOUT: Duration = Duration::from_millis(100); /// How often the non-blocking accept loop retries when no client is waiting. const ACCEPT_POLL_INTERVAL: Duration = Duration::from_millis(50); -/// Environment variable consulted as a fallback when the Python `CaptureSettings.recording_socket` -/// field is empty. -pub const RECORDING_SOCKET_ENV: &str = "PIXELFLUX_RECORDING_SOCKET"; +/// Per-client backlog bound. A recorder that falls this far behind is dropped rather than +/// allowed to grow memory or push back on the shared encode thread. +const CLIENT_QUEUE_CAP: usize = 256; + +/// A queued frame: the `Arc`-shared payload plus the byte offset where the recordable +/// Annex-B stream starts (past the wire header, or `0` when the payload is bare). +type QueuedFrame = (Arc>, usize); + +/// The sink's handle to one connected recorder: the feed end of its bounded queue and a kill +/// switch for its writer thread. The socket itself is owned solely by that writer thread. +struct ClientHandle { + tx: Sender, + stop: Arc, +} /// Unix-socket fan-out that broadcasts every encoded H.264 frame to connected consumers. /// -/// Internally a listener thread accepts connections on a [`UnixListener`] and appends the -/// resulting streams to a shared `Vec`. The encode / delivery threads then call -/// [`write_encoded_frame`] to push data to every client; clients that fall behind or disconnect -/// are removed automatically. -/// -/// [`write_encoded_frame`]: RecordingSink::write_encoded_frame +/// A listener thread accepts connections and gives each its own bounded queue and writer thread +/// (see [`ClientHandle`]) so one slow reader cannot stall the others or the encode thread. pub struct RecordingSink { /// Filesystem path of the Unix socket; removed on drop. path: String, - /// Connected client streams, shared between the accept thread and the write path. - clients: Arc>>, + /// Feed handles for the connected clients, shared with the accept thread. + clients: Arc>>, /// Signals the accept thread to exit; set in [`Drop`]. shutdown: Arc, /// Flipped to `true` each time a new client connects; consumed by [`should_force_idr`]. /// /// [`should_force_idr`]: RecordingSink::should_force_idr client_connected: Arc, + /// One-time notice that the session's H.264 frames are striped and unrecordable. + warned_unrecordable: AtomicBool, } impl RecordingSink { - /// Try to bind a Unix socket at `settings_path` (or the `PIXELFLUX_RECORDING_SOCKET` - /// environment variable when the path is empty). - /// - /// Returns `None` when no path is configured or when the bind fails. On success the - /// listener thread is spawned and the sink is ready to accept clients immediately. + /// Bind a Unix socket at `settings_path`, or return `None` when no path is configured or the + /// bind fails. Recording is an optional tap that must never take the pipeline down, so a bind + /// error is logged and swallowed. pub fn try_bind(settings_path: &str) -> Option> { - let path = if !settings_path.is_empty() { - settings_path.to_string() - } else { - match std::env::var(RECORDING_SOCKET_ENV) { - Ok(p) if !p.is_empty() => p, - _ => return None, - } - }; - - match Self::bind(path) { + if settings_path.is_empty() { + return None; + } + match Self::bind(settings_path.to_string()) { Ok(sink) => Some(Arc::new(sink)), Err(e) => { eprintln!("[recording_sink] bind failed: {:?}", e); @@ -76,13 +80,14 @@ impl RecordingSink { } } - /// Create the socket, spawn the accept thread, and return the sink. + /// Create the socket and spawn the accept thread. Each accepted connection gets a write + /// timeout, a bounded queue, and a writer thread; the sink keeps only the feed handle. fn bind(path: String) -> std::io::Result { let _ = fs::remove_file(&path); let listener = UnixListener::bind(&path)?; listener.set_nonblocking(true)?; - let clients = Arc::new(Mutex::new(Vec::new())); + let clients: Arc>> = Arc::new(Mutex::new(Vec::new())); let shutdown = Arc::new(AtomicBool::new(false)); let client_connected = Arc::new(AtomicBool::new(false)); @@ -100,13 +105,32 @@ impl RecordingSink { eprintln!("[recording_sink] set_write_timeout failed: {:?}", e); continue; } + + let (tx, rx) = bounded::(CLIENT_QUEUE_CAP); + let stop = Arc::new(AtomicBool::new(false)); + let stop_writer = stop.clone(); + thread::spawn(move || { + let mut stream = stream; + for (buf, offset) in rx.iter() { + if stop_writer.load(Ordering::Relaxed) { + break; + } + if let Err(e) = + write_all_frame(&mut stream, &buf[offset..], &stop_writer) + { + eprintln!( + "[recording_sink] writer thread exiting; write failed: {:?}", + e + ); + break; + } + } + }); + let mut guard = clients_acc.lock().unwrap(); - guard.push(stream); + guard.push(ClientHandle { tx, stop }); client_connected_acc.store(true, Ordering::Relaxed); - eprintln!( - "[recording_sink] client connected; total {}", - guard.len() - ); + eprintln!("[recording_sink] client connected; total {}", guard.len()); } Err(e) if e.kind() == ErrorKind::WouldBlock => { thread::sleep(ACCEPT_POLL_INTERVAL); @@ -125,65 +149,196 @@ impl RecordingSink { clients, shutdown, client_connected, + warned_unrecordable: AtomicBool::new(false), }) } - /// Returns `true` exactly once after a new client connects, signalling that the next - /// encode should produce an IDR so the consumer starts from a clean reference frame. + /// Returns `true` exactly once after a new client connects, signalling that the next encode + /// should produce an IDR so the consumer starts from a clean reference frame. pub fn should_force_idr(&self) -> bool { self.client_connected.swap(false, Ordering::Relaxed) } - /// Write `data` to every connected client, silently removing any that are disconnected - /// or too slow. - fn write_all_clients(&self, data: &[u8]) { - if data.is_empty() { + /// Delivery-layer tap for one encoded frame. The socket carries a single H.264 + /// elementary stream, so only a lone full-height stripe (`data_type == 2`) is + /// recordable: striped CPU encodes are N independent per-stripe streams, and + /// interleaving them would produce an undecodable file — those are skipped with a + /// one-time notice (live streaming is unaffected). The 10-byte wire header + /// (`0x04` tag) is skipped via the queued offset so consumers receive plain + /// Annex-B. + /// + /// Never blocks and never copies: the `Arc` payload is cloned into each client's + /// bounded queue with `try_send`, and a client whose queue is full or whose + /// writer died is dropped. + pub fn write_frame(&self, stripes: &[EncodedStripe], full_height: i32) { + let mut h264 = stripes + .iter() + .filter(|s| s.data_type == 2 && !s.data.is_empty()); + let Some(stripe) = h264.next() else { return }; + if h264.next().is_some() || stripe.stripe_y_start != 0 || stripe.stripe_height != full_height + { + if !self.warned_unrecordable.swap(true, Ordering::Relaxed) { + eprintln!( + "[recording_sink] WARNING: striped H.264 frames are not recordable \ + (the socket carries one elementary stream); use a full-frame encoder \ + to record this session" + ); + } return; } + let offset = if stripe.data.len() >= 10 && stripe.data[0] == 0x04 { + 10 + } else { + 0 + }; + if stripe.data.len() == offset { + return; + } + let mut clients = self.clients.lock().unwrap(); if clients.is_empty() { return; } let mut to_remove: Vec = Vec::new(); - for (idx, client) in clients.iter_mut().enumerate() { - if let Err(e) = client.write_all(data) { - if matches!( - e.kind(), - ErrorKind::BrokenPipe | ErrorKind::ConnectionReset | ErrorKind::UnexpectedEof - ) { + for (idx, client) in clients.iter().enumerate() { + match client.tx.try_send((stripe.data.clone(), offset)) { + Ok(()) => {} + Err(TrySendError::Full(_)) => { + eprintln!("[recording_sink] dropping slow client (idx {})", idx); to_remove.push(idx); - } else { - eprintln!("[recording_sink] dropping client (idx {}): {:?}", idx, e); + } + Err(TrySendError::Disconnected(_)) => { to_remove.push(idx); } } } for idx in to_remove.into_iter().rev() { - clients.swap_remove(idx); - } - } - - /// Write a single encoded stripe to all connected clients. - /// - /// Only H.264 frames (`data_type == 2`) are forwarded. When the frame carries the - /// pixelflux wire header (`0x04` tag, 10 bytes) the header is stripped so the output - /// is a plain Annex-B elementary stream suitable for direct muxing. - pub fn write_encoded_frame(&self, stripe: &crate::encoders::software::EncodedStripe) { - if stripe.data.is_empty() || stripe.data_type != 2 { - return; - } - let data = &stripe.data; - if data.len() > 10 && data[0] == 0x04 { - self.write_all_clients(&data[10..]); - } else { - self.write_all_clients(data); + let removed = clients.swap_remove(idx); + removed.stop.store(true, Ordering::Relaxed); } } } impl Drop for RecordingSink { + /// Stop accepting, release every writer thread (set each `stop`, then drop its sender so an + /// idle writer parked on `rx.iter()` wakes), and remove the socket file. fn drop(&mut self) { self.shutdown.store(true, Ordering::Relaxed); + if let Ok(mut clients) = self.clients.lock() { + for client in clients.iter() { + client.stop.store(true, Ordering::Relaxed); + } + clients.clear(); + } let _ = fs::remove_file(&self.path); } } + +/// Write one whole frame to a recorder's socket, resuming across the soft timeouts a slow reader +/// induces so a partial Annex-B NAL is never left behind. Aborts if `stop` is set (the client was +/// dropped by [`RecordingSink::write_encoded_frame`]) or a hard error occurs. +fn write_all_frame(stream: &mut W, buf: &[u8], stop: &AtomicBool) -> std::io::Result<()> { + let mut written = 0usize; + while written < buf.len() { + if stop.load(Ordering::Relaxed) { + return Err(std::io::Error::other("writer stopped (client dropped)")); + } + match stream.write(&buf[written..]) { + Ok(0) => { + return Err(std::io::Error::new( + ErrorKind::WriteZero, + "failed to write whole frame", + )); + } + Ok(n) => written += n, + Err(ref e) if e.kind() == ErrorKind::TimedOut => {} + Err(ref e) if e.kind() == ErrorKind::WouldBlock => { + thread::sleep(Duration::from_millis(5)); + } + Err(ref e) if e.kind() == ErrorKind::Interrupted => {} + Err(e) => return Err(e), + } + } + Ok(()) +} + +#[cfg(test)] +mod cost_tests { + //! The sink's isolation contract, measured: feeding a frame must cost nothing when no + //! recorder is connected (empty-clients early return), microseconds when a healthy + //! recorder drains its queue, and stay bounded (a lock + `try_send`, never a blocking + //! write) when a recorder stalls completely — until the bounded queue overflows and the + //! client is dropped, returning the tap to idle cost. + + use super::*; + use std::io::Read; + use std::os::unix::net::UnixStream; + use std::time::Instant; + + fn frame(len: usize) -> EncodedStripe { + let mut data = vec![0u8; len]; + data[0] = 0x04; // wire-header tag so the 10-byte strip path runs + EncodedStripe { + data: Arc::new(data), + data_type: 2, + stripe_y_start: 0, + stripe_height: 720, + frame_id: 0, + } + } + + fn feed_timed(sink: &RecordingSink, n: usize, len: usize) -> (f64, f64) { + let f = frame(len); + let mut max_us = 0f64; + let mut total_us = 0f64; + for _ in 0..n { + let t = Instant::now(); + sink.write_frame(std::slice::from_ref(&f), 720); + let us = t.elapsed().as_secs_f64() * 1e6; + total_us += us; + max_us = max_us.max(us); + thread::sleep(Duration::from_micros(200)); + } + (total_us / n as f64, max_us) + } + + #[test] + fn stalled_recorder_isolation_cost() { + let path = format!("/tmp/pf-sink-cost-{}.sock", std::process::id()); + let sink = RecordingSink::try_bind(&path).expect("bind"); + + // Idle: no client connected. + let (idle_mean, idle_max) = feed_timed(&sink, 500, 100_000); + + // Healthy: a client draining as fast as it can. + let mut healthy = UnixStream::connect(&path).expect("connect"); + thread::sleep(Duration::from_millis(200)); + let drain = thread::spawn(move || { + let mut buf = vec![0u8; 1 << 20]; + while healthy.read(&mut buf).map(|n| n > 0).unwrap_or(false) {} + }); + let (healthy_mean, healthy_max) = feed_timed(&sink, 500, 100_000); + + // Stalled: a connected client that never reads. The socket buffer fills, then the + // bounded queue fills, then the client is dropped (~256 frames later). + let stalled = UnixStream::connect(&path).expect("connect"); + thread::sleep(Duration::from_millis(200)); + let (stalled_mean, stalled_max) = feed_timed(&sink, 500, 100_000); + drop(stalled); + + println!( + "[sink-cost] idle mean {idle_mean:.3}us max {idle_max:.3}us\n\ + [sink-cost] healthy mean {healthy_mean:.3}us max {healthy_max:.3}us\n\ + [sink-cost] stalled mean {stalled_mean:.3}us max {stalled_max:.3}us" + ); + drop(sink); + let _ = drain.join(); + + assert!(idle_mean < 5.0, "idle feed should be sub-5us, was {idle_mean:.3}us"); + assert!(healthy_mean < 100.0, "healthy feed should be tens of us, was {healthy_mean:.3}us"); + assert!( + stalled_max < 10_000.0, + "a stalled recorder must never block the tap >10ms, was {stalled_max:.3}us" + ); + } +} diff --git a/pixelflux/src/wayland/cursor.rs b/pixelflux/src/wayland/cursor.rs index f5e8086..d8959b7 100644 --- a/pixelflux/src/wayland/cursor.rs +++ b/pixelflux/src/wayland/cursor.rs @@ -8,7 +8,10 @@ //! suitable for the Python cursor callback. Uses `xcursor` to load the system cursor theme //! and render the appropriate frame at the current scale. -use image::{ImageBuffer, Rgba}; +use image::{ImageBuffer, Rgba, RgbaImage}; +use pyo3::prelude::*; +use pyo3::types::PyBytes; +use std::collections::HashMap; use std::io::Cursor as IoCursor; use std::io::Read; use std::time::Duration; @@ -17,6 +20,278 @@ use xcursor::{ CursorTheme, }; +/// One unit of cursor-callback work handed from the calloop thread to the `wl-cursor` +/// worker. The calloop resolves everything renderer/surface-affine (SHM copies, dmabuf +/// readbacks, hotspots); the worker does the PNG encode, the cache, and the GIL-bound Python +/// call so none of that latency lands on the input/render thread. One channel keeps cursor +/// updates ordered with callback (re)registration. +pub enum CursorJob { + SetCallback(Py), + /// Reload the worker's theme handle at a new pixel size; later `Named` jobs render at it. + SetSize(i32), + /// Cap the longest delivered cursor edge in pixels; larger sprites are downscaled with + /// the hotspot. `<= 0` delivers them uncapped. + SetSizeCap(i32), + Named { name: &'static str }, + Hide, + /// wl_shm cursor sprite: raw pool bytes plus the sub-image descriptor. + Shm { + hash: u64, + width: i32, + height: i32, + stride: i32, + offset: i32, + opaque: bool, + bytes: Vec, + hot_x: i32, + hot_y: i32, + }, + /// dmabuf cursor sprite already read back to tightly-mapped RGBA on the calloop. + Gles { + hash: u64, + width: i32, + height: i32, + bytes: Vec, + hot_x: i32, + hot_y: i32, + }, +} + +/// Spawn the cursor delivery worker; returns its job channel. The worker owns its own +/// theme handle, the PNG cache, and the Python callback for the life of the process (like the +/// compositor thread itself); `PY_SHUTDOWN` gates every Python call. +pub fn spawn_cursor_worker(cursor_size: i32, size_cap: i32) -> std::sync::mpsc::Sender { + let (tx, rx) = std::sync::mpsc::channel::(); + let _ = std::thread::Builder::new().name("wl-cursor".into()).spawn(move || { + let mut helper = Cursor::load(cursor_size); + let mut cap = size_cap; + let mut cache: HashMap<(u64, i32), CappedSprite> = HashMap::new(); + let mut callback: Option> = None; + while let Ok(job) = rx.recv() { + let (msg_type, data, hot_x, hot_y): (&str, Vec, i32, i32) = match job { + CursorJob::SetCallback(cb) => { + callback = Some(cb); + continue; + } + CursorJob::SetSize(size) => { + helper = Cursor::load(size); + continue; + } + CursorJob::SetSizeCap(c) => { + cap = c; + continue; + } + CursorJob::Named { name } => match helper.get_sprite(name) { + Some((img, x, y)) => match cap_and_encode(img, cap) { + Some(sprite) => { + let (sx, sy) = scaled_hotspot(&sprite, x as i32, y as i32); + ("png", sprite.png, sx, sy) + } + None => ("error", Vec::new(), 0, 0), + }, + None => ("error", Vec::new(), 0, 0), + }, + CursorJob::Hide => ("hide", Vec::new(), 0, 0), + CursorJob::Shm { + hash, + width, + height, + stride, + offset, + opaque, + bytes, + hot_x, + hot_y, + } => capped_job( + &mut cache, + hash, + cap, + hot_x, + hot_y, + || decode_shm_cursor(width, height, stride, offset, opaque, &bytes), + ), + CursorJob::Gles { hash, width, height, bytes, hot_x, hot_y } => capped_job( + &mut cache, + hash, + cap, + hot_x, + hot_y, + || decode_gles_cursor(width, height, &bytes), + ), + }; + // A sprite whose pixels could not be read yields empty data; suppressing it + // preserves the consumer's last cursor instead of blanking it (only an + // intentional hide passes with no payload). + if data.is_empty() && msg_type != "hide" { + continue; + } + if crate::PY_SHUTDOWN.load(std::sync::atomic::Ordering::Relaxed) { + continue; + } + if let Some(ref cb) = callback { + Python::attach(|py| { + let py_bytes = PyBytes::new(py, &data); + let _ = cb.call1(py, (msg_type, py_bytes, hot_x, hot_y)); + }); + } + } + }); + tx +} + +/// A cursor sprite encoded at its delivered size: the straight-alpha PNG, the factor the +/// source was scaled by, and the payload's dimensions. Hotspots are per-job rather than a +/// property of the pixels, so they are scaled against `scale` instead of being stored here. +struct CappedSprite { + png: Vec, + scale: f64, + width: u32, + height: u32, +} + +/// Sprites retained before arbitrary eviction; an evicted sprite re-encodes on next appearance. +const SPRITE_CACHE_MAX: usize = 100; + +/// Apply the size cap to a **premultiplied** sprite and encode it as a straight-alpha PNG. +/// +/// `cap <= 0`, or a sprite already within it, is encoded at its source size. Resizing happens +/// while the pixels are still premultiplied — filtering straight alpha pulls the (zero) colour +/// of transparent texels into the edges and leaves a dark fringe, and Lanczos3 overshoot has no +/// headroom to land in — which is the order the X11 XFixes path uses. +fn cap_and_encode(mut img: RgbaImage, cap: i32) -> Option { + let (w, h) = (img.width(), img.height()); + if w == 0 || h == 0 { + return None; + } + let longest = w.max(h); + let mut scale = 1.0; + if cap > 0 && longest > cap as u32 { + scale = cap as f64 / longest as f64; + let nw = ((w as f64 * scale).round() as u32).max(1); + let nh = ((h as f64 * scale).round() as u32).max(1); + img = image::imageops::resize(&img, nw, nh, image::imageops::FilterType::Lanczos3); + } + crate::unpremultiply_rgba(&mut img); + let mut png = Vec::new(); + img.write_to(&mut IoCursor::new(&mut png), image::ImageFormat::Png) + .ok()?; + Some(CappedSprite { png, scale, width: img.width(), height: img.height() }) +} + +/// Scale a source hotspot onto a delivered sprite. Consumers treat the hotspot as an offset +/// INTO the bitmap, so the rounded coordinate is clamped to the payload rather than allowed to +/// land one pixel past its right or bottom edge. +fn scaled_hotspot(sprite: &CappedSprite, hot_x: i32, hot_y: i32) -> (i32, i32) { + ( + ((hot_x as f64 * sprite.scale).round() as i32).clamp(0, sprite.width as i32 - 1), + ((hot_y as f64 * sprite.scale).round() as i32).clamp(0, sprite.height as i32 - 1), + ) +} + +/// Cache wrapper for sprite jobs, keyed by `(sprite hash, cap)` so a repeat of a sprite already +/// delivered under the same cap is a plain clone of its PNG; a miss decodes the sprite, caps it +/// and encodes it once. Keying on the cap keeps a live `SetSizeCap` from serving stale sizes. +fn capped_job( + cache: &mut HashMap<(u64, i32), CappedSprite>, + hash: u64, + cap: i32, + hot_x: i32, + hot_y: i32, + decode: impl FnOnce() -> Option, +) -> (&'static str, Vec, i32, i32) { + use std::collections::hash_map::Entry; + let key = (hash, cap); + if cache.len() >= SPRITE_CACHE_MAX && !cache.contains_key(&key) { + if let Some(&evict) = cache.keys().next() { + cache.remove(&evict); + } + } + let sprite = match cache.entry(key) { + Entry::Occupied(e) => e.into_mut(), + // An unreadable sprite yields an empty payload, which the worker suppresses so the + // consumer keeps the cursor it already has. + Entry::Vacant(e) => match decode().and_then(|img| cap_and_encode(img, cap)) { + Some(sprite) => e.insert(sprite), + None => return ("png", Vec::new(), 0, 0), + }, + }; + let (sx, sy) = scaled_hotspot(sprite, hot_x, hot_y); + ("png", sprite.png.clone(), sx, sy) +} + +/// Read a wl_shm BGRA/XRGB sprite sub-image into a premultiplied RGBA image, the form +/// `cap_and_encode` must resize before it unpremultiplies. Stride/offset are clamped +/// non-negative with checked arithmetic so a garbage descriptor skips pixels instead of +/// panicking; sprites larger than 128x128 are ignored (never a hardware cursor). +fn decode_shm_cursor( + width: i32, + height: i32, + stride: i32, + offset: i32, + opaque: bool, + raw_bytes: &[u8], +) -> Option { + if width <= 0 || height <= 0 || width > 128 || height > 128 || raw_bytes.is_empty() { + return None; + } + let mut img_buf = ImageBuffer::, Vec>::new(width as u32, height as u32); + let stride_usize = stride.max(0) as usize; + let base_offset = offset.max(0) as usize; + for y in 0..(height as u32) { + for x in 0..(width as u32) { + let offset = (y as usize) + .checked_mul(stride_usize) + .and_then(|row| base_offset.checked_add(row)) + .and_then(|o| o.checked_add((x as usize) * 4)); + let offset = match offset { + Some(o) => o, + None => continue, + }; + if offset.checked_add(4).is_some_and(|end| end <= raw_bytes.len()) { + let alpha = if opaque { 255 } else { raw_bytes[offset + 3] }; + img_buf.put_pixel( + x, + y, + Rgba([raw_bytes[offset + 2], raw_bytes[offset + 1], raw_bytes[offset], alpha]), + ); + } + } + } + Some(img_buf) +} + +/// Read a dmabuf sprite's RGBA readback (stride recovered from the mapping length) into a +/// premultiplied RGBA image. +fn decode_gles_cursor(width: i32, height: i32, raw_bytes: &[u8]) -> Option { + if width <= 0 || height <= 0 || width > 128 || height > 128 || raw_bytes.is_empty() { + return None; + } + let stride = super::frontend::rgba_readback_stride( + raw_bytes.len(), + height as usize, + width as usize, + ); + let mut img_buf = ImageBuffer::, Vec>::new(width as u32, height as u32); + for y in 0..(height as u32) { + for x in 0..(width as u32) { + let offset = (y as usize * stride) + (x as usize * 4); + if offset + 4 <= raw_bytes.len() { + img_buf.put_pixel( + x, + y, + Rgba([ + raw_bytes[offset], + raw_bytes[offset + 1], + raw_bytes[offset + 2], + raw_bytes[offset + 3], + ]), + ); + } + } + } + Some(img_buf) +} + /// The loaded XCursor theme, held for the whole capture so cursor lookups stay cheap. /// /// The default cursor's frames are parsed once and kept here because they are consulted on nearly @@ -74,27 +349,21 @@ impl Cursor { Some(frame(time.as_millis() as u32, size, &icons)) } - /// A named cursor icon as PNG bytes plus its hotspot (x, y), for web clients. - /// Xcursor stores premultiplied color; the PNG carries straight alpha. The - /// compositing paths (`get_image*`) keep the premultiplied pixels blending needs. - pub fn get_png_data(&self, name: &str) -> Option<(Vec, u32, u32)> { + /// A named cursor icon as a premultiplied RGBA image plus its hotspot (x, y), ready for + /// `cap_and_encode` to size and convert to the straight alpha web clients want. Xcursor + /// stores premultiplied colour, which is also what the compositing paths (`get_image*`) + /// need for blending, so it is carried through unconverted. + pub fn get_sprite(&self, name: &str) -> Option<(RgbaImage, u32, u32)> { let icons = load_icon(&self.theme, name).ok()?; let image_data = nearest_images(self.size, &icons).next()?; - let mut img_buf: ImageBuffer, Vec> = ImageBuffer::from_raw( + let img_buf: RgbaImage = ImageBuffer::from_raw( image_data.width, image_data.height, image_data.pixels_rgba.clone(), )?; - crate::unpremultiply_rgba(&mut img_buf); - - let mut bytes: Vec = Vec::new(); - let mut cursor = IoCursor::new(&mut bytes); - img_buf - .write_to(&mut cursor, image::ImageFormat::Png) - .ok()?; - Some((bytes, image_data.xhot, image_data.yhot)) + Some((img_buf, image_data.xhot, image_data.yhot)) } } diff --git a/pixelflux/src/wayland/dcclient.rs b/pixelflux/src/wayland/dcclient.rs new file mode 100644 index 0000000..bc442ec --- /dev/null +++ b/pixelflux/src/wayland/dcclient.rs @@ -0,0 +1,383 @@ +//! `zwlr_data_control_v1` CLIENT: the native clipboard bridge to a NESTED app +//! compositor (labwc/kwin running under pixelflux). +//! +//! Apps in a nested session use the inner compositor's selection, which +//! pixelflux's own clipboard machinery never sees. selkies bridges it over the +//! ScreenCapture ABI backed by this module instead of forking wl-copy/wl-paste +//! per operation: one-shot [`list_types`]/[`read`], a persistent-source [`write`] +//! (a detached thread serves paste requests until another client takes the +//! selection), and [`watch`] (a thread reporting selection changes to a Python +//! callback). Every compositor round-trip is deadline-bounded via +//! [`wlclient::bounded_roundtrip`]. + +use std::collections::HashMap; +use std::os::fd::AsFd; +use std::os::unix::net::UnixStream; +use std::sync::atomic::{AtomicBool, Ordering}; +use std::sync::{Arc, Mutex}; +use std::time::Duration; + +use pyo3::{Py, PyAny, Python}; +use wayland_client::backend::ObjectId; +use wayland_client::protocol::{wl_registry, wl_seat}; +use wayland_client::{delegate_noop, Connection, Dispatch, Proxy, QueueHandle}; +use wayland_protocols_wlr::data_control::v1::client::{ + zwlr_data_control_device_v1::{self, ZwlrDataControlDeviceV1}, + zwlr_data_control_manager_v1::ZwlrDataControlManagerV1, + zwlr_data_control_offer_v1::{self, ZwlrDataControlOfferV1}, + zwlr_data_control_source_v1::{self, ZwlrDataControlSourceV1}, +}; + +use crate::wayland::wlclient::{ + bounded_roundtrip, impl_sync_callback, pipe_cloexec, read_fd_to_end, wait_readable, + write_fd_all, SyncState, IO_TIMEOUT, +}; + +/// How often a background thread wakes from its socket poll to check its stop +/// flag, bounding unwatch/shutdown latency. +const STOP_POLL: Duration = Duration::from_millis(500); + +#[derive(Default)] +struct DcState { + seat: Option, + manager: Option, + /// Advertised mimes per live offer. + offer_mimes: HashMap>, + selection: Option, + /// Set on every `selection` event (the watch loop's change edge). + selection_changed: bool, + /// Compositor told this device it is done (seat gone). + finished: bool, + /// The write path's source lost the selection to another client. + cancelled: bool, + /// Mime -> bytes served by the write path's source. + serve: Vec<(String, Vec)>, + sync_done: bool, +} + +impl SyncState for DcState { + fn sync_done_mut(&mut self) -> &mut bool { + &mut self.sync_done + } +} +impl_sync_callback!(DcState); + +impl Dispatch for DcState { + fn event( + state: &mut Self, + registry: &wl_registry::WlRegistry, + event: wl_registry::Event, + _: &(), + _: &Connection, + qh: &QueueHandle, + ) { + if let wl_registry::Event::Global { name, interface, .. } = event { + // Version 1 of each suffices: the seat is only an argument, and v1 + // data-control carries the regular selection this bridge needs. + match interface.as_str() { + "wl_seat" if state.seat.is_none() => { + state.seat = Some(registry.bind(name, 1, qh, ())); + } + "zwlr_data_control_manager_v1" if state.manager.is_none() => { + state.manager = Some(registry.bind(name, 1, qh, ())); + } + _ => {} + } + } + } +} + +delegate_noop!(DcState: ignore wl_seat::WlSeat); +delegate_noop!(DcState: ZwlrDataControlManagerV1); + +impl Dispatch for DcState { + fn event( + state: &mut Self, + offer: &ZwlrDataControlOfferV1, + event: zwlr_data_control_offer_v1::Event, + _: &(), + _: &Connection, + _: &QueueHandle, + ) { + if let zwlr_data_control_offer_v1::Event::Offer { mime_type } = event { + state.offer_mimes.entry(offer.id()).or_default().push(mime_type); + } + } +} + +impl Dispatch for DcState { + fn event( + state: &mut Self, + _: &ZwlrDataControlDeviceV1, + event: zwlr_data_control_device_v1::Event, + _: &(), + _: &Connection, + _: &QueueHandle, + ) { + match event { + zwlr_data_control_device_v1::Event::DataOffer { id } => { + state.offer_mimes.entry(id.id()).or_default(); + } + zwlr_data_control_device_v1::Event::Selection { id } => { + // Replaced offers are dead objects; drop their proxy and mimes so + // a long-lived watch connection doesn't accumulate them. + if let Some(old) = state.selection.take() { + if id.as_ref().map(|o| o.id()) != Some(old.id()) { + state.offer_mimes.remove(&old.id()); + old.destroy(); + } + } + state.selection = id; + state.selection_changed = true; + } + zwlr_data_control_device_v1::Event::Finished => { + state.finished = true; + } + _ => {} + } + } + + wayland_client::event_created_child!(DcState, ZwlrDataControlDeviceV1, [ + zwlr_data_control_device_v1::EVT_DATA_OFFER_OPCODE => (ZwlrDataControlOfferV1, ()), + ]); +} + +impl Dispatch for DcState { + fn event( + state: &mut Self, + _: &ZwlrDataControlSourceV1, + event: zwlr_data_control_source_v1::Event, + _: &(), + _: &Connection, + _: &QueueHandle, + ) { + match event { + zwlr_data_control_source_v1::Event::Send { mime_type, fd } => { + if let Some((_, data)) = state.serve.iter().find(|(m, _)| *m == mime_type) { + let _ = write_fd_all(&fd, data, IO_TIMEOUT); + } + // fd drops here, closing the pipe so the paster sees EOF. + } + zwlr_data_control_source_v1::Event::Cancelled => { + state.cancelled = true; + } + _ => {} + } + } +} + +/// Connect to `socket_path` and return (connection, queue, state, device) with +/// the current selection already delivered. +fn open_device( + socket_path: &str, +) -> Result< + (Connection, wayland_client::EventQueue, DcState, ZwlrDataControlDeviceV1), + String, +> { + let stream = + UnixStream::connect(socket_path).map_err(|e| format!("connect {socket_path}: {e}"))?; + let conn = Connection::from_socket(stream).map_err(|e| format!("wayland setup: {e}"))?; + let mut queue = conn.new_event_queue(); + let qh = queue.handle(); + let _registry = conn.display().get_registry(&qh, ()); + let mut state = DcState::default(); + bounded_roundtrip(&conn, &mut queue, &mut state)?; + let seat = state.seat.clone().ok_or("app compositor advertises no wl_seat")?; + let manager = state + .manager + .clone() + .ok_or("app compositor does not advertise zwlr_data_control_manager_v1")?; + let device = manager.get_data_device(&seat, &qh, ()); + bounded_roundtrip(&conn, &mut queue, &mut state)?; + Ok((conn, queue, state, device)) +} + +/// Mimes offered by the current selection (empty when nothing is copied). +pub(crate) fn list_types(socket_path: &str) -> Result, String> { + let (_conn, _queue, state, device) = open_device(socket_path)?; + let out = state + .selection + .as_ref() + .and_then(|o| state.offer_mimes.get(&o.id()).cloned()) + .unwrap_or_default(); + device.destroy(); + Ok(out) +} + +/// The current selection's payload for `mime`, or None when there is no +/// selection or it does not offer that mime. +pub(crate) fn read(socket_path: &str, mime: &str) -> Result>, String> { + let (conn, mut queue, mut state, device) = open_device(socket_path)?; + let Some(offer) = state.selection.clone() else { + device.destroy(); + return Ok(None); + }; + let offered = state.offer_mimes.get(&offer.id()).is_some_and(|m| m.iter().any(|x| x == mime)); + if !offered { + device.destroy(); + return Ok(None); + } + let (rd, wr) = pipe_cloexec()?; + offer.receive(mime.to_string(), wr.as_fd()); + queue.flush().map_err(|e| format!("flush: {e}"))?; + drop(wr); + // The source app writes into the pipe as it pleases; dispatch is not needed + // for the bytes, only the fd read. + let data = read_fd_to_end(&rd, IO_TIMEOUT)?; + let _ = bounded_roundtrip(&conn, &mut queue, &mut state); + device.destroy(); + Ok(Some(data)) +} + +/// Take the selection, serving `entries` (mime, bytes) to every paster from a +/// detached thread until another client takes the selection (or the compositor +/// goes away). Replacing a previous write is implicit: the compositor cancels +/// the old source when the new one takes the selection. +pub(crate) fn write(socket_path: &str, entries: Vec<(String, Vec)>) -> Result<(), String> { + let path = socket_path.to_string(); + std::thread::Builder::new() + .name("pf-dc-selection".into()) + .spawn(move || { + if let Err(e) = serve_selection(&path, entries) { + eprintln!("[Clipboard] app-compositor selection serve ended: {e}"); + } + }) + .map_err(|e| format!("spawn: {e}"))?; + Ok(()) +} + +fn serve_selection(socket_path: &str, entries: Vec<(String, Vec)>) -> Result<(), String> { + let stream = + UnixStream::connect(socket_path).map_err(|e| format!("connect {socket_path}: {e}"))?; + let conn = Connection::from_socket(stream).map_err(|e| format!("wayland setup: {e}"))?; + let mut queue = conn.new_event_queue(); + let qh = queue.handle(); + let _registry = conn.display().get_registry(&qh, ()); + let mut state = DcState::default(); + bounded_roundtrip(&conn, &mut queue, &mut state)?; + let seat = state.seat.clone().ok_or("app compositor advertises no wl_seat")?; + let manager = state + .manager + .clone() + .ok_or("app compositor does not advertise zwlr_data_control_manager_v1")?; + let device = manager.get_data_device(&seat, &qh, ()); + let source = manager.create_data_source(&qh, ()); + for (mime, _) in &entries { + source.offer(mime.clone()); + } + state.serve = entries; + device.set_selection(Some(&source)); + bounded_roundtrip(&conn, &mut queue, &mut state)?; + while !state.cancelled && !state.finished { + // Sends are served inside dispatch; block until the compositor has + // something (with a poll so a dead compositor can't pin the thread). + queue.flush().map_err(|e| format!("flush: {e}"))?; + if let Some(guard) = conn.prepare_read() { + use std::os::fd::AsRawFd; + if wait_readable(guard.connection_fd().as_raw_fd(), STOP_POLL)? { + guard.read().map_err(|e| format!("read: {e}"))?; + } + } + queue.dispatch_pending(&mut state).map_err(|e| format!("dispatch: {e}"))?; + } + source.destroy(); + device.destroy(); + let _ = queue.flush(); + Ok(()) +} + +/// Drop the selection (the compositor also cancels whatever source held it). +pub(crate) fn clear(socket_path: &str) -> Result<(), String> { + let (conn, mut queue, mut state, device) = open_device(socket_path)?; + device.set_selection(None); + bounded_roundtrip(&conn, &mut queue, &mut state)?; + device.destroy(); + Ok(()) +} + +struct WatchHandle { + stop: Arc, +} + +static WATCHERS: Mutex>> = Mutex::new(None); + +/// Report every selection change on `socket_path` (including the one current at +/// start) to `callback(mimes: list[str])` from a background thread. A second +/// watch on the same socket replaces the first. +pub(crate) fn watch(socket_path: &str, callback: Py) -> Result<(), String> { + let stop = Arc::new(AtomicBool::new(false)); + { + let mut reg = WATCHERS.lock().unwrap(); + let map = reg.get_or_insert_with(HashMap::new); + if let Some(old) = map.insert(socket_path.to_string(), WatchHandle { stop: stop.clone() }) + { + old.stop.store(true, Ordering::Relaxed); + } + } + let path = socket_path.to_string(); + std::thread::Builder::new() + .name("pf-dc-watch".into()) + .spawn(move || { + if let Err(e) = watch_loop(&path, callback, &stop) { + eprintln!("[Clipboard] app-compositor watch ended: {e}"); + } + }) + .map_err(|e| format!("spawn: {e}"))?; + Ok(()) +} + +/// Stop every clipboard watch (process teardown sweep; watches are NOT tied to +/// captures, so the global stop helper must reach them explicitly). +pub(crate) fn unwatch_all() { + let mut reg = WATCHERS.lock().unwrap(); + if let Some(map) = reg.as_mut() { + for (_, handle) in map.drain() { + handle.stop.store(true, Ordering::Relaxed); + } + } +} + +/// Stop the watch on `socket_path` (no-op when none is running). +pub(crate) fn unwatch(socket_path: &str) { + let mut reg = WATCHERS.lock().unwrap(); + if let Some(map) = reg.as_mut() { + if let Some(handle) = map.remove(socket_path) { + handle.stop.store(true, Ordering::Relaxed); + } + } +} + +fn watch_loop(socket_path: &str, callback: Py, stop: &AtomicBool) -> Result<(), String> { + let (conn, mut queue, mut state, device) = open_device(socket_path)?; + while !stop.load(Ordering::Relaxed) && !state.finished { + if state.selection_changed { + state.selection_changed = false; + let mimes = state + .selection + .as_ref() + .and_then(|o| state.offer_mimes.get(&o.id()).cloned()) + .unwrap_or_default(); + if !mimes.is_empty() { + if crate::PY_SHUTDOWN.load(Ordering::Relaxed) { + break; + } + Python::attach(|py| { + if let Err(e) = callback.call1(py, (mimes,)) { + e.print(py); + } + }); + } + } + queue.flush().map_err(|e| format!("flush: {e}"))?; + if let Some(guard) = conn.prepare_read() { + use std::os::fd::AsRawFd; + if wait_readable(guard.connection_fd().as_raw_fd(), STOP_POLL)? { + guard.read().map_err(|e| format!("read: {e}"))?; + } + } + queue.dispatch_pending(&mut state).map_err(|e| format!("dispatch: {e}"))?; + } + device.destroy(); + let _ = queue.flush(); + Ok(()) +} diff --git a/pixelflux/src/wayland/frontend.rs b/pixelflux/src/wayland/frontend.rs index fd93224..512179b 100644 --- a/pixelflux/src/wayland/frontend.rs +++ b/pixelflux/src/wayland/frontend.rs @@ -20,11 +20,9 @@ use std::borrow::Cow; use std::fs::File; -use std::io::Cursor as IoCursor; use std::time::Instant; use gbm::{BufferObject, Device as RawGbmDevice}; -use image::{ImageBuffer, ImageFormat, Rgba}; use pyo3::prelude::*; use pyo3::types::PyBytes; use std::sync::Mutex; @@ -33,12 +31,19 @@ use std::hash::{Hash, Hasher}; use smithay::backend::renderer::utils::RendererSurfaceState; use smithay::backend::allocator::dmabuf::Dmabuf; use smithay::backend::allocator::{ Buffer, Fourcc}; +use smithay::backend::renderer::damage::OutputDamageTracker; use smithay::backend::renderer::{ Bind, ExportMem, gles::GlesRenderer, pixman::PixmanRenderer, ImportDma, }; use smithay::input::dnd::{DndFocus, Source}; use std::sync::Arc; -use crate::wayland::cursor::Cursor; +use crate::wayland::cursor::{Cursor, CursorJob}; +use crate::wayland::keymap::KeymapPolicy; +use smithay::reexports::wayland_protocols_misc::zwp_virtual_keyboard_v1::server::{ + zwp_virtual_keyboard_manager_v1::{self, ZwpVirtualKeyboardManagerV1}, + zwp_virtual_keyboard_v1::{self, ZwpVirtualKeyboardV1}, +}; +use smithay::reexports::wayland_server::{DataInit, Dispatch, GlobalDispatch, New}; use smithay::wayland::viewporter::ViewporterState; use smithay::delegate_viewporter; use smithay::wayland::pointer_warp::{PointerWarpHandler, PointerWarpManager}; @@ -78,8 +83,8 @@ use smithay::delegate_primary_selection; use smithay::{ delegate_compositor, delegate_data_device, delegate_dmabuf, delegate_fractional_scale, - delegate_output, delegate_seat, delegate_shm, delegate_virtual_keyboard_manager, - delegate_xdg_shell, delegate_relative_pointer, delegate_pointer_warp, + delegate_output, delegate_seat, delegate_shm, + delegate_xdg_shell, delegate_relative_pointer, delegate_pointer_warp, delegate_pointer_constraints, desktop::{Space, Window}, input::{ @@ -97,7 +102,7 @@ use smithay::{ reexports::{ wayland_protocols::xdg::shell::server::xdg_toplevel::State as XdgState, wayland_server::{ - backend::{ClientData, ClientId, DisconnectReason, ObjectId}, + backend::{ClientData, ClientId, DisconnectReason, GlobalId, ObjectId}, protocol::{wl_buffer::WlBuffer, wl_surface::WlSurface}, Client, DisplayHandle, Resource, }, @@ -115,8 +120,8 @@ use smithay::{ seat::WaylandFocus, selection::{ data_device::{ - request_data_device_client_selection, DataDeviceHandler, DataDeviceState, - WaylandDndGrabHandler, + request_data_device_client_selection, set_data_device_focus, DataDeviceHandler, + DataDeviceState, WaylandDndGrabHandler, }, SelectionHandler, SelectionSource, SelectionTarget, }, @@ -125,7 +130,6 @@ use smithay::{ XdgToplevelSurfaceData, }, shm::{with_buffer_contents, ShmHandler, ShmState, BufferAccessError}, - virtual_keyboard::VirtualKeyboardManagerState, }, }; @@ -134,7 +138,7 @@ use crate::encoders::vaapi::VaapiEncoder; use crate::nvenc::NvencEncoder; use crate::{RustCaptureSettings, StripeState}; -use std::sync::atomic::{AtomicU32, Ordering}; +use std::sync::atomic::{AtomicBool, AtomicU32, Ordering}; static SERIAL_COUNTER: AtomicU32 = AtomicU32::new(1); @@ -193,14 +197,151 @@ pub enum GpuEncoder { Nvenc(NvencEncoder), } +/// One capture pipeline bound to one output (display id): its settings, encoder set, +/// frame pools, delivery thread, and per-stream bookkeeping. Exactly one capture may run per +/// output; all fields mirror the pipeline strategy documented on [`AppState`], instantiated +/// per display. +pub struct WlCapture { + pub settings: RustCaptureSettings, + /// Zero-copy GPU session (GLES render + same-GPU dmabuf encode), calloop-affine; + /// `None` whenever a readback path is active for this display. + pub video_encoder: Option, + pub vaapi_state: StripeState, + pub recording_sink: Option>, + pub deliver_tx: Option>>, + pub deliver_join: Option>, + pub pending_hw_delivery: Option>, + pub pending_hw_damage: bool, + pub encode_pool: Option>, + pub encode_join: Option>>, + pub encode_controls: Arc, + pub encode_stats: Arc, + pub pool_last_render: Vec, + pub render_seq: u64, + pub pool_content_gen: Vec, + pub content_gen: u64, + pub frame_counter: u16, + pub pending_force_idr: bool, + pub needs_full_render: bool, + /// Last tick this capture actually rendered; paces per-display fps under the one + /// shared render timer (which fires at the fastest active capture's rate). + pub last_tick: Option, + /// Consecutive zero-copy encode failures; reset by any frame that encodes cleanly. + /// Reaching `HW_ERROR_RECOVERY_THRESHOLD` triggers recovery. + pub hw_error_streak: u32, + /// Whether the zero-copy session has already been rebuilt without a clean frame since. + /// Recovery rebuilds once and demotes to readback on the next streak, so a session that + /// constructs but never encodes cannot loop rebuilding forever. + pub hw_rebuilt: bool, +} + +impl WlCapture { + /// Arm a keyframe on whichever path is live, plus a full render so a static + /// screen still produces the frame. Exactly ONE flag is set: the pool encode + /// loop consumes the atomic, every other path consumes `pending_force_idr` — + /// setting both would leave one armed forever (the host-mode idle gate polls + /// them every tick, so a stuck flag disables idle skipping for the session). + pub fn request_idr(&mut self) { + if self.encode_pool.is_some() { + self.encode_controls.force_idr.store(true, Ordering::Relaxed); + } else { + self.pending_force_idr = true; + } + self.needs_full_render = true; + } +} + +/// One virtual output and everything sized to it: the Smithay `Output` + its advertised +/// global, its layout position, the damage tracker, render targets, and the (at most one) +/// capture bound to it. `id` is the Python-facing display key; id 0 is the primary +/// HEADLESS-1 output, which is never destroyed. +pub struct OutputNode { + pub id: u32, + pub output: Output, + pub global: GlobalId, + /// Layout offset: the output's logical position in the Space AND its physical offset + /// for absolute input injection. The two coincide at scale 1; with mixed scales the + /// caller must place outputs so neither the logical nor the physical rectangles + /// overlap. + pub pos: (i32, i32), + pub damage_tracker: OutputDamageTracker, + /// Host-side scratch target: pixman throttle path renders here, GLES screenshots read + /// back here. + pub frame_buffer: Vec, + /// GPU render target for this output (GLES mode): the GBM BO and its dmabuf export. + pub offscreen_buffer: Option<(BufferObject<()>, Dmabuf)>, + /// Watermark overlay for THIS output: loaded at the output's scale, positioned (and, + /// for the bouncing anchor, animated) against the output's own frame dimensions. + pub overlay_state: OverlayState, + pub capture: Option, +} + +impl OutputNode { + /// The output's logical geometry in layout coordinates: origin plus mode/scale-derived + /// logical size. + pub fn logical_geometry(&self) -> Option> { + let mode = self.output.current_mode()?; + let scale = self.output.current_scale().fractional_scale(); + Some(Rectangle::new( + Point::from(self.pos), + ( + (mode.size.w as f64 / scale).round() as i32, + (mode.size.h as f64 / scale).round() as i32, + ) + .into(), + )) + } +} + +static NEXT_WINDOW_ID: AtomicU32 = AtomicU32::new(1); + +/// Per-window bookkeeping carried in the window's user-data map: a stable numeric id the +/// Python side addresses the window by, the display id of the output it is placed on, and +/// whether that output has been chosen yet. +pub struct WindowMeta { + pub id: u32, + pub output: AtomicU32, + /// Set once the first commit has picked the window's output. The choice cannot key off + /// xdg's `initial_configure_sent`: a client that negotiates xdg-decoration (every + /// wlroots-based nested compositor, GTK and Qt) is answered with a configure before it + /// ever commits, which would leave every window on the pointer's output. + pub placed: AtomicBool, + /// Set while the window is mapped outside every output, tagged to the output it will + /// take once one exists. See `park_window`. + pub parked: AtomicBool, +} + +/// Where a parked window is mapped: clear of every output, so it is composited into none of +/// them. Layout offsets are non-negative — the union layout Selkies computes re-anchors at +/// the origin — so far negative coordinates can never collide with a real output. +pub const PARKED_POS: (i32, i32) = (-(1 << 20), -(1 << 20)); + +/// The window's meta, inserted at `new_toplevel`; windows created before that (none in +/// practice) read as id 0 / primary output. +pub fn window_meta(window: &Window) -> Option<&WindowMeta> { + window.user_data().get::() +} + +/// The display id of the output this window is placed on (primary when untagged). +pub fn window_output_id(window: &Window) -> u32 { + window_meta(window).map(|m| m.output.load(Ordering::Relaxed)).unwrap_or(0) +} + +/// A queued computer-use screenshot as `(display id, reply)`; the reply carries the +/// encoded image or the reason it could not be produced. +pub type ScreenshotRequest = (u32, std::sync::mpsc::Sender, String>>); + /// Central context threaded through every Smithay handler; owns the Wayland globals, the /// GBM/EGL (or pixman) renderer state, and the capture/encode pipeline state. /// /// A single `AppState` lives for the whole compositor thread and is mutated in place by the -/// calloop event sources (client dispatch, input injection, the capture timer). The non-obvious -/// fields encode the pipeline's threading and buffer strategy: +/// calloop event sources (client dispatch, input injection, the capture timer). The frame +/// pipeline is instantiated PER DISPLAY: each `OutputNode` in `output_nodes` owns its output, +/// damage tracker, and render targets, and its optional `WlCapture` owns that display's +/// encoder set and delivery. The non-obvious `WlCapture`/`OutputNode` fields encode the +/// pipeline's threading and buffer strategy: /// -/// 1. **Render / encode targets**: +/// 1. **Render / encode targets** (per display): /// - **`video_encoder`**: the zero-copy GPU session only (GLES render + same-GPU dmabuf /// encode). Its EGL/dmabuf handles are calloop-affine, so this encoder runs inline on the /// calloop thread; it is `None` whenever a readback path is active. @@ -255,7 +396,9 @@ pub struct AppState { pub dh: DisplayHandle, #[allow(dead_code)] pub seat: Seat, - pub outputs: Vec, + /// Every live output with its per-display render/capture state; index 0 is the + /// primary (display id 0), which is never destroyed. + pub output_nodes: Vec, pub pending_windows: Vec, pub foreign_toplevel_list: ForeignToplevelListState, @@ -263,69 +406,57 @@ pub struct AppState { pub xdg_activation_state: XdgActivationState, pub primary_selection_state: PrimarySelectionState, pub popups: PopupManager, - pub frame_buffer: Vec, pub gles_renderer: Option, pub pixman_renderer: Option, pub gbm_device: Option>, - pub offscreen_buffer: Option<(BufferObject<()>, Dmabuf)>, - pub is_capturing: bool, + /// Mirror of the PRIMARY display's capture settings (geometry fallbacks, computer-use + /// info); per-display settings live on each capture. pub settings: RustCaptureSettings, - pub callback: Option>, - pub cursor_callback: Option>, + /// A cursor callback is registered on the `wl-cursor` worker (which owns the actual + /// `Py` object); tracked here so sprite resolution is skipped while nobody listens. + pub cursor_callback_set: bool, + /// Cursor delivery jobs to the `wl-cursor` worker (PNG encode + Python call off-thread). + pub cursor_tx: std::sync::mpsc::Sender, pub clipboard_callback: Option>, pub pending_clipboard_read: Option, + /// Preferred mime of the current CLIENT-owned clipboard selection, recorded even while + /// no callback is registered so `SetClipboardCallback` can re-stage a read of a copy made + /// in the gap; `None` when the selection is cleared or compositor-owned. + pub current_selection_mime: Option, pub last_log_time: Instant, pub start_time: Instant, pub clock: Clock, - pub frame_counter: u16, - pub pending_force_idr: bool, - pub needs_full_render: bool, pub use_gpu: bool, - pub video_encoder: Option, - pub vaapi_state: StripeState, pub cursor_helper: Cursor, - pub overlay_state: OverlayState, - - pub virtual_keyboard_state: VirtualKeyboardManagerState, + /// Seat keymap owner: base layout plus batched overlay binds. Every seat keymap swap + /// flows through this policy, so keymap identity has exactly one writer. + pub keymap_policy: KeymapPolicy, + /// Host-capture session when pixelflux captures an EXTERNAL compositor: + /// frames arrive by screencopy and input routes to its virtual devices. + pub host: Option, pub current_cursor_icon: Option, pub cursor_buffer: Option, - pub cursor_cache: std::collections::HashMap>, pub render_cursor_on_framebuffer: bool, pub pointer_warp_state: PointerWarpManager, pub relative_pointer_state: RelativePointerManagerState, pub pointer_constraints_state: PointerConstraintsState, pub render_node_path: String, pub auto_gpu_selected: bool, - pub recording_sink: Option>, - pub deliver_tx: Option>>, - pub deliver_join: Option>, - /// Zero-copy frame parked after a full delivery channel. The hardware encode and - /// its delivery both run on the calloop thread, and a blocking send there would - /// freeze input/command/Wayland dispatch for as long as the Python consumer - /// stalls. An encoded frame is part of the H.264 reference chain and can never - /// be dropped, so it parks here instead, and new encodes pause until it leaves. - pub pending_hw_delivery: Option>, - /// Damage seen on ticks skipped while `pending_hw_delivery` was parked, folded - /// into the next encode's change detection so a change that happened during the - /// pause is never lost (each tick's damage list is otherwise discarded). - pub pending_hw_damage: bool, - pub encode_pool: Option>, - pub encode_join: Option>>, - pub encode_controls: Arc, - pub encode_stats: Arc, - pub pool_last_render: Vec, - pub render_seq: u64, - pub pool_content_gen: Vec, - pub content_gen: u64, - pub pending_screenshot: Option>>, + /// Computer-use screenshot request; served from that output's next render (the id + /// was validated live when the request was queued). + pub pending_screenshot: Option, + /// The command channel, drained in place (wakeups arrive on a separate ping channel) so + /// the render tick can apply every queued command BEFORE starting a long render/encode — + /// queued input is never starved behind the tick it arrived during. + pub command_rx: Option>, } /// Pointer-constraints protocol wiring. The headless capture path never enforces a lock or @@ -440,9 +571,9 @@ impl WlrLayerShellHandler for AppState { namespace: String, ) { let smithay_output = if let Some(wlo) = output.as_ref() { - self.outputs.iter().find(|o| o.owns(wlo)) + self.output_nodes.iter().map(|n| &n.output).find(|o| o.owns(wlo)) } else { - self.outputs.first() + self.primary_output() }; if let Some(output) = smithay_output { @@ -505,17 +636,12 @@ impl CompositorHandler for AppState { fn commit(&mut self, surface: &WlSurface) { smithay::backend::renderer::utils::on_commit_buffer_handler::(surface); - if let Some(output) = self.outputs.first() { - let mut layer_map = layer_map_for_output(output); - let mut found = false; - for layer in layer_map.layers() { - if layer.wl_surface() == surface { - found = true; - break; - } - } + for node in &self.output_nodes { + let mut layer_map = layer_map_for_output(&node.output); + let found = layer_map.layers().any(|layer| layer.wl_surface() == surface); if found { layer_map.arrange(); + break; } } @@ -541,12 +667,26 @@ impl CompositorHandler for AppState { } } - if let Some(window) = self + let mapped = self .space .elements() .find(|w| w.toplevel().map(|tl| tl.wl_surface() == surface).unwrap_or(false)) - { + .cloned(); + if let Some(window) = mapped { window.on_commit(); + // A null-buffer commit unmaps the toplevel (xdg-shell): purge it from the + // space so it no longer lists or renders, and re-queue it so a client that + // maps again goes back through the configure handshake. + let has_buffer = smithay::backend::renderer::utils::with_renderer_surface_state( + surface, + |s| s.buffer().is_some(), + ) + .unwrap_or(false); + if !has_buffer { + self.space.unmap_elem(&window); + self.pending_windows.push(window); + return; + } } if let Some(idx) = self.pending_windows.iter().position(|w| { @@ -555,31 +695,52 @@ impl CompositorHandler for AppState { let window = self.pending_windows.remove(idx); let toplevel = window.toplevel().unwrap(); - let initial_configure_sent = with_states(surface, |states| { - states - .data_map - .get::() - .unwrap() - .lock() - .unwrap() - .initial_configure_sent - }); - - if !initial_configure_sent { - let (logical_width, logical_height) = if let Some(output) = self.outputs.first() { - let mode = output.current_mode().unwrap(); - let scale = output.current_scale().fractional_scale(); - ( - (mode.size.w as f64 / scale).round() as i32, - (mode.size.h as f64 / scale).round() as i32, - ) - } else { - let scale = self.settings.scale.max(0.1); - ( - (self.settings.width as f64 / scale).round() as i32, - (self.settings.height as f64 / scale).round() as i32, - ) - }; + // Whether this window still needs an output. Deliberately not xdg's + // `initial_configure_sent`: the decoration handshake answers with a configure + // before the client's first commit, which would skip the choice below for + // every client that negotiates decorations. + let needs_placement = window_meta(&window) + .map(|meta| !meta.placed.swap(true, Ordering::Relaxed)) + .unwrap_or(false); + + if needs_placement { + // A new toplevel opens fullscreened on the output the pointer is on + // (primary when indeterminate); the choice is pinned on the window's + // meta so the acked commit maps to the same output. + let mut target_id = self.pointer_display(); + // A second fullscreen surface from a client that already owns one on + // the target output is screen-like (a nested compositor opens one + // host toplevel per screen): give it an empty output when one exists, + // and park it otherwise rather than let it cover the screen already + // there. `create_output` hands it the output it is waiting for. + let mut park = false; + if self.would_cover_screen(&window, target_id) { + let empty = self.output_nodes.iter().map(|n| n.id).find(|oid| { + !self + .space + .elements() + .chain(self.pending_windows.iter()) + .any(|w| window_output_id(w) == *oid) + }); + match empty { + Some(oid) => target_id = oid, + None => park = true, + } + } + if let Some(meta) = window_meta(&window) { + meta.output.store(target_id, Ordering::Relaxed); + meta.parked.store(park, Ordering::Relaxed); + } + let (logical_width, logical_height) = + if let Some(size) = self.logical_size_of(target_id) { + size + } else { + let scale = self.settings.scale.max(0.1); + ( + (self.settings.width as f64 / scale).round() as i32, + (self.settings.height as f64 / scale).round() as i32, + ) + }; toplevel.with_pending_state(|state| { state.states.set(XdgState::Activated); @@ -589,35 +750,68 @@ impl CompositorHandler for AppState { toplevel.send_configure(); self.pending_windows.push(window); + } else if smithay::backend::renderer::utils::with_renderer_surface_state( + surface, + |s| s.buffer().is_none(), + ) + .unwrap_or(true) + { + // Configured but still buffer-less (a decoration-triggered configure, an + // ack-only commit, or a remap after a null-buffer unmap — xdg + // initial_configure_sent stays true there): keep waiting; mapping now would + // list and hit-test a phantom window. Answer with the forced-fullscreen + // configure so the client draws its first buffer at the right geometry. + let tl = toplevel.clone(); + self.pending_windows.push(window); + self.send_forced_fullscreen_configure(&tl); } else { - self.space.map_element(window.clone(), (0, 0), true); + let target_id = window_output_id(&window); + let parked = window_meta(&window) + .map(|meta| meta.parked.load(Ordering::Relaxed)) + .unwrap_or(false); + let node_idx = self.node_idx_for_id(target_id).unwrap_or(0); + let (target_output, pos) = { + let node = &self.output_nodes[node_idx]; + (node.output.clone(), node.pos) + }; + // A parked window enters no output and takes no focus: it is waiting for + // an output of its own, and holds its size until one arrives. + self.space.map_element( + window.clone(), + if parked { PARKED_POS } else { pos }, + !parked, + ); window.on_commit(); - if let Some(output) = self.outputs.first() { - output.enter(surface); - - let mode = output.current_mode().unwrap(); - let scale = output.current_scale().fractional_scale(); - let (expected_w, expected_h) = ( - (mode.size.w as f64 / scale).round() as i32, - (mode.size.h as f64 / scale).round() as i32, - ); - - let geo = window.geometry(); - if (geo.size.w - expected_w).abs() > 1 || (geo.size.h - expected_h).abs() > 1 { - toplevel.with_pending_state(|state| { - state.states.set(XdgState::Activated); - state.states.set(XdgState::Fullscreen); - state.size = Some((expected_w, expected_h).into()); + if !parked { + target_output.enter(surface); + let scale = target_output.current_scale().fractional_scale(); + with_states(surface, |states| { + smithay::wayland::fractional_scale::with_fractional_scale(states, |fs| { + fs.set_preferred_scale(scale); }); - toplevel.send_configure(); + }); + + if let Some(geo_out) = self.output_nodes[node_idx].logical_geometry() { + let (expected_w, expected_h) = (geo_out.size.w, geo_out.size.h); + let geo = window.geometry(); + if (geo.size.w - expected_w).abs() > 1 + || (geo.size.h - expected_h).abs() > 1 + { + toplevel.with_pending_state(|state| { + state.states.set(XdgState::Activated); + state.states.set(XdgState::Fullscreen); + state.size = Some((expected_w, expected_h).into()); + }); + toplevel.send_configure(); + } } - } - let serial = next_serial(); - let target = FocusTarget::Window(window.clone()); - if let Some(keyboard) = self.seat.get_keyboard() { - keyboard.set_focus(self, Some(target.clone()), serial); + let serial = next_serial(); + let target = FocusTarget::Window(window.clone()); + if let Some(keyboard) = self.seat.get_keyboard() { + keyboard.set_focus(self, Some(target.clone()), serial); + } } } } @@ -626,18 +820,179 @@ impl CompositorHandler for AppState { impl AppState { + /// The primary (display id 0) output. + pub(crate) fn primary_output(&self) -> Option<&Output> { + self.output_nodes.first().map(|n| &n.output) + } + + pub(crate) fn node_idx_for_id(&self, id: u32) -> Option { + self.output_nodes.iter().position(|n| n.id == id) + } + + /// Index of the node whose LOGICAL rect contains `p`. + pub(crate) fn node_idx_under(&self, p: Point) -> Option { + self.output_nodes.iter().position(|n| { + n.logical_geometry() + .map(|g| g.to_f64().contains(p)) + .unwrap_or(false) + }) + } + + /// Map absolute PHYSICAL union-layout coordinates to a logical layout point: each + /// output occupies the physical rectangle at its layout offset sized by its mode; the + /// point is clamped into the nearest output when it falls outside all of them, so the + /// pointer can never leave the layout. + pub(crate) fn layout_physical_to_logical(&self, x: f64, y: f64) -> Point { + let mut best: Option<(f64, Point)> = None; + for node in &self.output_nodes { + let Some(mode) = node.output.current_mode() else { continue }; + let scale = node.output.current_scale().fractional_scale(); + let (px, py) = (node.pos.0 as f64, node.pos.1 as f64); + let cx = x.max(px).min(px + mode.size.w as f64 - 1.0); + let cy = y.max(py).min(py + mode.size.h as f64 - 1.0); + let d2 = (x - cx).powi(2) + (y - cy).powi(2); + let logical = Point::from(( + node.pos.0 as f64 + (cx - px) / scale, + node.pos.1 as f64 + (cy - py) / scale, + )); + if best.as_ref().map(|(bd, _)| d2 < *bd).unwrap_or(true) { + best = Some((d2, logical)); + } + } + best.map(|(_, p)| p).unwrap_or_else(|| (0.0, 0.0).into()) + } + + /// Clamp a logical layout point into the nearest output's logical rectangle. + pub(crate) fn clamp_logical(&self, p: Point) -> Point { + let mut best: Option<(f64, Point)> = None; + for node in &self.output_nodes { + let Some(geo) = node.logical_geometry() else { continue }; + let scale = node.output.current_scale().fractional_scale(); + let g = geo.to_f64(); + let margin = 1.0 / scale.max(0.1); + let cx = p.x.max(g.loc.x).min(g.loc.x + g.size.w - margin); + let cy = p.y.max(g.loc.y).min(g.loc.y + g.size.h - margin); + let d2 = (p.x - cx).powi(2) + (p.y - cy).powi(2); + if best.as_ref().map(|(bd, _)| d2 < *bd).unwrap_or(true) { + best = Some((d2, (cx, cy).into())); + } + } + best.map(|(_, p)| p).unwrap_or(p) + } + + /// Logical layout point -> physical union-layout coordinates (inverse of + /// `layout_physical_to_logical` for in-bounds points; primary-relative otherwise). + pub(crate) fn layout_logical_to_physical(&self, p: Point) -> (f64, f64) { + let idx = self.node_idx_under(p).unwrap_or(0); + let Some(node) = self.output_nodes.get(idx) else { return (p.x, p.y) }; + let scale = node.output.current_scale().fractional_scale(); + ( + node.pos.0 as f64 + (p.x - node.pos.0 as f64) * scale, + node.pos.1 as f64 + (p.y - node.pos.1 as f64) * scale, + ) + } + + /// Logical size of the given display's output. + pub(crate) fn logical_size_of(&self, id: u32) -> Option<(i32, i32)> { + let idx = self.node_idx_for_id(id)?; + let geo = self.output_nodes[idx].logical_geometry()?; + Some((geo.size.w, geo.size.h)) + } + + /// The display id under the pointer (primary when indeterminate). + pub(crate) fn pointer_display(&self) -> u32 { + self.seat + .get_pointer() + .map(|p| p.current_location()) + .and_then(|pos| self.node_idx_under(pos)) + .map(|idx| self.output_nodes[idx].id) + .unwrap_or(0) + } + + /// Whether `window` would land on top of a screen another window from the same client + /// already occupies. A nested session opens one host toplevel per screen, and a second + /// one covering the first would replace what the display shows rather than add to it. + pub(crate) fn would_cover_screen(&self, window: &Window, id: u32) -> bool { + let Some(client) = window.wl_surface().and_then(|s| s.client()) else { return false }; + self.space + .elements() + .chain(self.pending_windows.iter()) + .filter(|w| !std::ptr::eq(*w, window) && w.wl_surface() != window.wl_surface()) + .any(|w| { + window_output_id(w) == id + && !window_meta(w).map(|m| m.parked.load(Ordering::Relaxed)).unwrap_or(false) + && w.wl_surface() + .and_then(|s| s.client()) + .is_some_and(|c| c.id() == client.id()) + }) + } + + /// Map `window` clear of every output while keeping it tagged for `id`, so it holds its + /// size and its place in the window list without being composited anywhere. A nested + /// session's spare screens wait here until `create_output` gives them one. + pub(crate) fn park_window(&mut self, window: &Window, id: u32) { + let old_id = window_output_id(window); + if let Some(meta) = window_meta(window) { + meta.output.store(id, Ordering::Relaxed); + meta.parked.store(true, Ordering::Relaxed); + } + if let (Some(surface), Some(idx)) = (window.wl_surface(), self.node_idx_for_id(old_id)) { + self.output_nodes[idx].output.leave(&surface); + } + self.space.map_element(window.clone(), PARKED_POS, false); + } + + /// Place `window` on output `id`: retag its meta, remap it at the output's layout + /// origin, move output enter/leave, push the output's fractional scale, and send the + /// forced-fullscreen configure at that output's logical size. + pub(crate) fn place_window_on_output(&mut self, window: &Window, id: u32) -> bool { + let Some(idx) = self.node_idx_for_id(id) else { return false }; + let old_id = window_output_id(window); + let (new_output, pos) = { + let node = &self.output_nodes[idx]; + (node.output.clone(), node.pos) + }; + let old_output = self + .node_idx_for_id(old_id) + .map(|i| self.output_nodes[i].output.clone()); + if let Some(meta) = window_meta(window) { + meta.output.store(id, Ordering::Relaxed); + meta.parked.store(false, Ordering::Relaxed); + } + self.space.map_element(window.clone(), pos, true); + if let Some(surface) = window.wl_surface() { + if let Some(old) = old_output { + if old_id != id { + old.leave(&surface); + } + } + new_output.enter(&surface); + let scale = new_output.current_scale().fractional_scale(); + with_states(&surface, |states| { + smithay::wayland::fractional_scale::with_fractional_scale(states, |fs| { + fs.set_preferred_scale(scale); + }); + }); + } + if let Some(toplevel) = window.toplevel() { + let toplevel = toplevel.clone(); + self.send_forced_fullscreen_configure(&toplevel); + } + true + } + /// Drain a clipboard read staged by `new_selection` and hand `(mime, bytes)` to the /// Python callback off-thread. /// /// Runs from the loop *after* the dispatch that stored the new client source, so the request /// targets the current selection rather than the previous one. It clones the callback, opens a /// pipe, and asks the owning client source to write the chosen mime into the pipe's writer. A - /// spawned reader thread then reads the response — capped at 64 MiB so a hostile client cannot - /// balloon memory, and bounded by a 10 s poll deadline so a client that takes the selection but - /// never writes nor closes its fd cannot pin the thread forever (each clipboard change would - /// otherwise leak one zombie thread + pipe) — and, unless the interpreter is finalizing, - /// delivers the bytes to Python. The `PY_SHUTDOWN` checks keep this off a shutting-down - /// interpreter. + /// spawned reader thread then reads the response. The overall bound is by SIZE (64 MiB, then + /// delivered truncated) so a hostile client cannot balloon memory; time only bounds + /// INACTIVITY — a producer that keeps bytes flowing may take as long as it needs (a large + /// transfer from a slow source still delivers), while one that goes silent for 10 s without + /// closing its fd is dropped so each clipboard change cannot leak a pinned thread + pipe. + /// The `PY_SHUTDOWN` checks keep this off a shutting-down interpreter. pub(crate) fn process_pending_clipboard_read(&mut self) { let Some(mime) = self.pending_clipboard_read.take() else { return }; if crate::PY_SHUTDOWN.load(std::sync::atomic::Ordering::Relaxed) { @@ -660,12 +1015,14 @@ impl AppState { use std::io::Read; use std::os::fd::AsRawFd; const CAP: usize = 64 * 1024 * 1024; - const DEADLINE: std::time::Duration = std::time::Duration::from_secs(10); - let start = Instant::now(); + const IDLE_DEADLINE: std::time::Duration = std::time::Duration::from_secs(10); + let mut last_data = Instant::now(); let mut buf = Vec::new(); let mut chunk = [0u8; 65536]; loop { - let Some(remaining) = DEADLINE.checked_sub(start.elapsed()) else { return }; + let Some(remaining) = IDLE_DEADLINE.checked_sub(last_data.elapsed()) else { + return; + }; let mut pfd = libc::pollfd { fd: reader.as_raw_fd(), events: libc::POLLIN, @@ -680,11 +1037,12 @@ impl AppState { return; } if ready == 0 { - return; + continue; } match (&reader).read(&mut chunk) { Ok(0) => break, Ok(n) => { + last_data = Instant::now(); let room = CAP - buf.len(); let take_n = n.min(room); buf.extend_from_slice(&chunk[..take_n]); @@ -711,253 +1069,244 @@ impl AppState { } - /// Resolve a `CursorImageStatus` to PNG bytes plus hotspot and invoke the Python cursor - /// callback. Also re-invoked from the calloop command handlers to replay the retained cursor - /// when a callback re-registers or a capture restarts. + /// Resolve a `CursorImageStatus` into a job for the `wl-cursor` worker, which does the + /// PNG encode, caching, and the GIL-bound Python call off the calloop thread. Also re-invoked + /// from the calloop command handlers to replay the retained cursor when a callback + /// (re)registers or a capture restarts. /// - /// Dispatches on the cursor status: + /// Only the renderer/surface-affine work happens here: /// - /// 1. **Named**: look the themed cursor up by CSS name and emit its cached PNG (`"png"`), or - /// `"error"` if the theme lacks it. - /// 2. **Hidden**: emit `"hide"` — the only message allowed through with empty data, since an - /// intentional pointer hide must blank the consumer's cursor. - /// 3. **Surface** (a client-supplied cursor sprite): ignore any surface without the + /// 1. **Named** / **Hidden**: forwarded as-is (the worker owns its own theme handle). + /// 2. **Surface** (a client-supplied cursor sprite): ignore any surface without the /// `cursor_image` role, read the hotspot, then read the backing buffer by one of two paths: /// - **SHM**: hash only the sprite's sub-region — width/height/stride/offset/format plus the /// pixel span — because many sprites share one pool and differ only by `offset`, so hashing - /// the whole pool would collide. On a cache miss (and only when ≤128×128) convert the - /// BGRA/XRGB pixels to RGBA, un-premultiply (wl_shm cursor content is premultiplied by - /// convention; PNG carries straight alpha), and PNG-encode; `Xrgb8888` has no alpha so - /// byte 3 is forced opaque, and stride/offset are clamped non-negative with checked - /// arithmetic so a garbage descriptor skips pixels instead of panicking. + /// the whole pool would collide; ship the raw pool bytes plus descriptor to the worker. /// - **dmabuf** (`NotManaged`): bind it to the GLES renderer, copy the framebuffer to - /// `Abgr8888`, map it back, un-premultiply, and PNG-encode using the derived readback - /// stride. - /// The PNG cache is bounded at 100 entries by arbitrary eviction; content-hashing means an - /// evicted sprite simply re-inserts on the next render. + /// `Abgr8888`, map it back (calloop-affine), and ship the raw RGBA readback. /// - /// A surface whose buffer could not be read yields `("surface", empty)`, which the final gate - /// suppresses — Python is called only for non-empty data or `"hide"` — so a transient read miss - /// (common during a stop/start replay) preserves the consumer's last cursor instead of blanking - /// it. `PY_SHUTDOWN` gates the whole call off a finalizing interpreter. + /// A sprite whose buffer could not be read is dropped by the worker, preserving the + /// consumer's last cursor instead of blanking it (only "hide" carries empty data). pub(crate) fn send_cursor_image(&mut self, image: &CursorImageStatus) { - if crate::PY_SHUTDOWN.load(std::sync::atomic::Ordering::Relaxed) { + if !self.cursor_callback_set { return; } - if let Some(ref cb) = self.cursor_callback { - let (msg_type, data, hot_x, hot_y) = match image { - CursorImageStatus::Named(icon) => { - self.cursor_buffer = None; - let name = cursor_icon_to_str(icon); - if let Some((png_bytes, x, y)) = self.cursor_helper.get_png_data(name) { - ("png", png_bytes, x as i32, y as i32) - } else { - ("error", Vec::new(), 0, 0) + let job = match image { + CursorImageStatus::Named(icon) => { + self.cursor_buffer = None; + CursorJob::Named { name: cursor_icon_to_str(icon) } + } + CursorImageStatus::Hidden => { + self.cursor_buffer = None; + CursorJob::Hide + } + CursorImageStatus::Surface(ref surface) => { + let mut hot_x = 0; + let mut hot_y = 0; + let mut is_cursor_role = false; + + with_states(surface, |states| { + if states.role == Some("cursor_image") { + is_cursor_role = true; + } + if let Some(attributes) = states.data_map.get::>() { + if let Ok(guard) = attributes.lock() { + hot_x = guard.hotspot.x; + hot_y = guard.hotspot.y; + } } + }); + + if !is_cursor_role { + return; } - CursorImageStatus::Hidden => { - self.cursor_buffer = None; - ("hide", Vec::new(), 0, 0) - }, - CursorImageStatus::Surface(ref surface) => { - let mut final_png = Vec::new(); - let mut hot_x = 0; - let mut hot_y = 0; - let mut is_cursor_role = false; - with_states(surface, |states| { - if states.role == Some("cursor_image") { - is_cursor_role = true; - } - if let Some(attributes) = states.data_map.get::>() { - if let Ok(guard) = attributes.lock() { - hot_x = guard.hotspot.x; - hot_y = guard.hotspot.y; - } - } - }); + let buffer_found = with_states(surface, |states| { + let mut attrs = states.cached_state.get::(); - if !is_cursor_role { - return; + if let Some(BufferAssignment::NewBuffer(b)) = &attrs.current().buffer { + return Some(b.clone()); } - let buffer_found = with_states(surface, |states| { - let mut attrs = states.cached_state.get::(); - - if let Some(BufferAssignment::NewBuffer(b)) = &attrs.current().buffer { - return Some(b.clone()); - } - - if let Some(mutex) = states.data_map.get::>() { - if let Ok(renderer_state) = mutex.try_lock() { - if let Some(b) = renderer_state.buffer() { - let wl_buffer: &wayland_server::protocol::wl_buffer::WlBuffer = b; - return Some(wl_buffer.clone()); - } + if let Some(mutex) = states.data_map.get::>() { + if let Ok(renderer_state) = mutex.try_lock() { + if let Some(b) = renderer_state.buffer() { + let wl_buffer: &wayland_server::protocol::wl_buffer::WlBuffer = b; + return Some(wl_buffer.clone()); } } - None - }); - - if let Some(buffer) = buffer_found { - let shm_result = with_buffer_contents(&buffer, |ptr, len, spec| { - let slice = unsafe { std::slice::from_raw_parts(ptr, len) }; - let mut hasher = DefaultHasher::new(); - spec.width.hash(&mut hasher); - spec.height.hash(&mut hasher); - spec.stride.hash(&mut hasher); - spec.offset.hash(&mut hasher); - spec.format.hash(&mut hasher); - let start = (spec.offset.max(0) as usize).min(len); - let span = (spec.stride.max(0) as usize) - .saturating_mul(spec.height.max(0) as usize); - let end = start.saturating_add(span).min(len); - slice[start..end].hash(&mut hasher); - let hash = hasher.finish(); - (hash, spec.width, spec.height, spec.stride, spec.format, spec.offset, slice.to_vec()) - }); + } + None + }); - match shm_result { - Ok((hash, width, height, stride, format, buf_offset, raw_bytes)) => { - if let Some(cached_png) = self.cursor_cache.get(&hash) { - final_png = cached_png.clone(); - } else { - if width <= 128 && height <= 128 && !raw_bytes.is_empty() { - let mut img_buf = ImageBuffer::, Vec>::new(width as u32, height as u32); - let stride_usize = stride.max(0) as usize; - let base_offset = buf_offset.max(0) as usize; - - for y in 0..(height as u32) { - for x in 0..(width as u32) { - let offset = (y as usize) - .checked_mul(stride_usize) - .and_then(|row| base_offset.checked_add(row)) - .and_then(|o| o.checked_add((x as usize) * 4)); - let offset = match offset { - Some(o) => o, - None => continue, - }; - if offset.checked_add(4).is_some_and(|end| end <= raw_bytes.len()) { - let alpha = if format == wl_shm::Format::Xrgb8888 { - 255 - } else { - raw_bytes[offset + 3] - }; - img_buf.put_pixel(x, y, Rgba([ - raw_bytes[offset + 2], - raw_bytes[offset + 1], - raw_bytes[offset], - alpha - ])); - } - } - } + let Some(buffer) = buffer_found else { return }; + + let shm_result = with_buffer_contents(&buffer, |ptr, len, spec| { + let slice = unsafe { std::slice::from_raw_parts(ptr, len) }; + let mut hasher = DefaultHasher::new(); + spec.width.hash(&mut hasher); + spec.height.hash(&mut hasher); + spec.stride.hash(&mut hasher); + spec.offset.hash(&mut hasher); + spec.format.hash(&mut hasher); + let start = (spec.offset.max(0) as usize).min(len); + let span = (spec.stride.max(0) as usize) + .saturating_mul(spec.height.max(0) as usize); + let end = start.saturating_add(span).min(len); + slice[start..end].hash(&mut hasher); + let hash = hasher.finish(); + (hash, spec.width, spec.height, spec.stride, spec.format, spec.offset, slice.to_vec()) + }); - crate::unpremultiply_rgba(&mut img_buf); - let mut bytes = Vec::new(); - if img_buf.write_to(&mut IoCursor::new(&mut bytes), ImageFormat::Png).is_ok() { - self.cursor_cache.insert(hash, bytes.clone()); - final_png = bytes; - if self.cursor_cache.len() > 100 { - let evict = *self.cursor_cache.keys().next().unwrap(); - self.cursor_cache.remove(&evict); - } - } - } - } - }, - Err(BufferAccessError::NotManaged) => { - let mut gles_data: Option<(u64, i32, i32, Vec)> = None; - - let dmabuf_opt = get_dmabuf(&buffer).ok().cloned(); - - if let Some(mut dmabuf) = dmabuf_opt { - if let Some(renderer) = self.gles_renderer.as_mut() { - let width = dmabuf.width() as i32; - let height = dmabuf.height() as i32; - - match renderer.bind(&mut dmabuf) { - Ok(frame) => { - let rect = Rectangle::new((0, 0).into(), (width, height).into()); - - match renderer.copy_framebuffer(&frame, rect, Fourcc::Abgr8888) { - Ok(mapping) => { - match renderer.map_texture(&mapping) { - Ok(data) => { - let mut hasher = DefaultHasher::new(); - data.hash(&mut hasher); - let hash = hasher.finish(); - gles_data = Some((hash, width, height, data.to_vec())); - }, - Err(e) => eprintln!("Failed to map texture: {:?}", e) - } - }, - Err(e) => eprintln!("Failed to copy framebuffer: {:?}", e) + let job = match shm_result { + Ok((hash, width, height, stride, format, buf_offset, raw_bytes)) => { + Some(CursorJob::Shm { + hash, + width, + height, + stride, + offset: buf_offset, + opaque: format == wl_shm::Format::Xrgb8888, + bytes: raw_bytes, + hot_x, + hot_y, + }) + } + Err(BufferAccessError::NotManaged) => { + let mut gles_job = None; + let dmabuf_opt = get_dmabuf(&buffer).ok().cloned(); + if let Some(mut dmabuf) = dmabuf_opt { + if let Some(renderer) = self.gles_renderer.as_mut() { + let width = dmabuf.width() as i32; + let height = dmabuf.height() as i32; + + match renderer.bind(&mut dmabuf) { + Ok(frame) => { + let rect = Rectangle::new((0, 0).into(), (width, height).into()); + match renderer.copy_framebuffer(&frame, rect, Fourcc::Abgr8888) { + Ok(mapping) => match renderer.map_texture(&mapping) { + Ok(data) => { + let mut hasher = DefaultHasher::new(); + data.hash(&mut hasher); + gles_job = Some(CursorJob::Gles { + hash: hasher.finish(), + width, + height, + bytes: data.to_vec(), + hot_x, + hot_y, + }); } + Err(e) => eprintln!("Failed to map texture: {:?}", e), }, - Err(e) => eprintln!("Failed to bind dmabuf to renderer: {:?}", e) + Err(e) => eprintln!("Failed to copy framebuffer: {:?}", e), } } + Err(e) => eprintln!("Failed to bind dmabuf to renderer: {:?}", e), } - - if let Some((hash, width, height, raw_bytes)) = gles_data { - if let Some(cached_png) = self.cursor_cache.get(&hash) { - final_png = cached_png.clone(); - } else { - if width <= 128 && height <= 128 && !raw_bytes.is_empty() { - let mut img_buf = ImageBuffer::, Vec>::new(width as u32, height as u32); - let stride_usize = rgba_readback_stride(raw_bytes.len(), height as usize, width as usize); - - for y in 0..(height as u32) { - for x in 0..(width as u32) { - let offset = (y as usize * stride_usize) + (x as usize * 4); - if offset + 4 <= raw_bytes.len() { - img_buf.put_pixel(x, y, Rgba([ - raw_bytes[offset], - raw_bytes[offset + 1], - raw_bytes[offset + 2], - raw_bytes[offset + 3] - ])); - } - } - } - - crate::unpremultiply_rgba(&mut img_buf); - let mut bytes = Vec::new(); - if img_buf.write_to(&mut IoCursor::new(&mut bytes), ImageFormat::Png).is_ok() { - self.cursor_cache.insert(hash, bytes.clone()); - final_png = bytes; - if self.cursor_cache.len() > 100 { - let evict = *self.cursor_cache.keys().next().unwrap(); - self.cursor_cache.remove(&evict); - } - } - } - } - } - }, - Err(_) => {} + } } - - self.cursor_buffer = Some(buffer); + gles_job } + Err(_) => None, + }; - if !final_png.is_empty() { - ("png", final_png, hot_x, hot_y) - } else { - ("surface", Vec::new(), 0, 0) - } - } - }; + self.cursor_buffer = Some(buffer); + let Some(job) = job else { return }; + job + } + }; + let _ = self.cursor_tx.send(job); + } - if !data.is_empty() || msg_type == "hide" { - Python::attach(|py| { - let py_bytes = PyBytes::new(py, &data); - let _ = cb.call1(py, (msg_type, py_bytes, hot_x, hot_y)); - }); + /// Re-apply the policy's keymap (base + overlays) to the seat keyboard, broadcasting to + /// clients only when the content actually changed (smithay dedupes by content hash). + pub(crate) fn apply_keymap_policy(&mut self) { + let text = self.keymap_policy.keymap_text(); + self.apply_keymap_text(text); + } + + /// Deliver `text` as the seat keymap (and the host virtual keyboard's, in + /// host-capture mode). Empty text is ignored. + pub(crate) fn apply_keymap_text(&mut self, text: String) { + if text.is_empty() { + return; + } + // Host-capture mode: the same managed keymap rides on the virtual + // keyboard, so the host compositor translates injected keycodes with + // selkies' keymap (overlay binds included) instead of its own. + if let Some(host) = &self.host { + host.set_keymap(&text); + } + if let Some(keyboard) = self.seat.get_keyboard() { + if let Err(e) = keyboard.set_keymap_from_string(self, text) { + eprintln!("[Wayland] keymap swap failed: {e:?}"); } } } + + /// Resolve `keysyms` to `(keycode, level)` pairs, overlay-binding whatever the base + /// cannot produce — at most ONE keymap swap for the whole batch, and never rebinding a + /// keycode that is currently held down. Serves computer-use; the interactive input path + /// resolves keysyms in selkies and injects plain keycodes. + pub(crate) fn bind_keysyms(&mut self, keysyms: &[u32]) -> Vec<(u32, u32)> { + let pressed: std::collections::HashSet = self + .seat + .get_keyboard() + .map(|k| k.pressed_keys().iter().map(|c| c.raw()).collect()) + .unwrap_or_default(); + let (out, changed) = self.keymap_policy.bind_many(keysyms, &pressed); + if changed { + self.apply_keymap_policy(); + } + out + } + + /// `bind_keysyms` restricted to level-0 resolutions (see + /// [`KeymapPolicy::bind_many_plain`]); used by the virtual-keyboard translation path, which + /// cannot synthesize modifiers. + pub(crate) fn bind_keysyms_plain(&mut self, keysyms: &[u32]) -> Vec { + let pressed: std::collections::HashSet = self + .seat + .get_keyboard() + .map(|k| k.pressed_keys().iter().map(|c| c.raw()).collect()) + .unwrap_or_default(); + let (out, changed) = self.keymap_policy.bind_many_plain(keysyms, &pressed); + if changed { + self.apply_keymap_policy(); + } + out + } + + /// Answer any client fullscreen/maximize (un)set request with the compositor's forced + /// policy: every toplevel is Fullscreen+Activated at the CURRENT logical size of the + /// output the window is placed on. The configure is always sent, so an app toggling + /// fullscreen gets an explicit, current-geometry answer instead of silence or stale + /// pending state. + pub(crate) fn send_forced_fullscreen_configure(&mut self, toplevel: &ToplevelSurface) { + let output_id = self + .space + .elements() + .chain(self.pending_windows.iter()) + .find(|w| w.toplevel().map(|t| t == toplevel).unwrap_or(false)) + .map(window_output_id) + .unwrap_or(0); + let (logical_width, logical_height) = if let Some(size) = self.logical_size_of(output_id) { + size + } else { + let scale = self.settings.scale.max(0.1); + ( + (self.settings.width as f64 / scale).round() as i32, + (self.settings.height as f64 / scale).round() as i32, + ) + }; + toplevel.with_pending_state(|state| { + state.states.set(XdgState::Fullscreen); + state.states.set(XdgState::Activated); + state.size = Some((logical_width, logical_height).into()); + }); + toplevel.send_configure(); + } } /// Clipboard mime types the bridge can hand to Python, most specific first; `new_selection` @@ -967,6 +1316,8 @@ const CLIPBOARD_MIME_PREFERENCE: &[&str] = &[ "image/jpeg", "image/webp", "image/bmp", + "image/svg+xml", + "image/svg", "text/plain;charset=utf-8", "UTF8_STRING", "text/plain", @@ -997,27 +1348,32 @@ impl SelectionHandler for AppState { if ty != SelectionTarget::Clipboard { return; } + let Some(source) = source else { + self.current_selection_mime = None; + return; + }; + let mimes = source.mime_types(); + let mime = CLIPBOARD_MIME_PREFERENCE + .iter() + .find(|want| mimes.iter().any(|m| m == *want)) + .map(|s| s.to_string()); + // Recorded even with no callback armed, so SetClipboardCallback can re-stage a + // read of a copy made while nobody was listening. + self.current_selection_mime = mime.clone(); if crate::PY_SHUTDOWN.load(std::sync::atomic::Ordering::Relaxed) { return; } if self.clipboard_callback.is_none() { return; } - let Some(source) = source else { return }; - let mimes = source.mime_types(); - let Some(mime) = CLIPBOARD_MIME_PREFERENCE - .iter() - .find(|want| mimes.iter().any(|m| m == *want)) - .map(|s| s.to_string()) - else { - return; - }; + let Some(mime) = mime else { return }; self.pending_clipboard_read = Some(mime); let _ = seat; } /// A client pastes the Python-owned selection: stream the stored bytes into the client's - /// fd on a spawned thread, since the receiving pipe may backpressure. + /// fd on a spawned thread, since the receiving pipe may backpressure. Bounded by an + /// idle deadline: a client that never reads its paste fd must not pin the thread. fn send_selection( &mut self, ty: SelectionTarget, @@ -1026,14 +1382,16 @@ impl SelectionHandler for AppState { _seat: Seat, user_data: &Self::SelectionUserData, ) { - if ty != SelectionTarget::Clipboard { + if ty != SelectionTarget::Clipboard && ty != SelectionTarget::Primary { return; } let payload = user_data.clone(); std::thread::spawn(move || { - use std::io::Write; - let mut f = std::fs::File::from(fd); - let _ = f.write_all(&payload.1); + let _ = crate::wayland::wlclient::write_fd_all( + &fd, + &payload.1, + crate::wayland::wlclient::IO_TIMEOUT, + ); }); } } @@ -1102,7 +1460,7 @@ impl FractionalScaleHandler for AppState { &mut self, surface: smithay::reexports::wayland_server::protocol::wl_surface::WlSurface, ) { - if let Some(output) = self.outputs.first() { + if let Some(output) = self.primary_output() { let scale = output.current_scale().fractional_scale(); with_states(&surface, |states| { smithay::wayland::fractional_scale::with_fractional_scale(states, |fs| { @@ -1613,17 +1971,17 @@ impl SeatHandler for AppState { self.send_cursor_image(&image); } - /// Keep the primary selection's focus following keyboard focus, so middle-click paste - /// targets the currently focused client (or clears it when nothing is focused). + /// Keep BOTH selections' focus following keyboard focus: without the data-device half, + /// the focused client never receives wl_data_offer events and Ctrl+V paste is a silent + /// no-op even while the compositor-side selection is correct (primary covers only + /// middle-click paste). fn focus_changed(&mut self, seat: &Seat, focus: Option<&Self::KeyboardFocus>) { - if let Some(focus_target) = focus { - let dh = &self.dh; - let client = focus_target.wl_surface().and_then(|s| dh.get_client(s.id()).ok()); - set_primary_focus(dh, seat, client); - } else { - let dh = &self.dh; - set_primary_focus(dh, seat, None); - } + let dh = &self.dh; + let client = focus + .and_then(|t| t.wl_surface()) + .and_then(|s| dh.get_client(s.id()).ok()); + set_data_device_focus(dh, seat, client.clone()); + set_primary_focus(dh, seat, client); } } @@ -1679,8 +2037,18 @@ impl XdgShellHandler for AppState { } /// A new toplevel appears: wrap it in a `Window`, queue it for mapping, and register a /// foreign-toplevel handle (seeded with title / app-id) stored on the surface for later updates. + /// The window is pinned to the pointer's output HERE — the first commit can't be relied on + /// for that, because a decoration-negotiating client (foot) has its initial configure sent + /// by `new_decoration` before it ever commits, skipping the pre-configure commit branch. fn new_toplevel(&mut self, surface: ToplevelSurface) { + let target_id = self.pointer_display(); let window = Window::new_wayland_window(surface.clone()); + window.user_data().insert_if_missing_threadsafe(|| WindowMeta { + id: NEXT_WINDOW_ID.fetch_add(1, Ordering::Relaxed), + output: AtomicU32::new(target_id), + placed: AtomicBool::new(false), + parked: AtomicBool::new(false), + }); self.pending_windows.push(window); let (title, app_id) = with_states(surface.wl_surface(), |states| { let attributes = states.data_map.get::().unwrap().lock().unwrap(); @@ -1729,19 +2097,261 @@ impl XdgShellHandler for AppState { } let _ = surface.send_repositioned(token); } + /// Client fullscreen request: always granted at the compositor's forced-fullscreen + /// geometry (the CURRENT logical size), so the toggle gets a definite answer. A request + /// naming an output moves the window there, which is how a nested compositor puts each + /// of its screens on a monitor of its choosing. + fn fullscreen_request( + &mut self, + surface: ToplevelSurface, + output: Option, + ) { + if let Some(id) = output + .as_ref() + .and_then(|wlo| self.output_nodes.iter().find(|n| n.output.owns(wlo)).map(|n| n.id)) + { + let mapped = self + .space + .elements() + .find(|w| w.toplevel().map(|tl| *tl == surface).unwrap_or(false)) + .cloned(); + if let Some(window) = mapped { + if self.place_window_on_output(&window, id) { + return; + } + } else if let Some(meta) = self + .pending_windows + .iter() + .find(|w| w.toplevel().map(|tl| *tl == surface).unwrap_or(false)) + .and_then(window_meta) + { + // Still waiting for its first buffer: record the output instead of mapping + // it, so nothing renders or hit-tests before the client has drawn. The + // configure below then carries that output's geometry. + meta.output.store(id, Ordering::Relaxed); + meta.placed.store(true, Ordering::Relaxed); + } + } + self.send_forced_fullscreen_configure(&surface); + } + /// Client unfullscreen request: the forced-fullscreen policy stands, but the client + /// still receives an explicit configure at the current geometry (the Smithay default sends + /// NOTHING here, leaving the app waiting on a toggle that never answers). + fn unfullscreen_request(&mut self, surface: ToplevelSurface) { + self.send_forced_fullscreen_configure(&surface); + } + /// Maximize request: answered with the forced-fullscreen configure (same geometry). + fn maximize_request(&mut self, surface: ToplevelSurface) { + self.send_forced_fullscreen_configure(&surface); + } + /// Unmaximize request: explicit current-geometry configure, policy unchanged. + fn unmaximize_request(&mut self, surface: ToplevelSurface) { + self.send_forced_fullscreen_configure(&surface); + } /// A toplevel closed: drop it from the pending-window queue so a window that never - /// finished mapping can't linger there, and remove its foreign-toplevel handle so taskbar-style - /// clients stop listing a window that is gone. + /// finished mapping can't linger there, unmap it from the space at once so `list_windows` + /// and hit-testing never see a husk (the periodic `space.refresh` would only reap it + /// later), and remove its foreign-toplevel handle so taskbar-style clients stop listing a + /// window that is gone. fn toplevel_destroyed(&mut self, surface: ToplevelSurface) { if let Some(idx) = self.pending_windows.iter().position(|w| w.toplevel().map(|t| *t == surface).unwrap_or(false)) { self.pending_windows.remove(idx); } + let mapped = self + .space + .elements() + .find(|w| w.toplevel().map(|t| *t == surface).unwrap_or(false)) + .cloned(); + if let Some(window) = mapped { + self.space.unmap_elem(&window); + } if let Some(handle) = with_states(surface.wl_surface(), |states| states.data_map.get::().cloned()) { self.foreign_toplevel_list.remove_toplevel(&handle); } } } +/// In-house `zwp_virtual_keyboard_v1` implementation. Smithay's manager swaps the +/// client-visible seat keymap to the virtual keyboard's keymap on every VK event and never +/// restores it, leaving every client holding a foreign keymap (and killing the compositor's +/// overlay keycodes) after any VK use. Here VK key events are TRANSLATED instead: each keycode +/// resolves to its level-0 keysym under the VK client's own uploaded keymap, maps onto the seat +/// keymap (overlay-binding on demand, batched at keymap upload), and injects through the seat's +/// regular input path — the seat keymap identity never changes and modifier/pressed-key state +/// stays coherent with server-side injection. VK `modifiers` requests are ignored: applying a +/// foreign modifier mask would corrupt the seat's own tracked state, and the supported VK +/// client (selkies' wayland_typer) binds every keysym at level 0 and never sends them. +pub struct PfVirtualKeyboard { + inner: Mutex, +} + +#[derive(Default)] +struct PfVkState { + /// Level-0 keysym per xkb keycode of the client's uploaded keymap. + syms: Option>, + /// VK xkb keycode -> injected seat keycode, so a release always matches its press even + /// across policy rebinds. + pressed: std::collections::HashMap, +} + +impl GlobalDispatch for AppState { + fn bind( + _state: &mut Self, + _dh: &DisplayHandle, + _client: &Client, + resource: New, + _global_data: &(), + data_init: &mut DataInit<'_, Self>, + ) { + data_init.init(resource, ()); + } +} + +impl Dispatch for AppState { + fn request( + _state: &mut Self, + _client: &Client, + _resource: &ZwpVirtualKeyboardManagerV1, + request: zwp_virtual_keyboard_manager_v1::Request, + _data: &(), + _dh: &DisplayHandle, + data_init: &mut DataInit<'_, Self>, + ) { + if let zwp_virtual_keyboard_manager_v1::Request::CreateVirtualKeyboard { seat: _, id } = + request + { + data_init.init(id, PfVirtualKeyboard { inner: Mutex::new(PfVkState::default()) }); + } + } +} + +impl Dispatch for AppState { + fn request( + state: &mut Self, + _client: &Client, + resource: &ZwpVirtualKeyboardV1, + request: zwp_virtual_keyboard_v1::Request, + data: &PfVirtualKeyboard, + _dh: &DisplayHandle, + _data_init: &mut DataInit<'_, Self>, + ) { + use smithay::input::keyboard::xkb; + match request { + zwp_virtual_keyboard_v1::Request::Keymap { format, fd, size } => { + if format != 1 { + return; + } + let ctx = xkb::Context::new(xkb::CONTEXT_NO_FLAGS); + let keymap = unsafe { + xkb::Keymap::new_from_fd( + &ctx, + fd, + size as usize, + xkb::KEYMAP_FORMAT_TEXT_V1, + xkb::KEYMAP_COMPILE_NO_FLAGS, + ) + }; + let Ok(Some(keymap)) = keymap else { + eprintln!("[Wayland] virtual-keyboard keymap failed to compile; ignoring."); + return; + }; + let syms = crate::wayland::keymap::level0_syms(&keymap); + // Pre-bind everything this keymap can type that the seat cannot, in ONE + // seat keymap swap, so the following key events bind nothing. + let missing: Vec = syms + .values() + .copied() + .filter(|&s| !state.keymap_policy.resolves_plain(s)) + .collect(); + if !missing.is_empty() { + let _ = state.bind_keysyms_plain(&missing); + } + data.inner.lock().unwrap().syms = Some(syms); + } + zwp_virtual_keyboard_v1::Request::Key { time: _, key, state: key_state } => { + let mut vk = data.inner.lock().unwrap(); + if vk.syms.is_none() { + drop(vk); + resource.post_error( + zwp_virtual_keyboard_v1::Error::NoKeymap, + "`key` sent before keymap.", + ); + return; + } + let vk_kc = key.wrapping_add(8); + let seat_kc = if key_state == 1 { + let sym = vk.syms.as_ref().and_then(|m| m.get(&vk_kc)).copied(); + // Translate through the seat keymap; an untranslatable keycode + // passes through raw (base sections of both keymaps agree for + // ordinary pc keycodes). + let kc = match sym { + Some(sym) => { + let bound = state.bind_keysyms_plain(&[sym])[0]; + if bound != 0 { bound } else { vk_kc } + } + None => vk_kc, + }; + vk.pressed.insert(vk_kc, kc); + kc + } else { + vk.pressed.remove(&vk_kc).unwrap_or(vk_kc) + }; + drop(vk); + let pressed = key_state == 1; + if let Some(keyboard) = state.seat.get_keyboard() { + let keyboard = keyboard.clone(); + let serial = next_serial(); + let time = wayland_time(); + keyboard.input( + state, + smithay::backend::input::Keycode::new(seat_kc), + if pressed { + smithay::backend::input::KeyState::Pressed + } else { + smithay::backend::input::KeyState::Released + }, + serial, + time, + |_, _, _| smithay::input::keyboard::FilterResult::<()>::Forward, + ); + } + } + zwp_virtual_keyboard_v1::Request::Modifiers { .. } => {} + zwp_virtual_keyboard_v1::Request::Destroy => {} + _ => {} + } + } + + /// Release every seat key this virtual keyboard still holds, so a VK client that + /// disconnects mid-press cannot leave keys logically stuck. + fn destroyed( + state: &mut Self, + _client: ClientId, + _resource: &ZwpVirtualKeyboardV1, + data: &PfVirtualKeyboard, + ) { + let held: Vec = data.inner.lock().unwrap().pressed.drain().map(|(_, kc)| kc).collect(); + if held.is_empty() { + return; + } + if let Some(keyboard) = state.seat.get_keyboard() { + let keyboard = keyboard.clone(); + for kc in held { + let serial = next_serial(); + let time = wayland_time(); + keyboard.input( + state, + smithay::backend::input::Keycode::new(kc), + smithay::backend::input::KeyState::Released, + serial, + time, + |_, _, _| smithay::input::keyboard::FilterResult::<()>::Forward, + ); + } + } + } +} + /// Per-client data attached to every Wayland client connection; holds the compositor's /// per-client surface state. #[derive(Default)] @@ -1761,7 +2371,6 @@ delegate_seat!(AppState); delegate_xdg_shell!(AppState); delegate_dmabuf!(AppState); delegate_fractional_scale!(AppState); -delegate_virtual_keyboard_manager!(AppState); delegate_data_device!(AppState); delegate_data_control!(AppState); delegate_pointer_warp!(AppState); @@ -1782,7 +2391,7 @@ delegate_primary_selection!(AppState); /// Dividing the buffer length by the height recovers a padded stride, so a padded readback cannot /// skew the cursor image; the result never drops below one full `width*4` row, and a zero height /// short-circuits to one row to avoid dividing by zero. -fn rgba_readback_stride(buf_len: usize, height: usize, width: usize) -> usize { +pub(crate) fn rgba_readback_stride(buf_len: usize, height: usize, width: usize) -> usize { let row = width.saturating_mul(4); if height == 0 { return row; diff --git a/pixelflux/src/wayland/host.rs b/pixelflux/src/wayland/host.rs new file mode 100644 index 0000000..5247699 --- /dev/null +++ b/pixelflux/src/wayland/host.rs @@ -0,0 +1,1393 @@ +//! Host-capture mode: pixelflux as a CLIENT of an external Wayland compositor +//! (e.g. labwc running `WLR_BACKENDS=headless`), inverting the nested topology. +//! +//! The host compositor owns the session — one seat, one selection, one screen +//! model — and pixelflux captures and injects as a privileged-protocol client: +//! frames via `zwlr_screencopy_v1` (v3 `copy_with_damage`, so idle screens cost +//! nothing), keyboard via a persistent `zwp_virtual_keyboard_v1` device carrying +//! selkies' own keymap text, pointer via `zwlr_virtual_pointer_v1`. Zero-copy is +//! preserved by allocating the capture buffers from pixelflux's OWN GBM device +//! (render node — no privileges): the compositor blits straight into the dmabufs +//! the encoder imports, so no CPU ever touches a frame. Without a GPU the same +//! loop degrades to wl_shm buffers feeding the CPU encode pool. +//! +//! Every host `wl_output` maps to one selkies display id, in registry order: a +//! control thread on the primary connection owns wlr-output-management and +//! applies the whole layout (per-output custom modes plus row positions) as one +//! atomic configuration, while each output gets its own capture connection and +//! thread. Input proxies are called from the calloop thread directly +//! (wayland-client proxies are thread-safe); pointer coordinates arrive in the +//! same union-layout space selkies uses, so the virtual-pointer extent is the +//! layout's bounding box. All capture-side waits poll a wake pipe next to the +//! Wayland socket, so a resize or teardown lands immediately even while the +//! thread is parked in a `copy_with_damage` wait on a static screen. + +use std::fs::File; +use std::os::fd::{AsFd, AsRawFd, OwnedFd, RawFd}; +use std::os::unix::net::UnixStream; +use std::sync::atomic::{AtomicBool, Ordering}; +use std::sync::mpsc::{Receiver, Sender, TryRecvError}; +use std::sync::{Arc, Mutex}; +use std::time::{Duration, Instant}; + +use gbm::{BufferObjectFlags, Device as GbmDevice, Format as GbmFormat}; +use smithay::backend::allocator::dmabuf::Dmabuf; +use smithay::backend::allocator::Buffer as _; +use smithay::utils::{Physical, Rectangle}; +use wayland_client::protocol::{wl_output, wl_pointer, wl_registry, wl_seat, wl_shm, wl_shm_pool}; +use wayland_client::{delegate_noop, Connection, Dispatch, EventQueue, Proxy, QueueHandle, WEnum}; +use wayland_protocols::wp::linux_dmabuf::zv1::client::{ + zwp_linux_buffer_params_v1::{self, ZwpLinuxBufferParamsV1}, + zwp_linux_dmabuf_v1::ZwpLinuxDmabufV1, +}; +use wayland_protocols_misc::zwp_virtual_keyboard_v1::client::{ + zwp_virtual_keyboard_manager_v1::ZwpVirtualKeyboardManagerV1, + zwp_virtual_keyboard_v1::ZwpVirtualKeyboardV1, +}; +use wayland_protocols_wlr::screencopy::v1::client::{ + zwlr_screencopy_frame_v1::{self, ZwlrScreencopyFrameV1}, + zwlr_screencopy_manager_v1::ZwlrScreencopyManagerV1, +}; +use wayland_protocols_wlr::output_management::v1::client::{ + zwlr_output_configuration_head_v1::ZwlrOutputConfigurationHeadV1, + zwlr_output_configuration_v1::{self, ZwlrOutputConfigurationV1}, + zwlr_output_head_v1::{self, ZwlrOutputHeadV1}, + zwlr_output_manager_v1::{self, ZwlrOutputManagerV1}, + zwlr_output_mode_v1::ZwlrOutputModeV1, +}; +use wayland_protocols_wlr::virtual_pointer::v1::client::{ + zwlr_virtual_pointer_manager_v1::ZwlrVirtualPointerManagerV1, + zwlr_virtual_pointer_v1::ZwlrVirtualPointerV1, +}; + +use crate::wayland::wlclient::{ + bounded_roundtrip, drain_pipe, impl_sync_callback, memfd_with, socket_path, wait_readable2, + wake_pipe, wake_write, SyncState, +}; + +const KEYMAP_FORMAT_XKB_V1: u32 = 1; +/// Frames in flight between a capture thread and the encode tick. Two slots: +/// one being blitted by the compositor while the previous one is encoded. +const SLOTS: usize = 2; + +/// A frame the compositor finished blitting, ready for the encoder. +pub struct HostFrame { + gen: u64, + slot: usize, + /// GPU path: the filled dmabuf (Arc-backed; the buffer itself stays owned by + /// the capture thread and is reused once the slot is released). + pub dmabuf: Option, + /// Software path: the shm mapping itself — the consumer converts straight + /// out of it (one copy total); the slot is not reused until release. + pub cpu: Option, + pub width: i32, + pub height: i32, + pub damage: Vec>, +} + +/// Borrow-by-Arc view of a software frame still sitting in its shm mapping. +pub struct HostCpuFrame { + map: Arc, + stride: usize, + format: u32, +} + +impl HostCpuFrame { + /// Convert the frame into tight BGRA rows in `dst` (sized `w*4*h`); the + /// announced shm format decides the per-pixel conversion (compositors pick + /// their renderer's preferred read format, e.g. 24-bit BGR on NVIDIA GLES). + pub fn write_bgra(&self, w: i32, h: i32, dst: &mut [u8]) { + let row = (w * 4) as usize; + let (src_bpp, swap_rb) = match self.format { + f if f == wl_shm::Format::Xbgr8888 as u32 + || f == wl_shm::Format::Abgr8888 as u32 => (4usize, true), + f if f == wl_shm::Format::Bgr888 as u32 => (3, false), + _ => (4, false), // xrgb/argb: already BGRA byte order + }; + for y in 0..h as usize { + let src_start = y * self.stride; + let src_end = (src_start + src_bpp * w as usize).min(self.map.len()); + if src_start >= src_end || (y + 1) * row > dst.len() { + break; + } + let src_row = &self.map[src_start..src_end]; + let dst_row = &mut dst[y * row..(y + 1) * row]; + match (src_bpp, swap_rb) { + (4, false) => { + let n = row.min(src_row.len()); + dst_row[..n].copy_from_slice(&src_row[..n]); + } + (4, true) => { + for (d, s) in dst_row.chunks_exact_mut(4).zip(src_row.chunks_exact(4)) { + d[0] = s[2]; + d[1] = s[1]; + d[2] = s[0]; + d[3] = s[3]; + } + } + _ => { + for (d, s) in dst_row.chunks_exact_mut(4).zip(src_row.chunks_exact(3)) { + d[0] = s[0]; + d[1] = s[1]; + d[2] = s[2]; + d[3] = 0xff; + } + } + } + } + } +} + +/// One display's slice of the union layout, as selkies configured it. Slots are +/// keyed by selkies display id ('display2' is id 2 — ids are sparse); active ids +/// map onto host outputs by ascending rank. +#[derive(Clone, Copy, Default)] +struct LayoutSlot { + want: (i32, i32), + pos: (i32, i32), + active: bool, +} + +enum ToHost { + Start { width: i32, height: i32 }, + /// Slot return; `gen` guards against slots recycled by a renegotiation + /// while the consumer still held the frame. + Release { gen: u64, slot: usize }, + /// Park the capture (display stopped) without ending the thread. + Idle, +} + +enum CtrlMsg { + Apply(Vec), +} + +// --------------------------------------------------------------------------- +// Control connection: seat + virtual input devices + wlr-output-management. +// --------------------------------------------------------------------------- + +#[derive(Default)] +struct CtrlState { + seat: Option, + vk_mgr: Option, + vptr_mgr: Option, + has_screencopy: bool, + outputs: Vec<(wl_output::WlOutput, Option)>, + /// Registry indices in display order (natural name sort — registry + /// announcement order is not guaranteed to match output creation order). + order: Vec, + output_mgr: Option, + heads: Vec<(ZwlrOutputHeadV1, Option)>, + om_serial: Option, + cfg_result: Option, + cfg_cancelled: bool, + sync_done: bool, +} + + +/// Natural sort key for an output name: text prefix plus trailing number, so +/// HEADLESS-2 orders before HEADLESS-10. Unnamed outputs keep registry order +/// after every named one. +fn output_order_key(name: Option<&String>, registry_idx: usize) -> (bool, String, u64, usize) { + match name { + Some(n) => { + let digits_at = n.rfind(|c: char| !c.is_ascii_digit()).map(|i| i + 1).unwrap_or(0); + let num = n[digits_at..].parse::().unwrap_or(0); + (false, n[..digits_at].to_string(), num, registry_idx) + } + None => (true, String::new(), 0, registry_idx), + } +} + +impl SyncState for CtrlState { + fn sync_done_mut(&mut self) -> &mut bool { + &mut self.sync_done + } +} +impl_sync_callback!(CtrlState); + +impl Dispatch for CtrlState { + fn event( + state: &mut Self, + registry: &wl_registry::WlRegistry, + event: wl_registry::Event, + _: &(), + _: &Connection, + qh: &QueueHandle, + ) { + if let wl_registry::Event::Global { name, interface, version } = event { + match interface.as_str() { + "wl_seat" if state.seat.is_none() => { + state.seat = Some(registry.bind(name, 1, qh, ())) + } + "zwp_virtual_keyboard_manager_v1" => { + state.vk_mgr = Some(registry.bind(name, 1, qh, ())) + } + "zwlr_virtual_pointer_manager_v1" => { + state.vptr_mgr = Some(registry.bind(name, version.min(2), qh, ())) + } + "zwlr_screencopy_manager_v1" if version >= 3 => state.has_screencopy = true, + "wl_output" => { + let idx = state.outputs.len(); + let out = registry.bind(name, version.min(4), qh, idx); + state.outputs.push((out, None)); + } + "zwlr_output_manager_v1" => { + state.output_mgr = Some(registry.bind(name, 1, qh, ())) + } + _ => {} + } + } + } +} + +macro_rules! impl_output_name { + ($t:ty) => { + impl Dispatch for $t { + fn event( + state: &mut Self, + _: &wl_output::WlOutput, + event: wl_output::Event, + idx: &usize, + _: &Connection, + _: &QueueHandle, + ) { + if let wl_output::Event::Name { name } = event { + if let Some(o) = state.outputs.get_mut(*idx) { + o.1 = Some(name); + } + } + } + } + }; +} +impl_output_name!(CtrlState); + +delegate_noop!(CtrlState: ignore wl_seat::WlSeat); +delegate_noop!(CtrlState: ZwpVirtualKeyboardManagerV1); +delegate_noop!(CtrlState: ZwpVirtualKeyboardV1); +delegate_noop!(CtrlState: ZwlrVirtualPointerManagerV1); +delegate_noop!(CtrlState: ZwlrVirtualPointerV1); +delegate_noop!(CtrlState: ignore ZwlrOutputModeV1); +delegate_noop!(CtrlState: ZwlrOutputConfigurationHeadV1); + +impl Dispatch for CtrlState { + fn event( + state: &mut Self, + _: &ZwlrOutputManagerV1, + event: zwlr_output_manager_v1::Event, + _: &(), + _: &Connection, + _: &QueueHandle, + ) { + match event { + zwlr_output_manager_v1::Event::Head { head } => state.heads.push((head, None)), + zwlr_output_manager_v1::Event::Done { serial } => state.om_serial = Some(serial), + _ => {} + } + } + + wayland_client::event_created_child!(CtrlState, ZwlrOutputManagerV1, [ + zwlr_output_manager_v1::EVT_HEAD_OPCODE => (ZwlrOutputHeadV1, ()), + ]); +} + +impl Dispatch for CtrlState { + fn event( + state: &mut Self, + head: &ZwlrOutputHeadV1, + event: zwlr_output_head_v1::Event, + _: &(), + _: &Connection, + _: &QueueHandle, + ) { + match event { + zwlr_output_head_v1::Event::Name { name } => { + if let Some(h) = state.heads.iter_mut().find(|(h, _)| h.id() == head.id()) { + h.1 = Some(name); + } + } + zwlr_output_head_v1::Event::Finished => { + state.heads.retain(|(h, _)| h.id() != head.id()); + } + _ => {} + } + } + + wayland_client::event_created_child!(CtrlState, ZwlrOutputHeadV1, [ + zwlr_output_head_v1::EVT_MODE_OPCODE => (ZwlrOutputModeV1, ()), + ]); +} + +impl Dispatch for CtrlState { + fn event( + state: &mut Self, + _: &ZwlrOutputConfigurationV1, + event: zwlr_output_configuration_v1::Event, + _: &(), + _: &Connection, + _: &QueueHandle, + ) { + match event { + zwlr_output_configuration_v1::Event::Succeeded => state.cfg_result = Some(true), + zwlr_output_configuration_v1::Event::Failed => state.cfg_result = Some(false), + zwlr_output_configuration_v1::Event::Cancelled => state.cfg_cancelled = true, + _ => {} + } + } +} + +// --------------------------------------------------------------------------- +// Capture connections: one per host output (screencopy + buffer allocation). +// --------------------------------------------------------------------------- + +#[derive(Default)] +struct CaptureState { + shm: Option, + dmabuf: Option, + screencopy: Option, + outputs: Vec<(wl_output::WlOutput, Option)>, + // Per-frame capture negotiation/results. + announce_dmabuf: Option<(u32, i32, i32)>, // fourcc, w, h + announce_shm: Option<(u32, i32, i32, i32)>, // format, w, h, stride + buffer_done: bool, + damage: Vec>, + ready: bool, + failed: bool, + sync_done: bool, +} + + +impl CaptureState { + fn reset_frame(&mut self) { + self.announce_dmabuf = None; + self.announce_shm = None; + self.buffer_done = false; + self.damage.clear(); + self.ready = false; + self.failed = false; + } +} + +impl SyncState for CaptureState { + fn sync_done_mut(&mut self) -> &mut bool { + &mut self.sync_done + } +} +impl_sync_callback!(CaptureState); +impl_output_name!(CaptureState); + +impl Dispatch for CaptureState { + fn event( + state: &mut Self, + registry: &wl_registry::WlRegistry, + event: wl_registry::Event, + _: &(), + _: &Connection, + qh: &QueueHandle, + ) { + if let wl_registry::Event::Global { name, interface, version } = event { + match interface.as_str() { + "wl_shm" => state.shm = Some(registry.bind(name, 1, qh, ())), + "zwp_linux_dmabuf_v1" if version >= 3 => { + state.dmabuf = Some(registry.bind(name, 3, qh, ())) + } + "zwlr_screencopy_manager_v1" if version >= 3 => { + state.screencopy = Some(registry.bind(name, 3, qh, ())) + } + "wl_output" => { + let idx = state.outputs.len(); + let out = registry.bind(name, version.min(4), qh, idx); + state.outputs.push((out, None)); + } + _ => {} + } + } + } +} + +delegate_noop!(CaptureState: ignore wl_shm::WlShm); +delegate_noop!(CaptureState: ignore wl_shm_pool::WlShmPool); +delegate_noop!(CaptureState: ignore wayland_client::protocol::wl_buffer::WlBuffer); +delegate_noop!(CaptureState: ZwlrScreencopyManagerV1); + +impl Dispatch for CaptureState { + fn event( + _: &mut Self, + _: &ZwpLinuxDmabufV1, + _: ::Event, + _: &(), + _: &Connection, + _: &QueueHandle, + ) { + // format/modifier advertisements: the allocation follows the screencopy + // frame's announcement instead. + } +} + +impl Dispatch for CaptureState { + fn event( + _: &mut Self, + _: &ZwpLinuxBufferParamsV1, + _: zwp_linux_buffer_params_v1::Event, + _: &(), + _: &Connection, + _: &QueueHandle, + ) { + // created/failed are only sent for non-immed creation. + } +} + +impl Dispatch for CaptureState { + fn event( + state: &mut Self, + _: &ZwlrScreencopyFrameV1, + event: zwlr_screencopy_frame_v1::Event, + _: &(), + _: &Connection, + _: &QueueHandle, + ) { + match event { + zwlr_screencopy_frame_v1::Event::Buffer { + format: WEnum::Value(f), width, height, stride, + } => { + state.announce_shm = Some((f as u32, width as i32, height as i32, stride as i32)); + } + zwlr_screencopy_frame_v1::Event::LinuxDmabuf { format, width, height } => { + state.announce_dmabuf = Some((format, width as i32, height as i32)); + } + zwlr_screencopy_frame_v1::Event::BufferDone => state.buffer_done = true, + zwlr_screencopy_frame_v1::Event::Damage { x, y, width, height } => { + state.damage.push(Rectangle::new( + (x as i32, y as i32).into(), + (width as i32, height as i32).into(), + )); + } + zwlr_screencopy_frame_v1::Event::Ready { .. } => state.ready = true, + zwlr_screencopy_frame_v1::Event::Failed => state.failed = true, + _ => {} + } + } +} + +enum SlotBuffer { + Gpu { + _bo: gbm::BufferObject<()>, + dmabuf: Dmabuf, + wl: wayland_client::protocol::wl_buffer::WlBuffer, + }, + Cpu { + _pool: wl_shm_pool::WlShmPool, + map: Arc, + stride: i32, + format: u32, + wl: wayland_client::protocol::wl_buffer::WlBuffer, + }, +} + +/// One host output's calloop-side handle: control channel, wake pipe, frames. +struct OutputHandle { + to_thread: Sender, + wake: OwnedFd, + frames: Receiver, + /// Newest frame, kept (slot and all) until replaced so an IDR request on a + /// static screen can re-encode current content like compositor mode does. + retained: Mutex>, + name: Option, +} + +impl OutputHandle { + fn send(&self, msg: ToHost) { + let _ = self.to_thread.send(msg); + wake_write(self.wake.as_raw_fd()); + } +} + +/// The calloop-side session: input proxies plus one capture handle per output. +pub struct HostSession { + conn: Connection, + vk: Option, + vptr: Option, + ctrl_tx: Sender, + ctrl_wake: OwnedFd, + outputs: Vec, + layout: Mutex>, + alive: Arc, +} + +impl HostSession { + /// Connect to `display`, bring up input devices, enumerate the host's + /// outputs and spawn one capture thread per output (idle until + /// [`start_capture`]) plus the layout-control thread. `gbm_path` (this + /// process's render node) enables the zero-copy path. + pub fn connect(display: &str, gbm_path: Option) -> Result { + let path = socket_path(display).ok_or("XDG_RUNTIME_DIR is unset")?; + let stream = UnixStream::connect(&path).map_err(|e| format!("connect {path}: {e}"))?; + let conn = Connection::from_socket(stream).map_err(|e| format!("wayland setup: {e}"))?; + let mut queue = conn.new_event_queue(); + let qh = queue.handle(); + let _registry = conn.display().get_registry(&qh, ()); + let mut state = CtrlState::default(); + bounded_roundtrip(&conn, &mut queue, &mut state)?; + + let seat = state.seat.clone().ok_or("host compositor advertises no wl_seat")?; + if !state.has_screencopy { + return Err("host compositor lacks zwlr_screencopy_manager_v1 (v3)".into()); + } + if state.outputs.is_empty() { + return Err("host compositor has no wl_output".into()); + } + + let vk = match &state.vk_mgr { + Some(mgr) => { + let vk = mgr.create_virtual_keyboard(&seat, &qh, ()); + // A keymap must precede any key event; selkies replaces this with + // its managed keymap through the ABI as soon as it starts. + if let Some(text) = crate::wayland::vkclient::us_base_text() { + let mut data = text.as_bytes().to_vec(); + data.push(0); + let fd = memfd_with(&data)?; + vk.keymap(KEYMAP_FORMAT_XKB_V1, fd.as_fd(), data.len() as u32); + } + Some(vk) + } + None => { + eprintln!("[HostCapture] no zwp_virtual_keyboard_manager_v1: keyboard injection disabled."); + None + } + }; + let vptr = match &state.vptr_mgr { + Some(mgr) => Some(mgr.create_virtual_pointer(Some(&seat), &qh, ())), + None => { + eprintln!("[HostCapture] no zwlr_virtual_pointer_manager_v1: pointer injection disabled."); + None + } + }; + // Second round-trip: wl_output v4 name events and output-management heads. + bounded_roundtrip(&conn, &mut queue, &mut state)?; + + let alive = Arc::new(AtomicBool::new(true)); + let mut order: Vec = (0..state.outputs.len()).collect(); + order.sort_by_key(|&i| output_order_key(state.outputs[i].1.as_ref(), i)); + let names: Vec> = + order.iter().map(|&i| state.outputs[i].1.clone()).collect(); + state.order = order; + println!( + "[HostCapture] host outputs: {}.", + names + .iter() + .enumerate() + .map(|(i, n)| format!("{i}={}", n.as_deref().unwrap_or("?"))) + .collect::>() + .join(", ") + ); + + let mut outputs = Vec::new(); + for (i, name) in names.iter().cloned().enumerate() { + let (wake_rd, wake_wr) = wake_pipe()?; + let (to_thread, from_main) = std::sync::mpsc::channel::(); + let (frame_tx, frames) = std::sync::mpsc::channel::(); + let display = display.to_string(); + let expect = name.clone(); + let gbm_path = gbm_path.clone(); + let alive = alive.clone(); + std::thread::Builder::new() + .name(format!("pf-host-cap{i}")) + .spawn(move || { + if let Err(e) = + capture_loop(&display, i, expect, gbm_path, from_main, wake_rd, frame_tx) + { + eprintln!("[HostCapture] output {i} capture ended: {e}"); + } + if i == 0 { + alive.store(false, Ordering::Relaxed); + } + }) + .map_err(|e| format!("spawn: {e}"))?; + outputs.push(OutputHandle { + to_thread, + wake: wake_wr, + frames, + retained: Mutex::new(None), + name, + }); + } + + let (ctrl_tx, ctrl_rx) = std::sync::mpsc::channel::(); + let (ctrl_wake_rd, ctrl_wake) = wake_pipe()?; + { + let conn = conn.clone(); + std::thread::Builder::new() + .name("pf-host-ctrl".into()) + .spawn(move || control_loop(conn, queue, state, ctrl_rx, ctrl_wake_rd)) + .map_err(|e| format!("spawn: {e}"))?; + } + + let layout = Mutex::new(std::collections::BTreeMap::new()); + Ok(Self { conn, vk, vptr, ctrl_tx, ctrl_wake, outputs, layout, alive }) + } + + pub fn output_count(&self) -> usize { + self.outputs.len() + } + + /// The host output backing `display_id`: active display ids map onto host + /// outputs by ascending rank (selkies ids are sparse — 'display2' is id 2). + fn output_index_for(&self, display_id: u32) -> Option { + let layout = self.layout.lock().unwrap(); + let rank = layout + .iter() + .filter(|(_, s)| s.active) + .position(|(id, _)| *id == display_id)?; + (rank < self.outputs.len()).then_some(rank) + } + + /// True when an active capture on `display_id` has a host output behind it. + pub fn has_output_for(&self, display_id: u32) -> bool { + self.output_index_for(display_id).is_some() + } + + /// Record where selkies laid out `display_id` (union coordinates); pushed + /// to the host with the next capture (re)start. + pub fn set_layout(&self, display_id: u32, x: i32, y: i32) { + let mut layout = self.layout.lock().unwrap(); + layout.entry(display_id).or_default().pos = (x, y); + } + + /// Aim `display_id` at `width`x`height`: assigns every active display its + /// host output (by rank), asks the host for all modes and positions in one + /// atomic configuration, and points the capture threads at their sizes; + /// frames gate until the host applies them. + pub fn start_capture(&self, display_id: u32, width: i32, height: i32) { + let assignments = { + let mut layout = self.layout.lock().unwrap(); + let slot = layout.entry(display_id).or_default(); + slot.want = (width, height); + slot.active = true; + let active: Vec<(u32, LayoutSlot)> = layout + .iter() + .filter(|(_, s)| s.active) + .map(|(id, s)| (*id, *s)) + .collect(); + if active.iter().position(|(id, _)| *id == display_id).unwrap_or(usize::MAX) + >= self.outputs.len() + { + eprintln!( + "[HostCapture] display {display_id} has no host output (host has {}); not captured.", + self.outputs.len() + ); + } + active + }; + // Ranks may have shifted (a lower id joined): repoint every assigned + // capture thread; a same-size Start is a no-op for an unaffected one. + let mut by_output: Vec = Vec::new(); + for (rank, (_, slot)) in assignments.iter().enumerate() { + if rank >= self.outputs.len() { + break; + } + self.outputs[rank].send(ToHost::Start { width: slot.want.0, height: slot.want.1 }); + by_output.push(*slot); + } + let _ = self.ctrl_tx.send(CtrlMsg::Apply(by_output)); + wake_write(self.ctrl_wake.as_raw_fd()); + } + + /// Park `display_id`'s capture (its slot leaves the pointer extent). The + /// retained frame goes back to the capture pool — dropping it silently + /// would wedge one of the two slots until the next renegotiation. + pub fn idle_output(&self, display_id: u32) { + let Some(idx) = self.output_index_for(display_id) else { + self.layout.lock().unwrap().entry(display_id).or_default().active = false; + return; + }; + self.layout.lock().unwrap().entry(display_id).or_default().active = false; + self.outputs[idx].send(ToHost::Idle); + if let Some(old) = self.outputs[idx].retained.lock().unwrap().take() { + self.outputs[idx].send(ToHost::Release { gen: old.gen, slot: old.slot }); + } + } + + pub fn alive(&self) -> bool { + self.alive.load(Ordering::Relaxed) + } + + /// Newest ready frame for `display_id`, releasing any staler ones straight + /// back to the pool. + pub fn try_take_frame(&self, display_id: u32) -> Option { + let handle = self.outputs.get(self.output_index_for(display_id)?)?; + let mut newest: Option = None; + while let Ok(frame) = handle.frames.try_recv() { + if let Some(stale) = newest.replace(frame) { + handle.send(ToHost::Release { gen: stale.gen, slot: stale.slot }); + } + } + newest + } + + /// Keep `frame` as `display_id`'s current content (releasing the one it + /// replaces). + pub fn retain_frame(&self, display_id: u32, frame: HostFrame) { + let Some(handle) = self.output_index_for(display_id).and_then(|i| self.outputs.get(i)) + else { + return; + }; + let old = handle.retained.lock().unwrap().replace(frame); + if let Some(old) = old { + handle.send(ToHost::Release { gen: old.gen, slot: old.slot }); + } + } + + /// Run `f` with `display_id`'s retained (current-content) frame, if any. + pub fn with_retained( + &self, + display_id: u32, + f: impl FnOnce(Option<&HostFrame>) -> R, + ) -> R { + match self.output_index_for(display_id).and_then(|i| self.outputs.get(i)) { + Some(handle) => f(handle.retained.lock().unwrap().as_ref()), + None => f(None), + } + } + + /// Upload selkies' managed keymap to the virtual keyboard verbatim. + pub fn set_keymap(&self, text: &str) { + let Some(vk) = &self.vk else { return }; + let mut data = text.as_bytes().to_vec(); + data.push(0); + match memfd_with(&data) { + Ok(fd) => { + vk.keymap(KEYMAP_FORMAT_XKB_V1, fd.as_fd(), data.len() as u32); + let _ = self.conn.flush(); + } + Err(e) => eprintln!("[HostCapture] keymap upload failed: {e}"), + } + } + + /// Key event in xkb numbering (evdev + 8), matching the seat injectors. + pub fn key(&self, xkb_keycode: u32, pressed: bool) { + let Some(vk) = &self.vk else { return }; + if xkb_keycode < 8 { + return; + } + vk.key(0, xkb_keycode - 8, if pressed { 1 } else { 0 }); + let _ = self.conn.flush(); + } + + /// Union-layout bounding box of every active, output-backed display (the + /// virtual-pointer extent, matching the space selkies injects in). + fn extent(&self) -> (i32, i32) { + let layout = self.layout.lock().unwrap(); + let mut w = 0; + let mut h = 0; + for (rank, (_, slot)) in layout.iter().filter(|(_, s)| s.active).enumerate() { + if rank >= self.outputs.len() { + break; + } + w = w.max(slot.pos.0 + slot.want.0); + h = h.max(slot.pos.1 + slot.want.1); + } + (w, h) + } + + pub fn pointer_motion_abs(&self, x: f64, y: f64) { + let Some(vp) = &self.vptr else { return }; + let (w, h) = self.extent(); + if w <= 0 || h <= 0 { + return; + } + let cx = x.clamp(0.0, (w - 1) as f64) as u32; + let cy = y.clamp(0.0, (h - 1) as f64) as u32; + vp.motion_absolute(0, cx, cy, w as u32, h as u32); + vp.frame(); + let _ = self.conn.flush(); + } + + pub fn pointer_button(&self, btn: u32, pressed: bool) { + let Some(vp) = &self.vptr else { return }; + vp.button( + 0, + btn, + if pressed { wl_pointer::ButtonState::Pressed } else { wl_pointer::ButtonState::Released }, + ); + vp.frame(); + let _ = self.conn.flush(); + } + + pub fn pointer_axis(&self, dx: f64, dy: f64) { + let Some(vp) = &self.vptr else { return }; + if dy != 0.0 { + vp.axis(0, wl_pointer::Axis::VerticalScroll, dy); + } + if dx != 0.0 { + vp.axis(0, wl_pointer::Axis::HorizontalScroll, dx); + } + vp.frame(); + let _ = self.conn.flush(); + } +} + +// Dropping the session drops every channel sender and wake-pipe write end; the +// capture and control threads observe the disconnect (poll sees POLLHUP) and +// exit on their own. + +// --------------------------------------------------------------------------- +// Control thread: pumps the primary connection and applies layout requests. +// --------------------------------------------------------------------------- + +fn control_loop( + conn: Connection, + mut queue: EventQueue, + mut state: CtrlState, + ctrl_rx: Receiver, + wake_rd: OwnedFd, +) { + let qh = queue.handle(); + loop { + let mut pending: Option> = None; + loop { + match ctrl_rx.try_recv() { + Ok(CtrlMsg::Apply(slots)) => pending = Some(slots), + Err(TryRecvError::Empty) => break, + Err(TryRecvError::Disconnected) => return, + } + } + if let Some(slots) = pending { + apply_layout(&conn, &mut queue, &mut state, &qh, &slots); + } + if queue.dispatch_pending(&mut state).is_err() { + return; + } + let _ = queue.flush(); + if let Some(guard) = conn.prepare_read() { + match wait_readable2( + guard.connection_fd().as_raw_fd(), + wake_rd.as_raw_fd(), + Some(Duration::from_secs(1)), + ) { + Ok((wl, wake)) => { + if wake { + drain_pipe(wake_rd.as_raw_fd()); + } + if wl { + let _ = guard.read(); + } else { + drop(guard); + } + } + Err(_) => return, + } + } + } +} + +/// Ask the host (wlr-output-management) to give every active output its wanted +/// mode and layout position in one atomic configuration. Retries across a +/// `cancelled` (stale serial); a refusal only warns — capture threads keep +/// gating on the size they were promised. +fn apply_layout( + conn: &Connection, + queue: &mut EventQueue, + state: &mut CtrlState, + qh: &QueueHandle, + slots: &[LayoutSlot], +) { + let Some(mgr) = state.output_mgr.clone() else { + eprintln!("[HostCapture] host lacks zwlr_output_manager_v1; capture follows the host's own size."); + return; + }; + let deadline = Instant::now() + Duration::from_secs(5); + for _ in 0..3 { + while state.om_serial.is_none() && Instant::now() < deadline { + if !pump_ctrl(conn, queue, state) { + return; + } + } + let Some(serial) = state.om_serial else { + eprintln!("[HostCapture] output-management serial never arrived; resize skipped."); + return; + }; + state.cfg_result = None; + state.cfg_cancelled = false; + let cfg = mgr.create_configuration(serial, qh, ()); + let mut any = false; + for (i, slot) in slots.iter().enumerate() { + if !slot.active { + continue; + } + let want_name = state + .order + .get(i) + .and_then(|&oi| state.outputs.get(oi)) + .and_then(|(_, n)| n.clone()); + let head = match want_name + .as_ref() + .and_then(|n| state.heads.iter().find(|(_, hn)| hn.as_ref() == Some(n))) + .or_else(|| state.heads.get(i)) + { + Some((h, _)) => h.clone(), + None => { + eprintln!("[HostCapture] no output-management head for output {i}; not resized."); + continue; + } + }; + let cfg_head = cfg.enable_head(&head, qh, ()); + cfg_head.set_custom_mode(slot.want.0, slot.want.1, 0); + cfg_head.set_position(slot.pos.0, slot.pos.1); + any = true; + } + if !any { + cfg.destroy(); + return; + } + cfg.apply(); + let _ = queue.flush(); + while state.cfg_result.is_none() && !state.cfg_cancelled && Instant::now() < deadline { + if !pump_ctrl(conn, queue, state) { + cfg.destroy(); + return; + } + } + cfg.destroy(); + if state.cfg_cancelled { + // Stale serial: the compositor re-announces its state with a fresh one. + state.om_serial = None; + continue; + } + if state.cfg_result != Some(true) { + eprintln!("[HostCapture] host refused the layout; capture follows the host's own size."); + } + return; + } +} + +/// One bounded dispatch step for the control connection (false = connection died). +fn pump_ctrl(conn: &Connection, queue: &mut EventQueue, state: &mut CtrlState) -> bool { + if queue.dispatch_pending(state).is_err() { + return false; + } + let _ = queue.flush(); + let Some(guard) = conn.prepare_read() else { return true }; + match crate::wayland::wlclient::wait_readable( + guard.connection_fd().as_raw_fd(), + Duration::from_millis(200), + ) { + Ok(readable) => { + if readable { + let _ = guard.read(); + } else { + drop(guard); + } + true + } + Err(_) => false, + } +} + +// --------------------------------------------------------------------------- +// Capture threads: one connection + screencopy loop per host output. +// --------------------------------------------------------------------------- + +fn fourcc_to_gbm(fourcc: u32) -> GbmFormat { + // XR24 / AR24; anything else falls back to ARGB (the encoder reads BGRA bytes + // and ignores alpha). + match fourcc { + 0x34325258 => GbmFormat::Xrgb8888, + _ => GbmFormat::Argb8888, + } +} + +/// What a control-channel drain decided while a capture wait was in progress. +enum Ctl { + None, + Renegotiate, + Idle, + Dead, +} + +enum Pump { + Done, + Control, + Timeout, +} + +/// Dispatch events until `done(state)` holds, waiting on the Wayland socket and +/// the wake pipe together so control messages interrupt any capture wait. The +/// predicate is re-checked after every dispatch, so a wait whose condition was +/// satisfied by already-queued events returns without ever blocking. +fn pump_until( + conn: &Connection, + queue: &mut EventQueue, + state: &mut CaptureState, + wake_rd: RawFd, + timeout: Option, + done: impl Fn(&CaptureState) -> bool, +) -> Result { + let deadline = timeout.map(|t| Instant::now() + t); + loop { + queue.dispatch_pending(state).map_err(|e| format!("dispatch: {e}"))?; + if done(state) { + return Ok(Pump::Done); + } + queue.flush().map_err(|e| format!("flush: {e}"))?; + let remaining = match deadline { + Some(d) => match d.checked_duration_since(Instant::now()) { + Some(r) => Some(r), + None => return Ok(Pump::Timeout), + }, + None => None, + }; + let Some(guard) = conn.prepare_read() else { continue }; + let (wl, wake) = wait_readable2(guard.connection_fd().as_raw_fd(), wake_rd, remaining)?; + if wake { + drop(guard); + drain_pipe(wake_rd); + return Ok(Pump::Control); + } + if wl { + let _ = guard.read(); + } else { + drop(guard); + if deadline.is_some_and(|d| Instant::now() >= d) { + return Ok(Pump::Timeout); + } + } + } +} + +fn capture_loop( + display: &str, + index: usize, + expect_name: Option, + gbm_path: Option, + from_main: Receiver, + wake_rd: OwnedFd, + frame_tx: Sender, +) -> Result<(), String> { + let path = socket_path(display).ok_or("XDG_RUNTIME_DIR is unset")?; + let stream = UnixStream::connect(&path).map_err(|e| format!("connect {path}: {e}"))?; + let conn = Connection::from_socket(stream).map_err(|e| format!("wayland setup: {e}"))?; + let mut queue = conn.new_event_queue(); + let qh = queue.handle(); + let _registry = conn.display().get_registry(&qh, ()); + let mut state = CaptureState::default(); + bounded_roundtrip(&conn, &mut queue, &mut state)?; + // Names (wl_output v4) arrive after the binds from the first round-trip. + bounded_roundtrip(&conn, &mut queue, &mut state)?; + + let screencopy = state.screencopy.clone().ok_or("no zwlr_screencopy_manager_v1")?; + // Select this thread's output by the name the control connection assigned + // it; ordering fallback only when the host names nothing. + let by_name = expect_name.as_ref().and_then(|expect| { + state.outputs.iter().find(|(_, n)| n.as_ref() == Some(expect)).cloned() + }); + let (output, _name) = match by_name { + Some(o) => o, + None => { + let mut order: Vec = (0..state.outputs.len()).collect(); + order.sort_by_key(|&i| output_order_key(state.outputs[i].1.as_ref(), i)); + let oi = *order + .get(index) + .ok_or_else(|| format!("host has no wl_output {index}"))?; + state.outputs[oi].clone() + } + }; + let gbm = gbm_path + .as_ref() + .and_then(|p| File::options().read(true).write(true).open(p).ok()) + .and_then(|f| GbmDevice::new(f).ok()); + let wake = wake_rd.as_raw_fd(); + + let mut slots: Vec> = (0..SLOTS).map(|_| None).collect(); + let mut free: Vec = (0..SLOTS).collect(); + let mut gen: u64 = 0; + let mut announced: Option<(i32, i32)> = None; + let mut warned_mismatch = false; + let mut consecutive_failures = 0u32; + let mut want: Option<(i32, i32)> = None; + + 'main: loop { + // Drain control; block while idle or out of slots. + loop { + let blocking = want.is_none() || free.is_empty(); + let msg = if blocking { + match from_main.recv() { + Ok(m) => m, + Err(_) => return Ok(()), + } + } else { + match from_main.try_recv() { + Ok(m) => m, + Err(TryRecvError::Empty) => break, + Err(TryRecvError::Disconnected) => return Ok(()), + } + }; + match msg { + ToHost::Release { gen: g, slot } => { + if g == gen { + free.push(slot); + } + } + ToHost::Idle => want = None, + ToHost::Start { width, height } => { + if want != Some((width, height)) { + warned_mismatch = false; + } + want = Some((width, height)); + } + } + } + let (want_w, want_h) = match want { + Some(w) => w, + None => continue, + }; + + // One frame: negotiate, attach our buffer, wait for damage. Control + // messages (resize, idle, teardown) interrupt any of the waits. + state.reset_frame(); + let frame = screencopy.capture_output(1, &output, &qh, ()); + loop { + match pump_until(&conn, &mut queue, &mut state, wake, None, |s| { + s.buffer_done || s.failed + })? { + Pump::Done => break, + Pump::Control => match drain_ctl(&from_main, gen, &mut free, &mut want) { + Ctl::None => {} + Ctl::Renegotiate | Ctl::Idle => { + frame.destroy(); + continue 'main; + } + Ctl::Dead => { + frame.destroy(); + return Ok(()); + } + }, + Pump::Timeout => {} + } + } + if state.failed { + frame.destroy(); + consecutive_failures += 1; + if consecutive_failures == 3 { + eprintln!("[HostCapture] output {index}: repeated screencopy failures; is the output alive?"); + } + let _ = pump_until( + &conn, + &mut queue, + &mut state, + wake, + Some(Duration::from_millis(100)), + |_| false, + ); + continue; + } + + let (fw, fh) = state + .announce_dmabuf + .map(|(_, w, h)| (w, h)) + .or(state.announce_shm.map(|(_, w, h, _)| (w, h))) + .ok_or("screencopy announced no buffer type")?; + if announced != Some((fw, fh)) { + announced = Some((fw, fh)); + eprintln!( + "[HostCapture] output {index} negotiated: {fw}x{fh} gbm={} dmabuf_global={} dmabuf_announce={:?} shm={:?}", + gbm.is_some(), + state.dmabuf.is_some(), + state.announce_dmabuf, + state.announce_shm, + ); + // Size change: drop stale buffers. The generation bump makes any + // release still in flight for the old buffers a no-op. + for s in slots.iter_mut() { + *s = None; + } + free = (0..SLOTS).collect(); + gen += 1; + } + if (fw, fh) != (want_w, want_h) { + // The encoder is configured for (want_w, want_h): hold frames until + // the host applies the resize rather than encode mismatched buffers. + frame.destroy(); + if !warned_mismatch { + warned_mismatch = true; + eprintln!( + "[HostCapture] output {index}: waiting for host {want_w}x{want_h} (currently {fw}x{fh})." + ); + } + let _ = pump_until( + &conn, + &mut queue, + &mut state, + wake, + Some(Duration::from_millis(150)), + |_| false, + ); + continue; + } + warned_mismatch = false; + + let slot_idx = *free.last().unwrap(); + if slots[slot_idx].is_none() { + slots[slot_idx] = Some(match (&gbm, &state.dmabuf, state.announce_dmabuf) { + (Some(dev), Some(dmabuf_global), Some((fourcc, w, h))) => { + let bo = dev + .create_buffer_object::<()>( + w as u32, + h as u32, + fourcc_to_gbm(fourcc), + BufferObjectFlags::RENDERING, + ) + .map_err(|e| format!("GBM allocation {w}x{h}: {e:?}"))?; + let dmabuf = crate::create_dmabuf_from_bo(&bo); + let params = dmabuf_global.create_params(&qh, ()); + let modifier: u64 = dmabuf.format().modifier.into(); + for (i, handle) in dmabuf.handles().enumerate() { + params.add( + handle, + i as u32, + dmabuf.offsets().nth(i).unwrap_or(0), + dmabuf.strides().nth(i).unwrap_or(0), + (modifier >> 32) as u32, + (modifier & 0xffff_ffff) as u32, + ); + } + let wl = params.create_immed(w, h, fourcc, zwp_linux_buffer_params_v1::Flags::empty(), &qh, ()); + params.destroy(); + SlotBuffer::Gpu { _bo: bo, dmabuf, wl } + } + _ => { + let (format, w, h, stride) = + state.announce_shm.ok_or("no shm fallback announced")?; + let shm = state.shm.clone().ok_or("host compositor lacks wl_shm")?; + let size = (stride * h) as usize; + let fd = memfd_with(&vec![0u8; size])?; + let pool = shm.create_pool(fd.as_fd(), size as i32, &qh, ()); + let wl = pool.create_buffer( + 0, + w, + h, + stride, + WEnum::::from(format).into_result().unwrap_or(wl_shm::Format::Xrgb8888), + &qh, + (), + ); + let file = File::from(fd); + let map = unsafe { memmap2::MmapMut::map_mut(&file) } + .map_err(|e| format!("shm map: {e}"))?; + SlotBuffer::Cpu { _pool: pool, map: Arc::new(map), stride, format, wl } + } + }); + } + free.pop(); + + { + let slot = slots[slot_idx].as_ref().unwrap(); + let wl = match slot { + SlotBuffer::Gpu { wl, .. } => wl, + SlotBuffer::Cpu { wl, .. } => wl, + }; + frame.copy_with_damage(wl); + } + queue.flush().map_err(|e| format!("flush: {e}"))?; + let mut aborted = false; + loop { + match pump_until(&conn, &mut queue, &mut state, wake, None, |s| s.ready || s.failed)? { + Pump::Done => break, + Pump::Control => match drain_ctl(&from_main, gen, &mut free, &mut want) { + Ctl::None => {} + Ctl::Renegotiate | Ctl::Idle => { + aborted = true; + break; + } + Ctl::Dead => { + frame.destroy(); + return Ok(()); + } + }, + Pump::Timeout => {} + } + } + frame.destroy(); + if aborted { + free.push(slot_idx); + continue 'main; + } + if state.failed { + free.push(slot_idx); + consecutive_failures += 1; + if consecutive_failures == 3 { + eprintln!("[HostCapture] output {index}: repeated screencopy failures; is the output alive?"); + } + let _ = pump_until( + &conn, + &mut queue, + &mut state, + wake, + Some(Duration::from_millis(50)), + |_| false, + ); + continue; + } + consecutive_failures = 0; + + let damage = std::mem::take(&mut state.damage); + let out = match slots[slot_idx].as_ref().unwrap() { + SlotBuffer::Gpu { dmabuf, .. } => HostFrame { + gen, + slot: slot_idx, + dmabuf: Some(dmabuf.clone()), + cpu: None, + width: fw, + height: fh, + damage, + }, + SlotBuffer::Cpu { map, stride, format, .. } => HostFrame { + gen, + slot: slot_idx, + dmabuf: None, + cpu: Some(HostCpuFrame { + map: map.clone(), + stride: *stride as usize, + format: *format, + }), + width: fw, + height: fh, + damage, + }, + }; + if frame_tx.send(out).is_err() { + return Ok(()); + } + } +} + +/// Drain the control channel from inside a capture wait: releases are applied +/// (generation-checked), and the newest Start/Idle decides the wait's fate. +fn drain_ctl( + rx: &Receiver, + gen: u64, + free: &mut Vec, + want: &mut Option<(i32, i32)>, +) -> Ctl { + let mut out = Ctl::None; + loop { + match rx.try_recv() { + Ok(ToHost::Release { gen: g, slot }) => { + if g == gen { + free.push(slot); + } + } + Ok(ToHost::Start { width, height }) => { + if *want != Some((width, height)) { + *want = Some((width, height)); + out = Ctl::Renegotiate; + } + } + Ok(ToHost::Idle) => { + *want = None; + out = Ctl::Idle; + } + Err(TryRecvError::Empty) => return out, + Err(TryRecvError::Disconnected) => return Ctl::Dead, + } + } +} diff --git a/pixelflux/src/wayland/keymap.rs b/pixelflux/src/wayland/keymap.rs new file mode 100644 index 0000000..0233284 --- /dev/null +++ b/pixelflux/src/wayland/keymap.rs @@ -0,0 +1,475 @@ +/* + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at https://mozilla.org/MPL/2.0/. + */ + +//! Compositor-side keymap policy: one owner for the seat keymap. +//! +//! The seat keymap is BASE text (US by default, replaceable at runtime with a full +//! XKB_KEYMAP_FORMAT_TEXT_V1 string or RMLVO names) plus an OVERLAY of spare keycodes bound to +//! keysyms the base cannot produce (Unicode / IME output). All rebinds are batched: resolving N +//! new keysyms produces ONE keymap swap, and a keycode that is currently held down is never +//! recycled, so its release event always means the symbol its press meant. + +use std::collections::{HashMap, HashSet, VecDeque}; + +use smithay::input::keyboard::xkb; + +/// First overlay keycode. Sits above both the evdev/pc105 range and the legacy selkies +/// overlay range (257-272) so a base keymap carrying those legacy binds cannot collide. +pub const OVERLAY_FIRST_KEYCODE: u32 = 0x120; +/// Last overlay keycode (inclusive). Keycodes past the X11 255 ceiling are fine for +/// pure-Wayland clients: they look keycodes up in the delivered keymap via xkbcommon. +pub const OVERLAY_LAST_KEYCODE: u32 = 0x2ff; +/// Overlay slot count (keycodes `OVERLAY_FIRST..=OVERLAY_LAST`). +pub const OVERLAY_CAPACITY: usize = (OVERLAY_LAST_KEYCODE - OVERLAY_FIRST_KEYCODE + 1) as usize; + +/// Highest shift level consulted when reverse-mapping the base keymap (plain, Shift, +/// AltGr, Shift+AltGr). +const MAX_LEVELS: u32 = 4; + +/// Seat keymap state: the base text, its reverse keysym map, and the overlay slots. +pub struct KeymapPolicy { + base_text: String, + /// keysym -> (xkb keycode, level) in the base keymap; lowest level wins. + base_map: HashMap, + /// slot index -> bound keysym. + slots: Vec>, + /// keysym -> slot index. + by_sym: HashMap, + /// Slot recycle order, oldest bind first. + lru: VecDeque, + /// First overlay keycode (xkb numbering); slot i lives at `overlay_first + i`. + overlay_first: u32, + /// Overlay slot count. + overlay_capacity: usize, +} + +/// Keysym one literal character types as: Latin-1 printables map 1:1, `\n` types +/// Return (the raw utf32 table maps it to Linefeed, which no keymap binds), other +/// control characters their `0xffXX` function keysyms, everything else the +/// `0x01000000 | codepoint` Unicode form (0 = unmappable). The keysym then resolves +/// against an ACTIVE keymap, never a hardcoded layout table. +pub fn keysym_for_char(c: char) -> u32 { + let c = if c == '\n' { '\r' } else { c }; + xkb::utf32_to_keysym(c as u32).raw() +} + +/// Compile an XKB_KEYMAP_FORMAT_TEXT_V1 string, or `None` when it does not compile. +pub fn compile_keymap(text: &str) -> Option { + let ctx = xkb::Context::new(xkb::CONTEXT_NO_FLAGS); + xkb::Keymap::new_from_string( + &ctx, + text.to_string(), + xkb::KEYMAP_FORMAT_TEXT_V1, + xkb::KEYMAP_COMPILE_NO_FLAGS, + ) +} + +/// Compile RMLVO names to keymap text, or `None` when compilation fails. Empty strings +/// select the xkbcommon defaults for that component. +pub fn compile_rmlvo( + rules: &str, + model: &str, + layout: &str, + variant: &str, + options: &str, +) -> Option { + let ctx = xkb::Context::new(xkb::CONTEXT_NO_FLAGS); + let options = (!options.is_empty()).then(|| options.to_string()); + let keymap = xkb::Keymap::new_from_names( + &ctx, + rules, + model, + layout, + variant, + options, + xkb::KEYMAP_COMPILE_NO_FLAGS, + )?; + Some(keymap.get_as_string(xkb::KEYMAP_FORMAT_TEXT_V1)) +} + +/// Level-0 keysym for every key of a compiled keymap (used to pre-bind a virtual-keyboard +/// client's keymap in one batch). Keys with zero or multiple level-0 syms are skipped. +pub fn level0_syms(keymap: &xkb::Keymap) -> HashMap { + let mut out = HashMap::new(); + let lo = keymap.min_keycode().raw(); + let hi = keymap.max_keycode().raw(); + for kc in lo..=hi { + let syms = keymap.key_get_syms_by_level(xkb::Keycode::new(kc), 0, 0); + if syms.len() == 1 { + let sym = syms[0].raw(); + if sym != 0 { + out.insert(kc, sym); + } + } + } + out +} + +impl KeymapPolicy { + /// Placeholder policy before the seat keymap is known; `rebuild_base` fills it in. + pub fn empty() -> Self { + Self::with_overlay_range(OVERLAY_FIRST_KEYCODE, OVERLAY_LAST_KEYCODE) + } + + /// Policy with a custom overlay keycode range (inclusive, xkb numbering). The seat + /// uses `empty()`'s above-255 range (pure-Wayland clients resolve it fine); the + /// virtual-keyboard client typing into a nested compositor uses a sub-256 range so + /// XWayland apps under that compositor stay reachable. + pub fn with_overlay_range(first: u32, last: u32) -> Self { + Self { + base_text: String::new(), + base_map: HashMap::new(), + slots: Vec::new(), + by_sym: HashMap::new(), + lru: VecDeque::new(), + overlay_first: first, + overlay_capacity: (last - first + 1) as usize, + } + } + + /// Replace the base keymap text and rebuild the reverse map. Overlay assignments are + /// retained (same keycodes), so keycodes already handed out stay valid across the swap. + /// Returns whether `base_text` compiled. A string that does not is rejected without + /// touching any state, so the caller needs no separate validation pass — compiling a + /// keymap is the expensive part of installing one and doing it twice is pure cost. + pub fn rebuild_base(&mut self, base_text: String) -> bool { + let Some(keymap) = compile_keymap(&base_text) else { + return false; + }; + self.base_map.clear(); + { + let lo = keymap.min_keycode().raw(); + let hi = keymap.max_keycode().raw(); + // Lower levels win across ALL keys, so a keysym reachable unshifted never + // resolves to a shifted position. + for level in 0..MAX_LEVELS { + for kc in lo..=hi { + let code = xkb::Keycode::new(kc); + if keymap.num_levels_for_key(code, 0) <= level { + continue; + } + for sym in keymap.key_get_syms_by_level(code, 0, level) { + let raw = sym.raw(); + if raw != 0 { + self.base_map.entry(raw).or_insert((kc, level)); + } + } + } + } + } + self.base_text = base_text; + true + } + + /// True once a base keymap has been installed. + pub fn has_base(&self) -> bool { + !self.base_text.is_empty() + } + + /// Resolve `keysym` without binding: base first, then an existing overlay slot. + pub fn resolve(&self, keysym: u32) -> Option<(u32, u32)> { + if let Some(&hit) = self.base_map.get(&keysym) { + return Some(hit); + } + self.by_sym + .get(&keysym) + .map(|&slot| (self.overlay_first + slot as u32, 0)) + } + + /// True when `keysym` resolves at level 0 (base or overlay) — i.e. typable without + /// synthetic modifiers. + pub fn resolves_plain(&self, keysym: u32) -> bool { + matches!(self.resolve(keysym), Some((_, 0))) + } + + /// Resolve every keysym, overlay-binding the unresolvable ones. Returns one + /// `(keycode, level)` per input keysym (`(0, 0)` when it cannot be bound) plus whether the + /// keymap changed and must be re-applied — at most ONE swap per call, however many new + /// keysyms were bound. Slots whose keycode is in `pressed` are never recycled. + pub fn bind_many( + &mut self, + keysyms: &[u32], + pressed: &HashSet, + ) -> (Vec<(u32, u32)>, bool) { + let mut out = Vec::with_capacity(keysyms.len()); + let mut changed = false; + for &sym in keysyms { + out.push(self.bind_one(sym, pressed, false, &mut changed)); + } + (out, changed) + } + + /// Like `bind_many` but only accepts level-0 resolutions: a keysym reachable in the + /// base solely behind a modifier (e.g. `A` behind Shift) is overlay-bound instead, so the + /// caller can inject it without synthesizing modifiers. Returns keycodes (0 = unbindable). + pub fn bind_many_plain(&mut self, keysyms: &[u32], pressed: &HashSet) -> (Vec, bool) { + let mut out = Vec::with_capacity(keysyms.len()); + let mut changed = false; + for &sym in keysyms { + out.push(self.bind_one(sym, pressed, true, &mut changed).0); + } + (out, changed) + } + + fn bind_one( + &mut self, + sym: u32, + pressed: &HashSet, + plain_only: bool, + changed: &mut bool, + ) -> (u32, u32) { + if sym == 0 { + return (0, 0); + } + if let Some(&(kc, level)) = self.base_map.get(&sym) { + if !plain_only || level == 0 { + return (kc, level); + } + } + if let Some(&slot) = self.by_sym.get(&sym) { + if let Some(at) = self.lru.iter().position(|&s| s == slot) { + self.lru.remove(at); + } + self.lru.push_back(slot); + return (self.overlay_first + slot as u32, 0); + } + let slot = if self.slots.len() < self.overlay_capacity { + self.slots.push(None); + self.slots.len() - 1 + } else { + match self.recycle_slot(pressed) { + Some(s) => s, + None => return (0, 0), + } + }; + if let Some(old) = self.slots[slot].replace(sym) { + self.by_sym.remove(&old); + } + self.by_sym.insert(sym, slot); + self.lru.push_back(slot); + *changed = true; + (self.overlay_first + slot as u32, 0) + } + + /// Oldest slot whose keycode is not currently held down; a held keycode must keep its + /// meaning until its release has been delivered. + fn recycle_slot(&mut self, pressed: &HashSet) -> Option { + let at = self + .lru + .iter() + .position(|&slot| !pressed.contains(&(self.overlay_first + slot as u32)))?; + self.lru.remove(at) + } + + /// The full seat keymap: the base text with every occupied overlay slot spliced into the + /// `xkb_keycodes` and `xkb_symbols` sections (and `maximum` raised to cover them). With no + /// overlays, the base text verbatim. + /// Splice explicit `(keycode, keysym)` binds onto the current base text. + /// + /// Which keysym lands on which keycode is the caller's decision — that part tracks + /// layouts and user reports and belongs where it can be changed quickly. Assembling + /// the xkb text is fixed grammar, so it is done here: no 30 KB string is built in the + /// caller, and the base keymap is reused rather than re-supplied and recompiled. + /// `None` when the base has no recognizable keycodes/symbols sections. + pub fn overlay_text_for(&self, binds: &[(u32, u32)]) -> Option { + if self.base_text.is_empty() { + return None; + } + if binds.is_empty() { + return Some(self.base_text.clone()); + } + let base = &self.base_text; + let max_at = base.find("maximum = ")?; + let num_at = max_at + "maximum = ".len(); + let num_len = base[num_at..].find(';')?; + let old_max: u32 = base[num_at..num_at + num_len].trim().parse().unwrap_or(255); + let need_max = binds.iter().map(|&(kc, _)| kc).max().unwrap_or(0); + let mut text = String::with_capacity(base.len() + binds.len() * 48); + text.push_str(&base[..num_at]); + text.push_str(&old_max.max(need_max).to_string()); + let rest = &base[num_at + num_len..]; + let kc_end = rest.find("};")?; + text.push_str(&rest[..kc_end]); + for &(kc, _) in binds { + text.push_str(&format!("\t = {kc};\n")); + } + let rest = &rest[kc_end..]; + let close_at = rest + .find("xkb_symbols") + .and_then(|sym_at| Self::section_close(rest, sym_at))?; + text.push_str(&rest[..close_at]); + for &(kc, sym) in binds { + text.push_str(&format!("\tkey {{ [ {sym:#x} ] }};\n")); + } + text.push_str(&rest[close_at..]); + Some(text) + } + + pub fn keymap_text(&self) -> String { + let occupied: Vec<(usize, u32)> = self + .slots + .iter() + .enumerate() + .filter_map(|(i, s)| s.map(|sym| (i, sym))) + .collect(); + if occupied.is_empty() { + return self.base_text.clone(); + } + let base = &self.base_text; + let Some(max_at) = base.find("maximum = ") else { + return self.base_text.clone(); + }; + let num_at = max_at + "maximum = ".len(); + let Some(num_len) = base[num_at..].find(';') else { + return self.base_text.clone(); + }; + let old_max: u32 = base[num_at..num_at + num_len].trim().parse().unwrap_or(255); + let need_max = self.overlay_first + occupied.last().map(|&(i, _)| i as u32).unwrap_or(0); + let mut text = String::with_capacity(base.len() + occupied.len() * 48); + text.push_str(&base[..num_at]); + text.push_str(&old_max.max(need_max).to_string()); + let rest = &base[num_at + num_len..]; + // First "};" after the maximum line closes xkb_keycodes. + let Some(kc_end) = rest.find("};") else { + return self.base_text.clone(); + }; + text.push_str(&rest[..kc_end]); + for &(i, _) in &occupied { + text.push_str(&format!("\t = {};\n", i, self.overlay_first + i as u32)); + } + let rest = &rest[kc_end..]; + let Some(close_at) = rest + .find("xkb_symbols") + .and_then(|sym_at| Self::section_close(rest, sym_at)) + else { + return self.base_text.clone(); + }; + text.push_str(&rest[..close_at]); + for &(i, sym) in &occupied { + text.push_str(&format!("\tkey {{ [ {:#x} ] }};\n", i, sym)); + } + text.push_str(&rest[close_at..]); + text + } + + /// Byte offset of the `}` closing the brace-block that starts at/after `from`. + fn section_close(text: &str, from: usize) -> Option { + let open = from + text[from..].find('{')?; + let mut depth = 0usize; + for (i, ch) in text[open..].char_indices() { + match ch { + '{' => depth += 1, + '}' => { + depth -= 1; + if depth == 0 { + return Some(open + i); + } + } + _ => {} + } + } + None + } +} + +#[cfg(test)] +mod tests { + //! Invariants: one `bind_many` call binds any number of new keysyms with a single + //! keymap change; base keysyms resolve without consuming overlay slots; a pressed + //! overlay keycode survives LRU pressure; the spliced keymap text compiles and + //! resolves the overlay keysyms at their assigned keycodes. + use super::*; + + fn us_base() -> String { + compile_rmlvo("", "", "us", "", "").expect("us keymap") + } + + fn policy() -> KeymapPolicy { + let mut p = KeymapPolicy::empty(); + p.rebuild_base(us_base()); + p + } + + #[test] + fn base_keysyms_resolve_without_overlay() { + let mut p = policy(); + // 'a' plain, 'A' shifted. + let (out, changed) = p.bind_many(&[0x61, 0x41], &HashSet::new()); + assert!(!changed); + assert_eq!(out[0].1, 0); + assert_eq!(out[1].0, out[0].0); + assert_eq!(out[1].1, 1); + } + + #[test] + fn batch_bind_is_one_swap_and_compiles() { + let mut p = policy(); + let syms: Vec = (0..30).map(|i| 0x1004E00 + i).collect(); + let (out, changed) = p.bind_many(&syms, &HashSet::new()); + assert!(changed); + let (_, changed_again) = p.bind_many(&syms, &HashSet::new()); + assert!(!changed_again, "re-binding bound keysyms must not swap"); + let text = p.keymap_text(); + let km = compile_keymap(&text).expect("overlay keymap compiles"); + for (i, &(kc, level)) in out.iter().enumerate() { + assert_eq!(level, 0); + let got = km.key_get_syms_by_level(xkb::Keycode::new(kc), 0, 0); + assert_eq!(got.len(), 1, "keycode {kc} has one sym"); + assert_eq!(got[0].raw(), syms[i]); + } + } + + #[test] + fn pressed_keycode_is_never_recycled() { + let mut p = policy(); + let syms: Vec = (0..OVERLAY_CAPACITY as u32).map(|i| 0x1005000 + i).collect(); + let (out, _) = p.bind_many(&syms, &HashSet::new()); + let held_kc = out[0].0; + let held_sym = syms[0]; + let pressed: HashSet = [held_kc].into_iter().collect(); + // Force full recycling pressure past capacity. + let extra: Vec = (0..8).map(|i| 0x1006000 + i).collect(); + let (extra_out, changed) = p.bind_many(&extra, &pressed); + assert!(changed); + for &(kc, _) in &extra_out { + assert_ne!(kc, held_kc, "held keycode must not be rebound"); + } + assert_eq!(p.resolve(held_sym), Some((held_kc, 0))); + } + + #[test] + fn sub256_overlay_range_overrides_base_keycode_names() { + // The virtual-keyboard client's range collides with keycodes the base + // already names (…); the spliced definitions must win so overlay + // keysyms resolve at their assigned keycodes. + let mut p = KeymapPolicy::with_overlay_range(150, 255); + p.rebuild_base(us_base()); + let (out, changed) = p.bind_many_plain(&[0x1004E2D, 0x61], &HashSet::new()); + assert!(changed); + assert_eq!(out[0], 150); + let km = compile_keymap(&p.keymap_text()).expect("sub-256 overlay keymap compiles"); + let got = km.key_get_syms_by_level(xkb::Keycode::new(out[0]), 0, 0); + assert_eq!(got.len(), 1); + assert_eq!(got[0].raw(), 0x1004E2D); + // 'a' resolves plain in the base without consuming a slot. + assert!(out[1] < 150); + assert_eq!(out[1], p.resolve(0x61).unwrap().0); + } + + #[test] + fn rebuild_base_keeps_overlay_assignments() { + let mut p = policy(); + let (out, _) = p.bind_many(&[0x1004E2D], &HashSet::new()); + let de = compile_rmlvo("", "", "de", "", "").expect("de keymap"); + p.rebuild_base(de); + assert_eq!(p.resolve(0x1004E2D), Some((out[0].0, 0))); + // udiaeresis resolves in the German base without an overlay. + let (u_out, changed) = p.bind_many(&[0xFC], &HashSet::new()); + assert!(!changed); + assert_eq!(u_out[0].1, 0); + assert!(compile_keymap(&p.keymap_text()).is_some()); + } +} diff --git a/pixelflux/src/wayland/mod.rs b/pixelflux/src/wayland/mod.rs index f2d0679..b9e94a3 100644 --- a/pixelflux/src/wayland/mod.rs +++ b/pixelflux/src/wayland/mod.rs @@ -11,5 +11,15 @@ /// Headless Smithay compositor, protocol handlers, and input routing. pub mod frontend; -/// Wayland cursor shape to PNG resolution. +/// Wayland cursor shape to PNG resolution and the cursor delivery worker. pub mod cursor; +/// Seat keymap ownership: base layout plus batched overlay keysym binding. +pub mod keymap; +/// Virtual-keyboard client for typing into a nested app compositor's socket. +pub mod vkclient; +/// Shared plumbing for outbound Wayland client connections. +pub mod wlclient; +/// Data-control clipboard client bridging a nested app compositor's selection. +pub mod dcclient; +/// Host-capture mode: capture/inject as a client of an external compositor. +pub mod host; diff --git a/pixelflux/src/wayland/vkclient.rs b/pixelflux/src/wayland/vkclient.rs new file mode 100644 index 0000000..9b2a5fa --- /dev/null +++ b/pixelflux/src/wayland/vkclient.rs @@ -0,0 +1,157 @@ +//! `zwp_virtual_keyboard_v1` client for typing Unicode text into ANOTHER Wayland +//! compositor's socket. +//! +//! pixelflux is normally the compositor, but in a nested deployment (a labwc/kwin +//! session running as a client of pixelflux) the apps live on that inner +//! compositor's socket, and keys injected into pixelflux's own seat resolve +//! against pixelflux's keymap — an overlay the inner compositor never sees. So +//! text is typed here as a client of whichever compositor the apps live under — +//! by Computer-Use actions and by selkies over the `type_text_wayland` ABI — +//! reusing the seat's [`KeymapPolicy`] over a US base: base-reachable characters +//! press their ordinary keycodes, everything else is overlay-bound, one upload +//! per batch. One-shot (connect, type, disconnect): a call carries a whole commit +//! of text, and a fresh connection leaves no stale-socket state to manage. +//! Blocking, off the compositor thread, with every round-trip deadline-bounded so +//! a wedged compositor cannot hang the caller forever. + +use std::collections::HashSet; +use std::os::fd::AsFd; +use std::os::unix::net::UnixStream; +use std::sync::OnceLock; +use std::time::Duration; + +use wayland_client::protocol::{wl_registry, wl_seat}; +use wayland_client::{delegate_noop, Connection, Dispatch, EventQueue, QueueHandle}; +use wayland_protocols_misc::zwp_virtual_keyboard_v1::client::{ + zwp_virtual_keyboard_manager_v1::ZwpVirtualKeyboardManagerV1, + zwp_virtual_keyboard_v1::ZwpVirtualKeyboardV1, +}; + +use crate::wayland::keymap::{compile_rmlvo, keysym_for_char, KeymapPolicy}; +use crate::wayland::wlclient::{bounded_roundtrip, impl_sync_callback, memfd_with, SyncState}; + +/// Overlay keycodes stay under the X11 255 ceiling so XWayland apps under the app +/// compositor can still receive them (the seat's own overlay sits above 255). +const OVERLAY_FIRST_XKB: u32 = 150; +const OVERLAY_LAST_XKB: u32 = 255; +const OVERLAY_SLOTS: usize = (OVERLAY_LAST_XKB - OVERLAY_FIRST_XKB + 1) as usize; +/// wl_keyboard / zwp_virtual_keyboard key events carry evdev codes (xkb - 8). +const EVDEV_OFFSET: u32 = 8; +const KEYMAP_FORMAT_XKB_V1: u32 = 1; + +#[derive(Default)] +struct Globals { + seat: Option, + manager: Option, + sync_done: bool, +} + +impl SyncState for Globals { + fn sync_done_mut(&mut self) -> &mut bool { + &mut self.sync_done + } +} +impl_sync_callback!(Globals); + +impl Dispatch for Globals { + fn event( + state: &mut Self, + registry: &wl_registry::WlRegistry, + event: wl_registry::Event, + _: &(), + _: &Connection, + qh: &QueueHandle, + ) { + if let wl_registry::Event::Global { name, interface, .. } = event { + // Version 1 of each suffices: the seat is only the manager argument. + match interface.as_str() { + "wl_seat" if state.seat.is_none() => { + state.seat = Some(registry.bind(name, 1, qh, ())); + } + "zwp_virtual_keyboard_manager_v1" if state.manager.is_none() => { + state.manager = Some(registry.bind(name, 1, qh, ())); + } + _ => {} + } + } + } +} + +delegate_noop!(Globals: ignore wl_seat::WlSeat); +delegate_noop!(Globals: ZwpVirtualKeyboardManagerV1); +delegate_noop!(Globals: ZwpVirtualKeyboardV1); + +/// The US base keymap text, compiled once per process: selkies types per text +/// commit, and xkbcommon compilation is the expensive part of a call. +pub(crate) fn us_base_text() -> Option<&'static str> { + static CACHE: OnceLock> = OnceLock::new(); + CACHE.get_or_init(|| compile_rmlvo("", "", "us", "", "")).as_deref() +} + +fn upload_keymap( + vk: &ZwpVirtualKeyboardV1, + queue: &mut EventQueue, + text: &str, +) -> Result<(), String> { + let mut data = text.as_bytes().to_vec(); + data.push(0); // compositors parse the mapping as a NUL-terminated string + let fd = memfd_with(&data)?; + vk.keymap(KEYMAP_FORMAT_XKB_V1, fd.as_fd(), data.len() as u32); + queue.flush().map_err(|e| format!("flush keymap: {e}")) +} + +/// One-shot: connect to `socket_path`, type `text` in order, disconnect. Codepoints +/// with no keysym are skipped. Blocking; call off the compositor's calloop thread. +pub fn type_text_to(socket_path: &str, text: &str) -> Result<(), String> { + let stream = + UnixStream::connect(socket_path).map_err(|e| format!("connect {socket_path}: {e}"))?; + let conn = Connection::from_socket(stream).map_err(|e| format!("wayland setup: {e}"))?; + let mut queue = conn.new_event_queue(); + let qh = queue.handle(); + let _registry = conn.display().get_registry(&qh, ()); + let mut state = Globals::default(); + bounded_roundtrip(&conn, &mut queue, &mut state)?; + let seat = state.seat.take().ok_or("app compositor advertises no wl_seat")?; + let manager = state + .manager + .take() + .ok_or("app compositor does not advertise zwp_virtual_keyboard_manager_v1")?; + let vk = manager.create_virtual_keyboard(&seat, &qh, ()); + // Surfaces an "unauthorized" bind error before the first keymap upload. + bounded_roundtrip(&conn, &mut queue, &mut state)?; + + let mut policy = KeymapPolicy::with_overlay_range(OVERLAY_FIRST_XKB, OVERLAY_LAST_XKB); + policy.rebuild_base( + us_base_text().ok_or("us base keymap failed to compile")?.to_string(), + ); + let syms: Vec = text.chars().map(keysym_for_char).filter(|&s| s != 0).collect(); + let none = HashSet::new(); + let mut uploaded = false; + // Chunk bound: at most OVERLAY_SLOTS keysyms per bind call, so a batch can + // never recycle a slot it assigned earlier in the same batch. + for chunk in syms.chunks(OVERLAY_SLOTS) { + let (keycodes, changed) = policy.bind_many_plain(chunk, &none); + // The protocol requires a keymap before the first key event even when the + // whole text resolves in the base. + if changed || !uploaded { + upload_keymap(&vk, &mut queue, &policy.keymap_text())?; + bounded_roundtrip(&conn, &mut queue, &mut state)?; + std::thread::sleep(Duration::from_millis(10)); + uploaded = true; + } + for &kc in &keycodes { + if kc < EVDEV_OFFSET { + continue; // unbindable keysym + } + for pressed in [1u32, 0] { + vk.key(0, kc - EVDEV_OFFSET, pressed); + queue.flush().map_err(|e| format!("flush key: {e}"))?; + std::thread::sleep(Duration::from_millis(2)); + } + } + } + bounded_roundtrip(&conn, &mut queue, &mut state)?; + vk.destroy(); + let _ = queue.flush(); + Ok(()) +} diff --git a/pixelflux/src/wayland/wlclient.rs b/pixelflux/src/wayland/wlclient.rs new file mode 100644 index 0000000..6a3850e --- /dev/null +++ b/pixelflux/src/wayland/wlclient.rs @@ -0,0 +1,295 @@ +//! Shared plumbing for pixelflux's outbound Wayland CLIENT connections (the +//! virtual-keyboard typer and the data-control clipboard bridge, both talking to +//! a nested app compositor): socket resolution, deadline-bounded round-trips so a +//! wedged compositor turns into an error instead of a hang, and fd read/write +//! helpers for clipboard pipes. + +use std::os::fd::{AsRawFd, FromRawFd, OwnedFd, RawFd}; +use std::time::{Duration, Instant}; + +use wayland_client::protocol::wl_callback; +use wayland_client::{Connection, Dispatch, EventQueue}; + +pub(crate) const IO_TIMEOUT: Duration = Duration::from_secs(5); + +/// Absolute socket path for a Wayland display name (absolute paths pass through, +/// names join XDG_RUNTIME_DIR). +pub(crate) fn socket_path(name: &str) -> Option { + if name.starts_with('/') { + return Some(name.to_string()); + } + let rt = std::env::var("XDG_RUNTIME_DIR").ok()?; + Some(format!("{}/{}", rt.trim_end_matches('/'), name)) +} + +/// Implemented by every client state so [`bounded_roundtrip`] can flag the sync +/// callback's completion; pair with [`impl_sync_callback`]. +pub(crate) trait SyncState { + fn sync_done_mut(&mut self) -> &mut bool; +} + +/// `Dispatch` for a [`SyncState`] type (a blanket impl would violate +/// the orphan rule, so each state stamps its own). +macro_rules! impl_sync_callback { + ($t:ty) => { + impl wayland_client::Dispatch + for $t + { + fn event( + state: &mut Self, + _: &wayland_client::protocol::wl_callback::WlCallback, + event: wayland_client::protocol::wl_callback::Event, + _: &(), + _: &wayland_client::Connection, + _: &wayland_client::QueueHandle, + ) { + if let wayland_client::protocol::wl_callback::Event::Done { .. } = event { + *crate::wayland::wlclient::SyncState::sync_done_mut(state) = true; + } + } + } + }; +} +pub(crate) use impl_sync_callback; + +/// Block until `fd` is readable or `timeout` passes (false = timed out). +pub(crate) fn wait_readable(fd: RawFd, timeout: Duration) -> Result { + loop { + let mut pfd = libc::pollfd { fd, events: libc::POLLIN, revents: 0 }; + let n = unsafe { libc::poll(&mut pfd, 1, timeout.as_millis().max(1) as libc::c_int) }; + if n < 0 { + let err = std::io::Error::last_os_error(); + if err.raw_os_error() == Some(libc::EINTR) { + continue; + } + return Err(format!("poll: {err}")); + } + return Ok(n > 0); + } +} + +/// Poll two fds for readability at once (wayland socket + a wake pipe); returns +/// `(a_readable, b_readable)`, both false on timeout. Hangup counts as readable +/// so a closed wake pipe unblocks its waiter. +pub(crate) fn wait_readable2( + a: RawFd, + b: RawFd, + timeout: Option, +) -> Result<(bool, bool), String> { + loop { + let mut pfds = [ + libc::pollfd { fd: a, events: libc::POLLIN, revents: 0 }, + libc::pollfd { fd: b, events: libc::POLLIN, revents: 0 }, + ]; + let ms = timeout.map(|t| t.as_millis().max(1) as libc::c_int).unwrap_or(-1); + let n = unsafe { libc::poll(pfds.as_mut_ptr(), 2, ms) }; + if n < 0 { + let err = std::io::Error::last_os_error(); + if err.raw_os_error() == Some(libc::EINTR) { + continue; + } + return Err(format!("poll: {err}")); + } + let hit = |r: libc::c_short| r & (libc::POLLIN | libc::POLLHUP | libc::POLLERR) != 0; + return Ok((hit(pfds[0].revents), hit(pfds[1].revents))); + } +} + +/// Drain every pending byte from a wake pipe without blocking. +pub(crate) fn drain_pipe(fd: RawFd) { + let mut buf = [0u8; 64]; + loop { + let n = unsafe { libc::read(fd, buf.as_mut_ptr() as *mut libc::c_void, buf.len()) }; + if n <= 0 { + return; + } + } +} + +/// One wake byte, non-blocking; a full pipe already guarantees a pending wake. +pub(crate) fn wake_write(fd: RawFd) { + let b = [1u8]; + unsafe { libc::write(fd, b.as_ptr() as *const libc::c_void, 1) }; +} + +/// `EventQueue::roundtrip` with a deadline: a wedged compositor becomes an error +/// instead of hanging the calling thread (and everything queued behind it). +pub(crate) fn bounded_roundtrip( + conn: &Connection, + queue: &mut EventQueue, + state: &mut S, +) -> Result<(), String> +where + S: SyncState + Dispatch + 'static, +{ + *state.sync_done_mut() = false; + let _cb = conn.display().sync(&queue.handle(), ()); + let deadline = Instant::now() + IO_TIMEOUT; + loop { + queue + .dispatch_pending(state) + .map_err(|e| format!("dispatch: {e}"))?; + if *state.sync_done_mut() { + return Ok(()); + } + queue.flush().map_err(|e| format!("flush: {e}"))?; + let remaining = deadline + .checked_duration_since(Instant::now()) + .ok_or("compositor round-trip timed out")?; + let Some(guard) = conn.prepare_read() else { + continue; + }; + if !wait_readable(guard.connection_fd().as_raw_fd(), remaining)? { + return Err("compositor round-trip timed out".into()); + } + guard.read().map_err(|e| format!("read: {e}"))?; + } +} + +/// Anonymous CLOEXEC memfd holding `data` (keymap uploads, shm-style payloads). +pub(crate) fn memfd_with(data: &[u8]) -> Result { + let name = b"pixelflux-wl\0"; + let fd = + unsafe { libc::memfd_create(name.as_ptr() as *const libc::c_char, libc::MFD_CLOEXEC) }; + if fd < 0 { + return Err(format!("memfd_create: {}", std::io::Error::last_os_error())); + } + let owned = unsafe { OwnedFd::from_raw_fd(fd) }; + let mut written = 0; + while written < data.len() { + let n = unsafe { + libc::write( + owned.as_raw_fd(), + data[written..].as_ptr() as *const libc::c_void, + data.len() - written, + ) + }; + if n < 0 { + return Err(format!("write memfd: {}", std::io::Error::last_os_error())); + } + written += n as usize; + } + Ok(owned) +} + +/// CLOEXEC pipe as (read end, write end). +pub(crate) fn pipe_cloexec() -> Result<(OwnedFd, OwnedFd), String> { + let mut fds = [0i32; 2]; + if unsafe { libc::pipe2(fds.as_mut_ptr(), libc::O_CLOEXEC) } < 0 { + return Err(format!("pipe2: {}", std::io::Error::last_os_error())); + } + Ok(unsafe { (OwnedFd::from_raw_fd(fds[0]), OwnedFd::from_raw_fd(fds[1])) }) +} + +/// Non-blocking CLOEXEC pipe for wake signalling (read end, write end). +pub(crate) fn wake_pipe() -> Result<(OwnedFd, OwnedFd), String> { + let mut fds = [0i32; 2]; + if unsafe { libc::pipe2(fds.as_mut_ptr(), libc::O_CLOEXEC | libc::O_NONBLOCK) } < 0 { + return Err(format!("pipe2: {}", std::io::Error::last_os_error())); + } + Ok(unsafe { (OwnedFd::from_raw_fd(fds[0]), OwnedFd::from_raw_fd(fds[1])) }) +} + +/// Read `fd` to EOF. The deadline is per-chunk (idle), so a large transfer that +/// keeps flowing is never cut off while a stalled writer still errors out. +pub(crate) fn read_fd_to_end(fd: &OwnedFd, idle: Duration) -> Result, String> { + read_fd_to_end_capped(fd, idle, READ_CAP_MAX) +} + +/// Ceiling on an unbounded clipboard source, matching what the selkies clipboard protocol +/// allows: a hostile or stuck peer could otherwise stream bytes into memory forever. +const READ_CAP_MAX: usize = 64 * 1024 * 1024; + +/// Same as `read_fd_to_end` but refuses a source larger than `cap`. +/// +/// `cap` is inclusive: a payload of exactly `cap` bytes is accepted. The limit is therefore +/// tested against the size this chunk *would* produce, before appending — a test on the +/// already-accumulated length is reached again before EOF can be observed, so it would reject +/// a payload that exactly fills the cap. +pub(crate) fn read_fd_to_end_capped( + fd: &OwnedFd, + idle: Duration, + cap: usize, +) -> Result, String> { + let mut out = Vec::new(); + let mut chunk = [0u8; 65536]; + loop { + if !wait_readable(fd.as_raw_fd(), idle)? { + return Err("clipboard source stalled".into()); + } + let n = unsafe { + libc::read(fd.as_raw_fd(), chunk.as_mut_ptr() as *mut libc::c_void, chunk.len()) + }; + if n < 0 { + let err = std::io::Error::last_os_error(); + if err.raw_os_error() == Some(libc::EINTR) { + continue; + } + return Err(format!("read: {err}")); + } + if n == 0 { + return Ok(out); + } + if out.len() + n as usize > cap { + return Err(format!("clipboard source exceeds the {cap} byte cap")); + } + out.extend_from_slice(&chunk[..n as usize]); + } +} + +/// Write all of `data` to `fd`, tolerating a slow reader up to `idle` per chunk. +/// EPIPE is success-shaped: the paster stopped reading, which is its right. +/// +/// The fd is switched to non-blocking for the transfer (original flags are +/// restored on every exit path): a single blocking write() of a large payload +/// would stay parked in the kernel once the peer stops draining mid-transfer, +/// and the poll deadline could never fire. With O_NONBLOCK, write() returns +/// EAGAIN instead and the idle deadline bounds the whole transfer. +pub(crate) fn write_fd_all(fd: &OwnedFd, data: &[u8], idle: Duration) -> Result<(), String> { + let raw = fd.as_raw_fd(); + let old_flags = unsafe { libc::fcntl(raw, libc::F_GETFL) }; + if old_flags < 0 { + return Err(format!("fcntl F_GETFL: {}", std::io::Error::last_os_error())); + } + if unsafe { libc::fcntl(raw, libc::F_SETFL, old_flags | libc::O_NONBLOCK) } < 0 { + return Err(format!("fcntl F_SETFL: {}", std::io::Error::last_os_error())); + } + let result = write_fd_all_nb(raw, data, idle); + unsafe { libc::fcntl(raw, libc::F_SETFL, old_flags) }; + result +} + +fn write_fd_all_nb(raw: i32, data: &[u8], idle: Duration) -> Result<(), String> { + let mut written = 0; + while written < data.len() { + let mut pfd = libc::pollfd { fd: raw, events: libc::POLLOUT, revents: 0 }; + let n = unsafe { libc::poll(&mut pfd, 1, idle.as_millis().max(1) as libc::c_int) }; + if n < 0 { + let err = std::io::Error::last_os_error(); + if err.raw_os_error() == Some(libc::EINTR) { + continue; + } + return Err(format!("poll: {err}")); + } + if n == 0 { + return Err("clipboard reader stalled".into()); + } + let w = unsafe { + libc::write( + raw, + data[written..].as_ptr() as *const libc::c_void, + data.len() - written, + ) + }; + if w < 0 { + let err = std::io::Error::last_os_error(); + match err.raw_os_error() { + Some(libc::EINTR) | Some(libc::EAGAIN) => continue, + Some(libc::EPIPE) => return Ok(()), + _ => return Err(format!("write: {err}")), + } + } + written += w as usize; + } + Ok(()) +} diff --git a/pixelflux/src/x11/computer_use.rs b/pixelflux/src/x11/computer_use.rs new file mode 100644 index 0000000..6e172bc --- /dev/null +++ b/pixelflux/src/x11/computer_use.rs @@ -0,0 +1,503 @@ +/* + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at https://mozilla.org/MPL/2.0/. + */ + +//! X11 backend for the Computer Use HTTP API: XTEST injection and one-shot root screenshots +//! over a private x11rb connection, so the agent can drive an X session with no active capture +//! and no shared state with any streaming capture thread (the X server itself serializes). +//! +//! Keysyms the active layout cannot produce (after base+AltGr resolution, which stays the +//! preferred path) are typed through a transient remap: under `XGrabServer` the keymap is +//! re-fetched, the needed keysyms are bound onto all-NoSymbol spare keycodes with ONE +//! `XChangeKeyboardMapping`, and the key sequence is injected; after a settle window (which +//! lets the focused client re-fetch the bound map — see `type_with_transient_binds`) one +//! conditional `XChangeKeyboardMapping` under a second grab puts the spares back — two +//! MappingNotify broadcasts per action. This coexists with a selkies session's own +//! spare-keycode overlay allocator on the same server: spares here are chosen DESCENDING +//! from the top of the keycode range while selkies allocates ASCENDING, each grab re-fetches +//! the keymap so every binding selkies already made is respected, the restore clears only +//! keycodes still carrying our content, and selkies' foreign-change invalidation (fired by +//! each MappingNotify) is self-healing because its held keys live on BOUND (non-NoSymbol) +//! keycodes an all-NoSymbol spare can never steal. + +use std::cell::RefCell; +use std::collections::HashMap; +use std::thread; +use std::time::Duration; + +use x11rb::connection::Connection; +use x11rb::protocol::xfixes::ConnectionExt as XfixesExt; +use x11rb::protocol::xproto::{ + ConnectionExt as XprotoExt, ImageFormat, BUTTON_PRESS_EVENT, BUTTON_RELEASE_EVENT, + KEY_PRESS_EVENT, KEY_RELEASE_EVENT, MOTION_NOTIFY_EVENT, +}; +use x11rb::protocol::xtest::ConnectionExt as XtestExt; +use x11rb::rust_connection::RustConnection; + +use crate::computer_use::{encode_png_rgba, CuBackend, CuButton}; + +/// One wheel "click" of scroll per unit of CU `scroll_amount`, capped so a hostile amount +/// cannot flood the server with press/release pairs. +const MAX_SCROLL_CLICKS: i32 = 100; + +const KEYSYM_ISO_LEVEL3_SHIFT: u32 = 0xfe03; +const KEYSYM_MODE_SWITCH: u32 = 0xff7e; + +/// Reverse view of the server's current keymap. +struct ServerKeymap { + /// keysym -> (keycode, shift level); level bit 0 = Shift, bit 1 = AltGr, lowest + /// level wins across ALL keys so a keysym reachable unshifted never resolves to a + /// modified position. + by_sym: HashMap, + /// Keycode carrying `ISO_Level3_Shift` (or, failing that, `Mode_switch`) in the core + /// map — the key synthesized around AltGr-level hits. 0 when the layout has neither, + /// in which case no AltGr levels are offered at all. + altgr_keycode: u32, +} + +pub struct CuX11Backend { + conn: RustConnection, + root: u32, + has_xfixes: bool, + /// Reverse map of the SERVER keymap, built lazily from `GetKeyboardMapping` and kept + /// for this backend's lifetime — one HTTP request (the backend is per-request), so a + /// `setxkbmap` switch is seen by the next request. + reverse_keymap: RefCell>, +} + +impl CuX11Backend { + /// Connect to the X server named by `DISPLAY` and negotiate XTEST (required for any + /// injection). XFixes is optional and only gates cursor compositing in screenshots. + pub fn connect() -> Result { + let (conn, screen_num) = + x11rb::connect(None).map_err(|e| format!("X11 connect failed: {e}"))?; + let root = conn.setup().roots[screen_num].root; + conn.xtest_get_version(2, 2) + .map_err(|e| format!("xtest_get_version: {e}"))? + .reply() + .map_err(|e| format!("XTEST unavailable: {e}"))?; + let has_xfixes = conn + .xfixes_query_version(5, 0) + .ok() + .and_then(|c| c.reply().ok()) + .is_some(); + Ok(Self { conn, root, has_xfixes, reverse_keymap: RefCell::new(None) }) + } + + /// Build the reverse view of the server's current keymap from `GetKeyboardMapping`. + /// + /// Column -> (group, level) convention, as observed on XKB-compat servers (Xvfb/Xorg + /// under `setxkbmap de`: keycode 24 = `q Q q Q at Greek_OMEGA at`): columns 0/1 are + /// group-1 plain/Shift, columns 2/3 mirror them as core group 2, and columns 4/5 + /// carry group-1 levels 3/4 (AltGr, Shift+AltGr). Columns 4/5 are consulted only + /// when the map carries a level-3 modifier key to synthesize; columns 2/3 (group 2) + /// never are. + fn build_reverse_keymap(&self) -> ServerKeymap { + let mut km = ServerKeymap { by_sym: HashMap::new(), altgr_keycode: 0 }; + let setup = self.conn.setup(); + let (lo, hi) = (setup.min_keycode, setup.max_keycode); + let Some(reply) = self + .conn + .get_keyboard_mapping(lo, hi - lo + 1) + .ok() + .and_then(|c| c.reply().ok()) + else { + return km; + }; + let per = reply.keysyms_per_keycode as usize; + if per == 0 { + return km; + } + let keycode_of = |wanted: u32| { + reply + .keysyms + .chunks_exact(per) + .position(|syms| syms.contains(&wanted)) + .map(|i| lo as u32 + i as u32) + }; + km.altgr_keycode = keycode_of(KEYSYM_ISO_LEVEL3_SHIFT) + .or_else(|| keycode_of(KEYSYM_MODE_SWITCH)) + .unwrap_or(0); + // Ascending level order so lower levels win; level bit 0 = Shift, bit 1 = AltGr. + let columns: [(usize, u32); 4] = [(0, 0), (1, 1), (4, 2), (5, 3)]; + for (col, level) in columns { + if col >= per || (level & 2 != 0 && km.altgr_keycode == 0) { + continue; + } + for (i, syms) in reply.keysyms.chunks_exact(per).enumerate() { + let sym = syms[col]; + if sym != 0 { + km.by_sym.entry(sym).or_insert((lo as u32 + i as u32, level)); + } + } + } + km + } + + /// Fire one XTEST fake event and flush so it reaches the server before the action + /// layer's inter-event pacing sleep, matching the immediacy of real input. + fn fake_input(&self, kind: u8, detail: u8, root: u32, x: i16, y: i16) { + let _ = self + .conn + .xtest_fake_input(kind, detail, x11rb::CURRENT_TIME, root, x, y, 0) + .map(|c| c.ignore_error()); + let _ = self.conn.flush(); + } + + fn root_geometry(&self) -> Result<(u16, u16), String> { + let geo = self + .conn + .get_geometry(self.root) + .map_err(|e| format!("get_geometry: {e}"))? + .reply() + .map_err(|e| format!("get_geometry reply: {e}"))?; + Ok((geo.width, geo.height)) + } + + /// One server round trip, forcing everything already sent on this connection to be + /// processed before the next request is issued (the XSync idiom). + fn sync(&self) -> Result<(), String> { + self.conn + .get_input_focus() + .map_err(|e| format!("sync: {e}"))? + .reply() + .map_err(|e| format!("sync reply: {e}"))?; + Ok(()) + } + + /// Bind `keysyms` onto spare keycodes and run `seq` with the keysym -> keycode map, + /// choose/bind/inject atomically under a server grab; then, after a settle window, + /// restore the spares under a second grab. The settle exists because clients + /// translate a keycode by RE-FETCHING the keymap when they process the bind's + /// MappingNotify — a fetch the grab itself blocks — so a restore issued inside the + /// first grab would be what they read back and every transient key would translate + /// to nothing (observed with xterm). Once `seq` has run, every failure is logged and + /// swallowed so the caller never re-runs the sequence. + fn type_with_transient_binds( + &self, + keysyms: &[u32], + seq: &mut dyn FnMut(&HashMap), + ) -> Result<(), String> { + let setup = self.conn.setup(); + let (lo, hi) = (setup.min_keycode, setup.max_keycode); + let mut chosen: Vec = Vec::with_capacity(keysyms.len()); + let (span_lo, count, per, bound_syms) = { + self.conn + .grab_server() + .map_err(|e| format!("grab_server: {e}"))?; + let _guard = ServerGrabGuard { conn: &self.conn }; + // Re-fetched UNDER the grab: a spare keycode any other client bound before + // the grab is visible as bound here and never chosen. + let reply = self + .conn + .get_keyboard_mapping(lo, hi - lo + 1) + .map_err(|e| format!("get_keyboard_mapping: {e}"))? + .reply() + .map_err(|e| format!("get_keyboard_mapping reply: {e}"))?; + let per = reply.keysyms_per_keycode as usize; + if per == 0 { + return Err("empty keymap".to_string()); + } + // Spares are all-NoSymbol keycodes taken DESCENDING from the top of the + // range. selkies' overlay allocator scans ASCENDING, so the two only meet + // when nearly every spare on the server is taken; and any held key + // (selkies' overlay binds included) lives on a BOUND, non-NoSymbol keycode, + // so an all-NoSymbol spare can never steal a key that is currently down. + for (i, syms) in reply.keysyms.chunks_exact(per).enumerate().rev() { + if syms.iter().all(|&s| s == 0) { + chosen.push(lo as u32 + i as u32); + if chosen.len() == keysyms.len() { + break; + } + } + } + if chosen.len() < keysyms.len() { + return Err(format!( + "only {} spare keycodes for {} unresolved keysyms", + chosen.len(), + keysyms.len() + )); + } + // ONE ChangeKeyboardMapping over the span from the lowest chosen spare + // upward. Every all-NoSymbol keycode above the lowest chosen one was itself + // chosen, so the span's other keycodes are bound ones, rewritten with their + // existing content (a content no-op). Each transient key carries its keysym + // at the plain and Shift levels so a stray held Shift cannot change what it + // types. + let span_lo = *chosen.last().unwrap(); + let span_hi = chosen[0]; + let base = ((span_lo - lo as u32) as usize) * per; + let end = ((span_hi - lo as u32) as usize + 1) * per; + let mut bound_syms = reply.keysyms[base..end].to_vec(); + let mut map = HashMap::new(); + for (&sym, &kc) in keysyms.iter().zip(chosen.iter()) { + let off = ((kc - span_lo) as usize) * per; + bound_syms[off] = sym; + if per > 1 { + bound_syms[off + 1] = sym; + } + map.insert(sym, kc); + } + let count = (span_hi - span_lo + 1) as u8; + self.conn + .change_keyboard_mapping(count, span_lo as u8, per as u8, &bound_syms) + .map_err(|e| format!("change_keyboard_mapping: {e}"))? + .check() + .map_err(|e| format!("change_keyboard_mapping check: {e}"))?; + // The binding must be live server-side before the first fake press resolves + // against it. + self.sync()?; + seq(&map); + (span_lo, count, per, bound_syms) + // Guard drops: ungrab + flush, releasing clients to process the injected + // events against the still-live bindings. + }; + // Settle: clients consume the queued MappingNotify + key events and re-fetch the + // BOUND map before the spares disappear again. + thread::sleep(TRANSIENT_BIND_SETTLE); + if let Err(e) = self.restore_transient_binds(span_lo, count, per, &bound_syms, &chosen) { + eprintln!("[ComputerUse] transient keysym restore failed: {e}"); + } + Ok(()) + } + + /// Return the transiently bound spares to all-NoSymbol with ONE conditional + /// `ChangeKeyboardMapping` under its own grab. The span is re-fetched under that + /// grab and only spares still carrying OUR content are cleared; a keycode another + /// client (selkies' allocator) claimed during the settle window keeps that client's + /// content — this restore can never clobber a foreign binding. + fn restore_transient_binds( + &self, + span_lo: u32, + count: u8, + per: usize, + bound_syms: &[u32], + chosen: &[u32], + ) -> Result<(), String> { + self.conn + .grab_server() + .map_err(|e| format!("grab_server: {e}"))?; + let _guard = ServerGrabGuard { conn: &self.conn }; + let reply = self + .conn + .get_keyboard_mapping(span_lo as u8, count) + .map_err(|e| format!("get_keyboard_mapping: {e}"))? + .reply() + .map_err(|e| format!("get_keyboard_mapping reply: {e}"))?; + let cur_per = reply.keysyms_per_keycode as usize; + if cur_per == 0 || per == 0 { + return Err("empty keymap".to_string()); + } + let mut restore = reply.keysyms.clone(); + let mut changed = false; + for &kc in chosen { + let sym = bound_syms[((kc - span_lo) as usize) * per]; + let cur = &mut restore[((kc - span_lo) as usize) * cur_per..][..cur_per]; + // Still ours when every populated level carries OUR keysym: the server's XKB + // integration mirrors a core single-group binding into the group-2 columns, + // so the refetch shows `sym` at more levels than the bind wrote. + let still_ours = + cur.contains(&sym) && cur.iter().all(|&s| s == 0 || s == sym); + if still_ours { + cur.fill(0); + changed = true; + } + } + if changed { + self.conn + .change_keyboard_mapping(count, span_lo as u8, cur_per as u8, &restore) + .map_err(|e| format!("change_keyboard_mapping: {e}"))? + .check() + .map_err(|e| format!("change_keyboard_mapping check: {e}"))?; + } + Ok(()) + } +} + +/// How long transient binds outlive the injected key events before being restored: the +/// focused client has to wake up, see the bind's MappingNotify, and re-fetch the keymap +/// while the bindings are still live, or the presses translate against the restored map +/// and type nothing. +const TRANSIENT_BIND_SETTLE: Duration = Duration::from_millis(50); + +/// RAII server grab release: the grab is dropped (and the request flushed) on every exit +/// path, early error returns and panics included — a leaked server grab freezes every +/// client on the display. +struct ServerGrabGuard<'a> { + conn: &'a RustConnection, +} + +impl Drop for ServerGrabGuard<'_> { + fn drop(&mut self) { + let _ = self.conn.ungrab_server().map(|c| c.ignore_error()); + let _ = self.conn.flush(); + } +} + +impl Drop for CuX11Backend { + /// Closing the connection can race the server's client teardown against still-buffered + /// fake-input requests (observed as lost button releases); one round trip forces the + /// server to consume everything sent on this connection before it goes away. + fn drop(&mut self) { + if let Ok(cookie) = self.conn.get_input_focus() { + let _ = cookie.reply(); + } + } +} + +impl CuBackend for CuX11Backend { + fn name(&self) -> &'static str { + "x11" + } + + fn fb_size(&self) -> Result<(i32, i32), String> { + let (w, h) = self.root_geometry()?; + Ok((w as i32, h as i32)) + } + + fn key(&self, scancode: u32, pressed: bool) { + if scancode > u8::MAX as u32 { + return; + } + let kind = if pressed { KEY_PRESS_EVENT } else { KEY_RELEASE_EVENT }; + self.fake_input(kind, scancode as u8, x11rb::NONE, 0, 0); + } + + fn mouse_move(&self, x: f64, y: f64) { + // detail = 0 makes the motion absolute in root coordinates. + self.fake_input( + MOTION_NOTIFY_EVENT, + 0, + self.root, + x.round() as i16, + y.round() as i16, + ); + } + + fn button(&self, btn: CuButton, pressed: bool) { + let detail = match btn { + CuButton::Left => 1, + CuButton::Middle => 2, + CuButton::Right => 3, + }; + let kind = if pressed { BUTTON_PRESS_EVENT } else { BUTTON_RELEASE_EVENT }; + self.fake_input(kind, detail, x11rb::NONE, 0, 0); + } + + fn scroll(&self, dx: f64, dy: f64) { + // X has no smooth axis over XTEST: a scroll is N discrete clicks of the wheel + // buttons (4 = up, 5 = down, 6 = left, 7 = right), one press/release pair each. + let emit = |button: u8, clicks: i32| { + for _ in 0..clicks.min(MAX_SCROLL_CLICKS) { + self.fake_input(BUTTON_PRESS_EVENT, button, x11rb::NONE, 0, 0); + self.fake_input(BUTTON_RELEASE_EVENT, button, x11rb::NONE, 0, 0); + } + }; + let vy = dy.round() as i32; + let vx = dx.round() as i32; + if vy != 0 { + emit(if vy < 0 { 4 } else { 5 }, vy.abs()); + } + if vx != 0 { + emit(if vx < 0 { 6 } else { 7 }, vx.abs()); + } + } + + fn screenshot_png(&self, display: u32) -> Result, String> { + // One X server, one root: only display 0 exists on this backend. + if display != 0 { + return Err(format!("Unknown display: {display}")); + } + let (w, h) = self.root_geometry()?; + let img = self + .conn + .get_image(ImageFormat::Z_PIXMAP, self.root, 0, 0, w, h, !0u32) + .map_err(|e| format!("get_image: {e}"))? + .reply() + .map_err(|e| format!("get_image reply: {e}"))?; + let mut data = img.data; + let expected = w as usize * h as usize * 4; + if data.len() != expected { + return Err(format!( + "unexpected image size {} for {}x{} (only 32-bpp roots are supported)", + data.len(), w, h + )); + } + // The agent needs to see the pointer; the stream's cursor settings do not apply here. + if self.has_xfixes { + if let Some(c) = self + .conn + .xfixes_get_cursor_image() + .ok() + .and_then(|c| c.reply().ok()) + { + if c.width > 0 && c.height > 0 { + let (img_x, img_y) = + super::cursor_image_origin(c.x, c.y, c.xhot, c.yhot, 0, 0); + super::overlay_cursor( + &mut data, + w as usize * 4, + w as i32, + h as i32, + c.width as i32, + c.height as i32, + &c.cursor_image, + img_x, + img_y, + ); + } + } + } + // The grab is BGRX; the padding byte is undefined for depth-24 roots, so alpha is + // forced opaque or the PNG would come out transparent. + for px in data.chunks_exact_mut(4) { + px.swap(0, 2); + px[3] = 0xFF; + } + encode_png_rgba(&data, w as u32, h as u32) + } + + fn cursor_pos(&self) -> Result<(f64, f64), String> { + let ptr = self + .conn + .query_pointer(self.root) + .map_err(|e| format!("query_pointer: {e}"))? + .reply() + .map_err(|e| format!("query_pointer reply: {e}"))?; + Ok((ptr.root_x as f64, ptr.root_y as f64)) + } + + fn resolve_keysyms(&self, keysyms: &[u32]) -> Vec<(u32, u32)> { + let mut cached = self.reverse_keymap.borrow_mut(); + let km = cached.get_or_insert_with(|| self.build_reverse_keymap()); + keysyms + .iter() + .map(|sym| km.by_sym.get(sym).copied().unwrap_or((0, 0))) + .collect() + } + + fn altgr_keycode(&self) -> u32 { + let mut cached = self.reverse_keymap.borrow_mut(); + cached.get_or_insert_with(|| self.build_reverse_keymap()).altgr_keycode + } + + fn with_transient_keysyms(&self, keysyms: &[u32], seq: &mut dyn FnMut(&HashMap)) { + // Dedup defensively; a duplicate would burn a spare keycode for nothing. + let mut unique: Vec = Vec::with_capacity(keysyms.len()); + for &s in keysyms { + if s != 0 && !unique.contains(&s) { + unique.push(s); + } + } + if unique.is_empty() { + seq(&HashMap::new()); + return; + } + if let Err(e) = self.type_with_transient_binds(&unique, seq) { + eprintln!("[ComputerUse] transient keysym bind failed ({e}); typing without it"); + seq(&HashMap::new()); + } + } +} diff --git a/pixelflux/src/x11/cursor.rs b/pixelflux/src/x11/cursor.rs index 394cbe0..5f929a0 100644 --- a/pixelflux/src/x11/cursor.rs +++ b/pixelflux/src/x11/cursor.rs @@ -210,9 +210,18 @@ fn monitor_thread(stop: Arc, wake_win: Arc) { } let mut last: Option = fetch_payload(&conn); deliver(last.as_ref()); - // A registration that raced setup may have requested a replay before the wake window - // existed; the current cursor was just delivered, so the request is satisfied. - REPLAY.store(false, Ordering::Release); + // A registration that raced setup may have requested a replay before the wake + // window existed (its wake was skipped). The deliver above satisfied it only if + // the callback was already visible and the fetch produced a payload, so consume + // the flag and re-deliver rather than assume: an unconditional clear leaves that + // callback cursor-less until the next real cursor change. From here on the wake + // window exists, so later requests always reach the loop below. + if REPLAY.swap(false, Ordering::AcqRel) { + if last.is_none() { + last = fetch_payload(&conn); + } + deliver(last.as_ref()); + } loop { let event = match conn.wait_for_event() { Ok(ev) => ev, @@ -315,7 +324,9 @@ fn deliver(payload: Option<&Payload>) { }; if let Some(cb) = cb { let py_bytes = PyBytes::new(py, png); - let _ = cb.call1(py, (*msg_type, py_bytes, *hot_x, *hot_y)); + if let Err(e) = cb.call1(py, (*msg_type, py_bytes, *hot_x, *hot_y)) { + e.print(py); + } } }); } @@ -369,6 +380,12 @@ fn cursor_to_png(img: &GetCursorImageReply, cap: i32) -> (&'static str, Vec, hot_y = (hot_y as f32 * scale) as i32; } crate::unpremultiply_rgba(&mut image); + // A hotspot can lie outside the visible bbox (its neighborhood was cropped as + // fully transparent). Consumers treat the hotspot as an offset INTO the + // bitmap, so clamp to the cropped bounds instead of emitting off-image + // coordinates. + let hot_x = hot_x.clamp(0, image.width() as i32 - 1); + let hot_y = hot_y.clamp(0, image.height() as i32 - 1); let mut png = Vec::new(); match image.write_to(&mut std::io::Cursor::new(&mut png), image::ImageFormat::Png) { Ok(()) => ("png", png, hot_x, hot_y), @@ -406,16 +423,36 @@ mod tests { assert_eq!(t, "hide"); } - /// The image is cropped to its visible bbox and the hotspot re-based to the crop: one - /// opaque pixel at (2,1) with hotspot (3,3) yields a 1x1 PNG with hotspot (1,2). + /// The image is cropped to its visible bbox and the hotspot re-based to the crop: + /// a 2x2 visible block at (1,1)..(2,2) with hotspot (2,2) yields a 2x2 PNG with + /// hotspot (1,1). #[test] fn crop_rebases_hotspot() { let mut px = vec![0u32; 16]; - px[1 * 4 + 2] = 0xFF00_0000; - let (t, data, hx, hy) = cursor_to_png(&reply(4, 4, 3, 3, px), 32); + for (x, y) in [(1, 1), (2, 1), (1, 2), (2, 2)] { + px[y * 4 + x] = 0xFF00_0000; + } + let (t, data, hx, hy) = cursor_to_png(&reply(4, 4, 2, 2, px), 32); assert_eq!(t, "png"); assert!(!data.is_empty()); - assert_eq!((hx, hy), (1, 2)); + assert_eq!((hx, hy), (1, 1)); + } + + /// A hotspot whose neighborhood was cropped away as transparent is clamped into + /// the emitted bitmap — consumers use it as an offset INTO the image, and the + /// Wayland path never emits out-of-bounds hotspots either. + #[test] + fn out_of_bbox_hotspot_clamped() { + let mut px = vec![0u32; 16]; + let (x, y) = (2usize, 1usize); + px[y * 4 + x] = 0xFF00_0000; + // Visible pixel at (2,1) only. Hotspot (0,0): rebased (-2,-1) -> (0,0). + let (t, _, hx, hy) = cursor_to_png(&reply(4, 4, 0, 0, px.clone()), 32); + assert_eq!(t, "png"); + assert_eq!((hx, hy), (0, 0)); + // Hotspot (3,3): rebased (1,2) past the 1x1 crop -> clamps to (0,0). + let (_, _, hx, hy) = cursor_to_png(&reply(4, 4, 3, 3, px), 32); + assert_eq!((hx, hy), (0, 0)); } /// Premultiplied color becomes straight alpha in the PNG: a half-alpha pixel stored diff --git a/pixelflux/src/x11/mod.rs b/pixelflux/src/x11/mod.rs index a664bbe..4a9a486 100644 --- a/pixelflux/src/x11/mod.rs +++ b/pixelflux/src/x11/mod.rs @@ -33,12 +33,13 @@ use x11rb::protocol::xfixes::ConnectionExt as XfixesExt; use x11rb::protocol::xproto::{ConnectionExt as XprotoExt, ImageFormat}; use x11rb::rust_connection::RustConnection; -use crate::encoders::overlay::WatermarkLocation; +use crate::encoders::overlay::blend_pixel_premultiplied; use crate::encoders::software::EncodedStripe; use crate::pipeline::X11Pipeline; use crate::recording_sink::RecordingSink; use crate::RustCaptureSettings; +pub mod computer_use; pub mod cursor; /// Cross-thread controls for a running capture: a bag of atomics (plus two mutex-guarded @@ -194,20 +195,48 @@ impl ShmSurface { } } -/// Resolve the capture dimensions `shm_get_image` will read, bounded so the region can never -/// run past the live root from the capture offset. +/// Reject a server whose root visual is not 32 bits per pixel. /// -/// The bound is the reason this function exists: a grab that ran off the root edge would fail the -/// request outright, so the size is always clamped to what is actually there. With auto-adjust (or -/// an unset width/height `<= 0`) the capture tracks the full root minus the capture offset; -/// otherwise the requested size is clamped to what is available from that offset. `capture_x` / -/// `capture_y` are floored at `>= 0` exactly as `shm_get_image` consumes them, and saturating -/// subtraction plus a final `u16` clamp (minimum 2) keep pathological settings from overflowing or -/// collapsing to an unusable surface. H.264 (`output_mode == 1`) requires even dimensions, so both -/// are then rounded down to a multiple of two. +/// Every `ShmSurface` is sized at `width * 4` bytes per row and the capture is read as packed +/// BGRA, so a root of any other depth would be grabbed as garbage rather than failing. Checked +/// on the initial connection and again on every reconnect, since a restarted server can come +/// back with a different visual. +fn require_32bpp(conn: &RustConnection, screen: usize) -> Result<(), String> { + let root_depth = conn.setup().roots[screen].root_depth; + let bpp = conn + .setup() + .pixmap_formats + .iter() + .find(|f| f.depth == root_depth) + .map(|f| f.bits_per_pixel) + .unwrap_or(32); + if bpp != 32 { + return Err(format!( + "unsupported root depth {root_depth} ({bpp} bpp); only 32-bpp BGRA is supported" + )); + } + Ok(()) +} + +/// Clamp a capture offset to the live root so a grab always has at least one root pixel in +/// range: a past-end offset fails every `shm_get_image` with BadMatch and would burn out the +/// failure budget with no chance of recovery. The upper clamp also keeps protocol-range roots +/// (INT16 coordinates) from truncating a wider value through the `as i16` cast. +fn clamp_offset(x: i32, root: u16) -> i16 { + let upper = (root as i32).saturating_sub(1).min(i16::MAX as i32 - 1); + x.max(0).min(upper) as i16 +} + +/// Resolve the capture dimensions `shm_get_image` will read, bounded so the region can never +/// run past the live root from the (clamped) capture offset: with auto-adjust (or unset +/// width/height `<= 0`) the capture tracks the full root minus the offset, otherwise the +/// requested size is clamped to what is available from it. Saturating subtraction plus a final +/// `u16` clamp (minimum 2) keep pathological settings from overflowing or collapsing to an +/// unusable surface. H.264 (`output_mode == 1`) requires even dimensions, so both are then +/// rounded down to a multiple of two. fn resolve_dims(root_w: u16, root_h: u16, s: &RustCaptureSettings) -> (u16, u16) { - let cap_x = s.capture_x.max(0); - let cap_y = s.capture_y.max(0); + let cap_x = clamp_offset(s.capture_x, root_w) as i32; + let cap_y = clamp_offset(s.capture_y, root_h) as i32; let avail_w = (root_w as i32).saturating_sub(cap_x).max(2); let avail_h = (root_h as i32).saturating_sub(cap_y).max(2); let mut w = if s.auto_adjust_screen_capture_size || s.width <= 0 { @@ -229,6 +258,80 @@ fn resolve_dims(root_w: u16, root_h: u16, s: &RustCaptureSettings) -> (u16, u16) (w as u16, h as u16) } +/// Rebuild the X channel + capture surfaces after a connection-level failure (Xorg restart, +/// VT switch), refreshing the offset/dims state against the live root. The pool is drained +/// first so the encode thread can never be reading a surface as it is destroyed, and its +/// generation is bumped so the encoder rebuilds against the new layout. Every fallible step +/// completes before the swap, and the old channel/surfaces are destroyed best-effort after +/// it (their server is likely gone; the local detaches still run, and the kernel frees +/// each old segment once nothing maps it). +#[allow(clippy::too_many_arguments)] +fn try_rebuild_channel( + conn: &mut RustConnection, + root: &mut u32, + rsettings: &RustCaptureSettings, + cap_x: &mut i16, + cap_y: &mut i16, + cap_w: &mut u16, + cap_h: &mut u16, + root_w: &mut u16, + root_h: &mut u16, + pool_n: usize, + pool: &FramePool, + surfaces: &mut Vec, + stop: &AtomicBool, +) -> Result<(), String> { + let (new_conn, new_screen) = + x11rb::connect(None).map_err(|e| format!("X11 reconnect failed: {e}"))?; + require_32bpp(&new_conn, new_screen)?; + new_conn + .shm_query_version() + .map_err(|e| format!("shm_query_version: {e}"))? + .reply() + .map_err(|e| format!("XShm unavailable on reconnect: {e}"))?; + new_conn + .xfixes_query_version(5, 0) + .map_err(|e| format!("xfixes_query_version: {e}"))? + .reply() + .map_err(|e| format!("XFixes unavailable on reconnect: {e}"))?; + let new_root = new_conn.setup().roots[new_screen].root; + let geo = new_conn + .get_geometry(new_root) + .map_err(|e| format!("get_geometry: {e}"))? + .reply() + .map_err(|e| format!("get_geometry reply on reconnect: {e}"))?; + if !pool.drain_for_resize(pool_n, stop) { + return Err("pool drain interrupted by stop during reconnect".to_string()); + } + let (fw, fh) = resolve_dims(geo.width, geo.height, rsettings); + let mut fresh: Vec = Vec::with_capacity(pool_n); + for _ in 0..pool_n { + match ShmSurface::create(&new_conn, fw, fh) { + Ok(s) => fresh.push(s), + Err(e) => { + for s in fresh.iter_mut() { + s.destroy(&new_conn); + } + return Err(e); + } + } + } + for s in surfaces.iter_mut() { + s.destroy(conn); + } + *conn = new_conn; + *root = new_root; + *cap_x = clamp_offset(rsettings.capture_x, geo.width); + *cap_y = clamp_offset(rsettings.capture_y, geo.height); + *cap_w = fw; + *cap_h = fh; + *root_w = geo.width; + *root_h = geo.height; + *surfaces = fresh; + pool.bump_generation(); + Ok(()) +} + /// Grab one frame of the capture region into `surface` with a single XShm round-trip. /// /// One `shm_get_image` request is atomic — the server never interleaves another client @@ -261,28 +364,6 @@ fn grab_frame( Ok(()) } -/// Alpha-blend a source pixel (pre-split into r,g,b,a) over a BGRA destination pixel. -/// -/// Cursor and watermark pixels are overwhelmingly either fully opaque or fully transparent, and this -/// runs per pixel per frame on the CPU, so the two extremes are special-cased to skip the blend -/// arithmetic entirely: an opaque source (`a == 255`) simply overwrites, a fully transparent source -/// (`a == 0`) is left as-is, and only genuine edge pixels pay for the integer source-over. In every -/// case only the B / G / R bytes are written; the destination's alpha byte is left as the capture -/// delivered it. -#[inline] -fn blend_pixel(dst: &mut [u8], r: u8, g: u8, b: u8, a: u8) { - if a == 255 { - dst[0] = b; - dst[1] = g; - dst[2] = r; - } else if a > 0 { - let ia = 255 - a as u32; - dst[0] = ((b as u32 * a as u32 + dst[0] as u32 * ia) / 255) as u8; - dst[1] = ((g as u32 * a as u32 + dst[1] as u32 * ia) / 255) as u8; - dst[2] = ((r as u32 * a as u32 + dst[2] as u32 * ia) / 255) as u8; - } -} - /// Frame-space top-left of the cursor image, given the XFixes hotspot position. /// /// XFixes reports the cursor position at its HOTSPOT; X draws the image with its top-left at @@ -290,15 +371,16 @@ fn blend_pixel(dst: &mut [u8], r: u8, g: u8, b: u8, a: u8) { /// The result may go negative near the frame edges, which `overlay_cursor` clips per pixel to match /// the server's own edge clipping. #[inline] -fn cursor_image_origin(x: i16, y: i16, xhot: u16, yhot: u16, cap_x: i32, cap_y: i32) -> (i32, i32) { +pub(crate) fn cursor_image_origin(x: i16, y: i16, xhot: u16, yhot: u16, cap_x: i32, cap_y: i32) -> (i32, i32) { (x as i32 - xhot as i32 - cap_x, y as i32 - yhot as i32 - cap_y) } -/// Composite the XFixes cursor (ARGB `u32` per pixel) onto the BGRA frame with its top-left -/// at `(img_x, img_y)`, blending each pixel through `blend_pixel` with per-pixel bounds clipping so -/// an image straddling a frame edge writes only its in-frame portion. +/// Composite the XFixes cursor (ARGB `u32` per pixel, premultiplied by the XFixes +/// format definition) onto the BGRA frame with its top-left at `(img_x, img_y)`, +/// blending each pixel through `blend_pixel_premultiplied` with per-pixel bounds +/// clipping so an image straddling a frame edge writes only its in-frame portion. #[allow(clippy::too_many_arguments)] -fn overlay_cursor( +pub(crate) fn overlay_cursor( frame: &mut [u8], stride: usize, frame_w: i32, @@ -325,148 +407,7 @@ fn overlay_cursor( let g = ((px >> 8) & 0xFF) as u8; let b = (px & 0xFF) as u8; let off = ty as usize * stride + tx as usize * 4; - blend_pixel(&mut frame[off..off + 4], r, g, b, a); - } - } -} - -/// CPU watermark: the raw RGBA pixels (row-major, `w*h*4`) in host memory plus the -/// placement / animation state, blended directly into the captured BGRA frame. -/// -/// The blend is on the CPU because the frame is already sitting in host shm memory before it reaches -/// any encoder, so stamping the overlay there is both the cheapest place to do it and the one place -/// it applies identically regardless of which encoder (hardware or software) runs downstream. The -/// pixels are kept as RGBA straight from `image`'s decode and converted per pixel as they are blended. -struct X11Watermark { - pixels: Vec, - w: i32, - h: i32, - pos_x: i32, - pos_y: i32, - sub_x: f64, - sub_y: f64, - vel_x: f64, - vel_y: f64, - loaded: bool, -} - -impl X11Watermark { - /// Load the watermark image at `path` into host RGBA, or return an unloaded stub. - /// - /// An empty path or any decode failure yields `loaded == false`, which every method treats as a - /// no-op, so a missing or broken watermark simply disables the overlay. The bouncing-animation - /// velocities are seeded here. - fn load(path: &str) -> Self { - let mut wm = Self { - pixels: Vec::new(), - w: 0, - h: 0, - pos_x: 0, - pos_y: 0, - sub_x: 0.0, - sub_y: 0.0, - vel_x: 2.0, - vel_y: 2.0, - loaded: false, - }; - if path.is_empty() { - return wm; - } - if let Ok(img) = image::open(std::path::Path::new(path)) { - let rgba = img.to_rgba8(); - wm.w = rgba.width() as i32; - wm.h = rgba.height() as i32; - wm.pixels = rgba.into_vec(); - wm.loaded = wm.w > 0 && wm.h > 0; - } - wm - } - - /// Set the watermark's top-left placement for this frame from the location enum; a stub - /// (unloaded) watermark returns immediately. - /// - /// The fixed corners and center are direct arithmetic. The animated mode advances a bouncing - /// position and reflects velocity off each frame edge, accumulating in the fractional `sub_x` / - /// `sub_y` rather than the integer `pos_x` / `pos_y`: a sub-pixel-per-frame velocity has to - /// survive between frames or the motion would either stall or snap by whole pixels, so the float - /// carries the remainder and `pos_x` / `pos_y` take its floor for the actual blit. - fn update_position(&mut self, frame_w: i32, frame_h: i32, loc_enum: i32) { - if !self.loaded { - return; - } - match WatermarkLocation::from(loc_enum) { - WatermarkLocation::TL => { - self.pos_x = 0; - self.pos_y = 0; - } - WatermarkLocation::TR => { - self.pos_x = frame_w - self.w; - self.pos_y = 0; - } - WatermarkLocation::BL => { - self.pos_x = 0; - self.pos_y = frame_h - self.h; - } - WatermarkLocation::BR => { - self.pos_x = frame_w - self.w; - self.pos_y = frame_h - self.h; - } - WatermarkLocation::MI => { - self.pos_x = (frame_w - self.w) / 2; - self.pos_y = (frame_h - self.h) / 2; - } - WatermarkLocation::AN => { - self.sub_x += self.vel_x; - self.sub_y += self.vel_y; - if self.sub_x <= 0.0 { - self.sub_x = 0.0; - self.vel_x = self.vel_x.abs(); - } else if self.sub_x + self.w as f64 >= frame_w as f64 { - self.sub_x = (frame_w - self.w) as f64; - self.vel_x = -self.vel_x.abs(); - } - if self.sub_y <= 0.0 { - self.sub_y = 0.0; - self.vel_y = self.vel_y.abs(); - } else if self.sub_y + self.h as f64 >= frame_h as f64 { - self.sub_y = (frame_h - self.h) as f64; - self.vel_y = -self.vel_y.abs(); - } - self.pos_x = self.sub_x as i32; - self.pos_y = self.sub_y as i32; - } - WatermarkLocation::None => (), - } - } - - /// Alpha-blend the loaded watermark into the BGRA frame at its current position; a no-op - /// when unloaded. Clips per pixel at the frame bounds because the animated position — or a - /// watermark larger than the capture — can leave part of the image off-frame, and only the - /// in-frame portion may be written. - fn blend_into(&self, frame: &mut [u8], stride: usize, frame_w: i32, frame_h: i32) { - if !self.loaded { - return; - } - for y in 0..self.h { - let ty = self.pos_y + y; - if ty < 0 || ty >= frame_h { - continue; - } - for x in 0..self.w { - let tx = self.pos_x + x; - if tx < 0 || tx >= frame_w { - continue; - } - let src = ((y * self.w + x) * 4) as usize; - let (r, g, b, a) = ( - self.pixels[src], - self.pixels[src + 1], - self.pixels[src + 2], - self.pixels[src + 3], - ); - let off = ty as usize * stride + tx as usize * 4; - blend_pixel(&mut frame[off..off + 4], r, g, b, a); - } + blend_pixel_premultiplied(&mut frame[off..off + 4], r, g, b, a); } } } @@ -719,7 +660,7 @@ where .is_some_and(|pl| pl.reshape(&psettings, size_changed)); if !reshaped { drop(pipeline.take()); - pipeline = Some(X11Pipeline::new(psettings.clone(), recording_sink.clone())); + pipeline = Some(X11Pipeline::new(psettings.clone())); if let Some(pl) = &pipeline { let enc_name = pl.encoder_name(); let mut log_msg = format!( @@ -729,7 +670,14 @@ where if psettings.output_mode == 0 { log_msg.push_str(&format!(" | Mode: JPEG | Quality: {}", psettings.jpeg_quality)); } else { - log_msg.push_str(&format!(" | Mode: H264 | CRF: {}", psettings.video_crf)); + if psettings.video_cbr_mode { + log_msg.push_str(&format!(" | Mode: H264 CBR {}", psettings.video_bitrate_kbps)); + } else { + log_msg.push_str(&format!(" | Mode: H264 | CRF: {}", psettings.video_crf)); + if psettings.video_bitrate_kbps > 0 { + log_msg.push_str(&format!(" | VBV: {} kbps", psettings.video_bitrate_kbps)); + } + } if psettings.video_fullcolor { log_msg.push_str(" | Colorspace: I444 (Full Range)"); } else { @@ -749,7 +697,13 @@ where } let pl = pipeline.as_mut().unwrap(); - if controls.force_idr.swap(false, Ordering::Relaxed) { + // A recorder connecting to the socket sink needs a fresh decode entry point; it is + // folded into the same standard request-IDR path as a client-driven force_idr. + let sink_idr = recording_sink + .as_ref() + .map(|s| s.should_force_idr()) + .unwrap_or(false); + if controls.force_idr.swap(false, Ordering::Relaxed) || sink_idr { pl.request_idr(); } if controls.rate_dirty.swap(false, Ordering::Acquire) { @@ -771,13 +725,9 @@ where if !stripes.is_empty() { frame_count += 1; stripe_count += stripes.len() as u64; + // IDR arming is consumed inside X11Pipeline::process; this tap only fans out. if let Some(ref socket) = recording_sink { - for stripe in &stripes { - let _ = socket.write_encoded_frame(stripe); - } - if socket.should_force_idr() { - controls.force_idr.store(true, Ordering::Relaxed); - } + socket.write_frame(&stripes, psettings.height); } on_frame(stripes); } @@ -849,23 +799,10 @@ pub fn run_capture( where F: FnMut(Vec) + Send + 'static, { - let (conn, screen_num) = + let (mut conn, screen_num) = x11rb::connect(None).map_err(|e| format!("X11 connect failed: {e}"))?; - let root = conn.setup().roots[screen_num].root; - let root_depth = conn.setup().roots[screen_num].root_depth; - - let bpp = conn - .setup() - .pixmap_formats - .iter() - .find(|f| f.depth == root_depth) - .map(|f| f.bits_per_pixel) - .unwrap_or(32); - if bpp != 32 { - return Err(format!( - "unsupported root depth {root_depth} ({bpp} bpp); only 32-bpp BGRA is supported" - )); - } + let mut root = conn.setup().roots[screen_num].root; + require_32bpp(&conn, screen_num)?; conn.shm_query_version() .map_err(|e| format!("shm_query_version: {e}"))? @@ -884,26 +821,53 @@ where let mut rsettings = settings.clone(); let (mut cap_w, mut cap_h) = resolve_dims(geo.width, geo.height, &rsettings); - let mut cap_x = rsettings.capture_x.max(0) as i16; - let mut cap_y = rsettings.capture_y.max(0) as i16; + let mut cap_x = clamp_offset(rsettings.capture_x, geo.width); + let mut cap_y = clamp_offset(rsettings.capture_y, geo.height); + // Last root size seen. A live pan has to clamp against something even when the geometry + // round-trip fails, or the requested origin would be dropped instead of applied. + let (mut root_w, mut root_h) = (geo.width, geo.height); const POOL_N: usize = 3; let mut surfaces: Vec = Vec::with_capacity(POOL_N); for _ in 0..POOL_N { surfaces.push(ShmSurface::create(&conn, cap_w, cap_h)?); } + // Consecutive channel rebuilds without one good grab in between: geometry + // errors can burn out the grab budget repeatedly under a dead region, so + // recovery only gets a bounded number of channel rebuilds. + let mut reconnect_streak = 0u32; let pool = Arc::new(FramePool::new(POOL_N)); - let mut watermark = X11Watermark::load(&settings.watermark_path); + let mut watermark = crate::encoders::overlay::OverlayState::default(); + if !settings.watermark_path.is_empty() { + watermark.load_watermark(&settings.watermark_path, 1.0); + } let enc_pool = pool.clone(); let enc_controls = controls.clone(); let enc_settings = settings.clone(); + let encode_panicked = Arc::new(AtomicBool::new(false)); + let guard_panicked = encode_panicked.clone(); let encode_thread = thread::spawn(move || { crate::boost_thread_priority(-10); let _ = encode_tid_tx.send(thread::current().id()); let mut on_frame = on_frame; - encode_loop(&enc_pool, &enc_controls, &enc_settings, &mut on_frame); + // An encoder panic must never wedge the capture: without the pool shutdown + // the capture thread would block in publish forever while is_capturing still + // reports true on this thread's handle. Shutting the pool unblocks it through + // the same path a clean stop takes, so the panic is also recorded — otherwise + // the capture would return Ok and Python could not tell the two apart. + let guard_pool = enc_pool.clone(); + let guard_controls = enc_controls.clone(); + let result = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| { + encode_loop(&enc_pool, &enc_controls, &enc_settings, &mut on_frame); + })); + if result.is_err() { + eprintln!("[pixelflux x11] encode thread panicked; shutting the pool to fail the capture"); + guard_panicked.store(true, Ordering::Release); + guard_pool.shutdown(); + guard_controls.stop.store(true, Ordering::Relaxed); + } }); let mut next_frame = Instant::now(); @@ -941,9 +905,13 @@ where // settings) would glue resolve_dims back to the ROOT size, undoing the // re-target on the next geometry poll. w/h <= 0 keep following the root. rsettings.auto_adjust_screen_capture_size = nw <= 0 || nh <= 0; - cap_x = nx.max(0) as i16; - cap_y = ny.max(0) as i16; + cap_x = clamp_offset(nx, root_w); + cap_y = clamp_offset(ny, root_h); if let Some(g) = conn.get_geometry(root).ok().and_then(|c| c.reply().ok()) { + root_w = g.width; + root_h = g.height; + cap_x = clamp_offset(nx, root_w); + cap_y = clamp_offset(ny, root_h); let (fw, fh) = resolve_dims(g.width, g.height, &rsettings); if fw != cap_w || fh != cap_h { if !pool.drain_for_resize(POOL_N, &controls.stop) { @@ -973,6 +941,12 @@ where if (settings.auto_adjust_screen_capture_size || grab_failures > 0) && geometry_check <= 0 { geometry_check = GEOMETRY_POLL_FRAMES; if let Some(g) = conn.get_geometry(root).ok().and_then(|c| c.reply().ok()) { + // A root shrink can strand even a previously-valid offset past + // the new edge; re-clamp it against the live root. + root_w = g.width; + root_h = g.height; + cap_x = clamp_offset(rsettings.capture_x, root_w); + cap_y = clamp_offset(rsettings.capture_y, root_h); let (nw, nh) = resolve_dims(g.width, g.height, &rsettings); if nw != cap_w || nh != cap_h { if !pool.drain_for_resize(POOL_N, &controls.stop) { @@ -1004,8 +978,11 @@ where Some(i) => i, None => break, }; - let surface = &mut surfaces[idx]; - if let Err(e) = grab_frame(&conn, root, surface, cap_x, cap_y, cap_w, cap_h) { + let grab_result = { + let surface = &mut surfaces[idx]; + grab_frame(&conn, root, surface, cap_x, cap_y, cap_w, cap_h) + }; + if let Err(e) = grab_result { // Most likely an external root resize between geometry polls made // the grab run past the root edge: return the surface, re-poll // geometry immediately, and only give up if it never recovers. @@ -1013,11 +990,45 @@ where grab_failures += 1; geometry_check = 0; if grab_failures > 120 { - return Err(e); + // A burn-out usually means the X connection itself is gone + // (Xorg restart, VT switch): every request fails until the + // server is back. Rebuild the channel a few times before + // declaring the capture dead — a recovered server resumes the + // stream where a dead thread would leave a black screen until + // the Python watchdog restarts everything. + let mut recovered = false; + for _attempt in 0..5 { + if controls.stop.load(Ordering::Relaxed) { + break; + } + std::thread::sleep(Duration::from_secs(1)); + match try_rebuild_channel( + &mut conn, &mut root, &rsettings, + &mut cap_x, &mut cap_y, &mut cap_w, &mut cap_h, + &mut root_w, &mut root_h, + POOL_N, &pool, &mut surfaces, &controls.stop, + ) { + Ok(()) => { recovered = true; break; } + Err(re) => { + eprintln!("[pixelflux x11] reconnect attempt failed: {re}"); + } + } + } + if !recovered { + return Err(format!("capture could not recover: {e}")); + } + grab_failures = 0; + geometry_check = 0; + reconnect_streak += 1; + if reconnect_streak >= 4 { + return Err("capture recovered its channel but the region never grabbed again".to_string()); + } } continue; } grab_failures = 0; + reconnect_streak = 0; + let surface = &mut surfaces[idx]; let frame_w = cap_w as i32; let frame_h = cap_h as i32; @@ -1058,9 +1069,9 @@ where } } - if watermark.loaded { + if watermark.is_active() { watermark.update_position(frame_w, frame_h, settings.watermark_location_enum); - watermark.blend_into(buf, stride, frame_w, frame_h); + watermark.blend_bgra(buf, stride, frame_w, frame_h); } let published = pool.publish( @@ -1087,7 +1098,12 @@ where for s in surfaces.iter_mut() { s.destroy(&conn); } - result + match result { + // A real capture error is the more specific diagnosis, so it outranks the panic flag. + Err(e) => Err(e), + Ok(()) if encode_panicked.load(Ordering::Acquire) => Err("encode thread panicked".into()), + Ok(()) => Ok(()), + } } #[cfg(test)] @@ -1271,6 +1287,46 @@ mod cursor_tests { assert_eq!(px(5, 5), 0, "no pixel at the un-offset (hotspot) corner"); } + #[test] + fn offset_clamps_into_root() { + assert_eq!(clamp_offset(100, 1920), 100); + assert_eq!(clamp_offset(-50, 1920), 0); + // Past-end offsets keep one root pixel instead of failing forever. + assert_eq!(clamp_offset(100000, 1920), 1919); + // A protocol-max root still maps into i16 X coordinates. + assert_eq!(clamp_offset(40000, u16::MAX), 32766); + } + + #[test] + fn resolve_dims_survives_past_end_offset() { + let s = RustCaptureSettings { + capture_x: 100000, + capture_y: 100000, + width: 1920, + height: 1080, + output_mode: 1, + ..Default::default() + }; + let (w, h) = resolve_dims(1920, 1080, &s); + assert!(w >= 2 && h >= 2, "a past-end offset now degrades to a small grab, not death"); + assert_eq!(w % 2, 0); + assert_eq!(h % 2, 0); + } + + /// `overlay_cursor` treats XFixes pixels as premultiplied: half-alpha at half + /// intensity stays at that intensity over black (the straight-alpha formula + /// would multiply alpha twice and darken it). + #[test] + fn overlay_blends_premultiplied_pixels() { + let stride = 4 * 4; + let mut frame = vec![0u8; stride * 4]; + // a=128, g=128 (premultiplied half-intensity), b=64. + let pixels = [0x8000_8040u32; 4]; + overlay_cursor(&mut frame, stride, 4, 4, 2, 2, &pixels, 0, 0); + assert_eq!(frame[0], 64, "blue channel: dst = 64 over black"); + assert_eq!(frame[1], 128, "premultiplied half-alpha stays 128, not darkened to 64"); + } + /// `overlay_cursor` clips a negative origin at the frame edge: with the origin at (-1,-1) /// only the in-frame quadrant is written, landing the cursor's bottom-right pixel at (0,0). #[test] diff --git a/pyproject.toml b/pyproject.toml index 72feece..02c2e8a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -7,7 +7,7 @@ archs = [ "x86_64", "aarch64" ] manylinux-x86_64-image = "manylinux_2_28" manylinux-aarch64-image = "manylinux_2_28" build = "cp39-* cp310-* cp311-* cp312-* cp313-* cp314-*" -skip = "*t-* pp*" +skip = "*t-*" [tool.cibuildwheel.environment] LD_LIBRARY_PATH = "/usr/local/lib:/usr/local/lib64" @@ -16,7 +16,21 @@ PATH = "$HOME/.cargo/bin:$PATH" [tool.cibuildwheel.linux] before-all = """ set -euxo pipefail - + + # A transient clone failure fails the whole wheel build, so every source is + # retried; FFmpeg is additionally mirrored on GitHub at the identical tag. + clone_source() { + ref="$1"; dir="$2"; shift 2 + for url in "$@"; do + for attempt in 1 2 3; do + rm -rf "${dir}" + git clone --branch "${ref}" --depth 1 "${url}" "${dir}" && return 0 + sleep $((attempt * 5)) + done + done + return 1 + } + # --- MANYLINUX (RPM-based) --- # x264 is built from source for x264-sys (manylinux has no usable package); # ffmpeg n8.1 is built plain LGPL (no --enable-libx264/--enable-gpl: only @@ -48,17 +62,17 @@ before-all = """ cmake \ gcc-c++ && \ \ - curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y --profile minimal && \ + curl --proto '=https' --tlsv1.2 -sSf --retry 5 --retry-delay 3 --retry-connrefused https://sh.rustup.rs | sh -s -- -y --profile minimal && \ \ (cd /tmp && \ - git clone --branch stable --depth 1 https://code.videolan.org/videolan/x264.git && \ + clone_source stable x264 https://code.videolan.org/videolan/x264.git && \ cd x264 && \ ./configure --prefix=/usr/local --enable-shared --disable-cli && \ make -j$(nproc) && \ make install) && \ \ (cd /tmp && \ - git clone --branch n8.1 --depth 1 https://git.ffmpeg.org/ffmpeg.git && \ + clone_source n8.1 ffmpeg https://git.ffmpeg.org/ffmpeg.git https://github.com/FFmpeg/FFmpeg.git && \ cd ffmpeg && \ ./configure \ --prefix=/usr/local \ @@ -97,7 +111,7 @@ before-all = """ musl-dev \ ffmpeg-dev && \ \ - curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y --profile minimal + curl --proto '=https' --tlsv1.2 -sSf --retry 5 --retry-delay 3 --retry-connrefused https://sh.rustup.rs | sh -s -- -y --profile minimal else echo "Unsupported package manager" exit 1 diff --git a/setup.py b/setup.py index a18493b..916e1c1 100644 --- a/setup.py +++ b/setup.py @@ -2,12 +2,39 @@ # License, v. 2.0. If a copy of the MPL was not distributed with this # file, You can obtain one at https://mozilla.org/MPL/2.0/. +import os +import sys + from setuptools import setup from setuptools_rust import Binding, RustExtension, Strip with open("README.md", "r", encoding="utf-8") as fh: long_description = fh.read() +# GPL components (GPL-2.0+ libx264, used for striped software H.264) are enabled by +# default. Set PIXELFLUX_ENABLE_GPL=0 (or "false"/"no") to build without them; the +# BSD-licensed OpenH264 encoder is then substituted for software H.264. +_enable_gpl = os.environ.get("PIXELFLUX_ENABLE_GPL", "1").strip().lower() not in ( + "0", + "false", + "no", + "off", +) +if _enable_gpl: + print( + "NOTICE: pixelflux is being built WITH GPL-licensed components " + "(GPL-2.0+ libx264 for striped software H.264), which is the default. " + "Set PIXELFLUX_ENABLE_GPL=0 to exclude every GPL-licensed component.", + file=sys.stderr, + ) +else: + print( + "NOTICE: pixelflux is being built WITHOUT GPL-licensed components " + "(PIXELFLUX_ENABLE_GPL=0): libx264 is excluded and OpenH264 (BSD) " + "substitutes for software H.264.", + file=sys.stderr, + ) + setup( name="pixelflux", version="2.0.0", @@ -29,7 +56,8 @@ "pixelflux/Cargo.toml", binding=Binding.PyO3, debug=False, - strip=Strip.All + strip=Strip.All, + args=([] if _enable_gpl else ["--no-default-features"]), ) ],