feat(graphics): add direct pane frame streaming

docs/next/CHANGELOG.md

@@ -11,6 +11,7 @@
 
 ### Changed
 - Bumped the client/server protocol version to 20 for pane terminal bell forwarding.
+- Experimental pane graphics now support bounded named layers, acknowledged full-RGBA primary-layer direct file frames on audited local terminals, owned BGRA fallback, exact pixel mouse input, and placement-only resize replay.
 
 ### Fixed
 - `herdr config check` now reports unknown built-in theme names instead of silently accepting them. (#2452)

docs/next/api/herdr-api.schema.json

@@ -2452,6 +2452,12 @@
         },
         "PaneGraphicsClearParams": {
           "properties": {
+            "layer_id": {
+              "type": [
+                "string",
+                "null"
+              ]
+            },
             "pane_id": {
               "type": "string"
             }
@@ -2465,7 +2471,8 @@
           "enum": [
             "png",
             "rgb",
-            "rgba"
+            "rgba",
+            "bgra"
           ],
           "type": "string"
         },
@@ -2515,6 +2522,12 @@
               "minimum": 0,
               "type": "integer"
             },
+            "layer_id": {
+              "type": [
+                "string",
+                "null"
+              ]
+            },
             "pane_id": {
               "type": "string"
             },
@@ -2526,6 +2539,11 @@
                 "viewport_col": 0,
                 "viewport_row": 0
               }
+            },
+            "z_index": {
+              "default": 0,
+              "format": "int32",
+              "type": "integer"
             }
           },
           "required": [
@@ -9234,6 +9252,30 @@
               ],
               "type": "object"
             },
+            {
+              "properties": {
+                "revision": {
+                  "format": "uint64",
+                  "minimum": 0,
+                  "type": "integer"
+                },
+                "sequence": {
+                  "format": "uint64",
+                  "minimum": 0,
+                  "type": "integer"
+                },
+                "type": {
+                  "const": "pane_graphics_frame_ack",
+                  "type": "string"
+                }
+              },
+              "required": [
+                "type",
+                "sequence",
+                "revision"
+              ],
+              "type": "object"
+            },
             {
               "properties": {
                 "cell_height_px": {
@@ -9246,6 +9288,51 @@
                   "minimum": 0,
                   "type": "integer"
                 },
+                "file_frame_damage": {
+                  "default": false,
+                  "description": "Accepts damage metadata while still consuming a complete canonical file.",
+                  "type": "boolean"
+                },
+                "file_frame_directory": {
+                  "type": [
+                    "string",
+                    "null"
+                  ]
+                },
+                "file_frame_formats": {
+                  "items": {
+                    "type": "string"
+                  },
+                  "type": "array"
+                },
+                "file_frame_max_bytes": {
+                  "format": "uint",
+                  "minimum": 0,
+                  "type": [
+                    "integer",
+                    "null"
+                  ]
+                },
+                "file_frame_transport": {
+                  "type": [
+                    "string",
+                    "null"
+                  ]
+                },
+                "max_layers_per_pane": {
+                  "default": 0,
+                  "format": "uint",
+                  "minimum": 0,
+                  "type": "integer"
+                },
+                "pane_visible": {
+                  "description": "True only when this pane is on the currently rendered terminal surface.",
+                  "type": "boolean"
+                },
+                "pixel_mouse": {
+                  "default": false,
+                  "type": "boolean"
+                },
                 "type": {
                   "const": "pane_graphics_info",
                   "type": "string"
@@ -9254,7 +9341,8 @@
               "required": [
                 "type",
                 "cell_width_px",
-                "cell_height_px"
+                "cell_height_px",
+                "pane_visible"
               ],
               "type": "object"
             },

docs/next/website/src/content/docs/socket-api.mdx

@@ -177,10 +177,20 @@ Clients can treat `offset_from_bottom == 0` as at-bottom state.
 
 Pane graphics let a plugin place image data over a pane. They are available
 only when `[experimental].kitty_graphics = true`; otherwise every pane graphics
-method returns `feature_disabled`. Calling `pane.graphics.info` returns the
-attached client's cell width and height in pixels without creating a graphics
-layer. `pane.graphics.set` accepts `png`, `rgb`, or `rgba` data in `data_base64`,
-and `pane.graphics.clear` removes the layer.
+method returns `feature_disabled`. Calling `pane.graphics.info` explicitly
+activates capability discovery and returns the attached client's cell size,
+file-frame options, pixel-mouse support, the 16-layer limit, and `pane_visible`.
+`pane_visible` is true only when the target is in the active workspace and tab
+and is not hidden by zoom. Short-lived UI modes do not change it.
+
+`pane.graphics.set`, `pane.graphics.clear`, and `pane.graphics.stream` accept an
+optional `layer_id` (default `primary`). Set and stream also accept `z_index`;
+layers are placed in stable `(z_index, layer_id)` order. Each stream exclusively owns its layer, and
+closing it removes that layer. Inline frames accept `png`, `rgb`, `rgba`, or
+`bgra`; BGRA is normalized once to owned RGBA. Herdr advances the host cache
+one image transaction per render pass, so arbitrary layer sets progress without
+an aggregate frame. Headless transport keeps each transaction within its 32 MiB
+wire limit; local monolithic rendering does not apply that transport limit.
 
 ```json
 {"id":"graphics_info","method":"pane.graphics.info","params":{"pane_id":"w1:p1"}}
@@ -190,14 +200,31 @@ and `pane.graphics.clear` removes the layer.
 
 For repeated frames, open a dedicated socket with `pane.graphics.stream`. After
 Herdr replies with `ok`, send one JSON header and then exactly `data_length` raw
-bytes per frame. A stream owns that pane's graphics layer until the socket
-closes; concurrent set, clear, or stream requests return `stream_conflict`.
+bytes per inline frame. Concurrent operations on that layer return
+`stream_conflict`.
 
 ```json
-{"id":"graphics_stream","method":"pane.graphics.stream","params":{"pane_id":"w1:p1"}}
+{"id":"graphics_stream","method":"pane.graphics.stream","params":{"pane_id":"w1:p1","z_index":0}}
 {"format":"png","image_width":800,"image_height":600,"data_length":12345,"placement":{"viewport_col":0,"viewport_row":0,"grid_cols":80,"grid_rows":30}}
 ```
 
+When `pane.graphics.info` advertises `file_frame_transport: "direct-kitty"`, an
+eligible local Ghostty, kitty, or WezTerm client may submit an immutable private
+`rgba` or `bgra` file with `file.path`, `sequence`, and `revision`. Direct Kitty
+file transport is reserved for the default `primary` page layer; named secondary
+layers use owned inline RGBA. BGRA is always copied, swizzled, and rendered
+inline. Herdr replies with a `pane_graphics_frame_ack` only after the terminal
+accepts the file, or after a safe owned inline fallback is installed. Confirmed
+file-transport failure disables direct files for that client connection without
+disabling exact pixel mouse. A timeout or client loss closes the stream without
+acknowledging source reuse. Monolithic `--no-session` mode advertises neither
+fast file transport nor exact pixel mouse and remains on owned inline fallback.
+
+Direct files are always complete canonical `width * height * 4` RGBA frames.
+`file_frame_damage: true` means Herdr accepts optional damage metadata for
+producer-side canonical-ring efficiency; it still copies or presents the full file.
+Resize and full redraw replay placements without retransmitting pixels.
+
 `pane.layout` returns the tab layout snapshot with `workspace_id`, `tab_id`,
 `zoomed`, outer `area`, `focused_pane_id`, pane rects, and split rects/ratios.
 `pane.neighbor` and `pane.edges` include that same layout snapshot so clients

src/api/mod.rs

@@ -7,7 +7,6 @@ mod subscriptions;
 mod wait;
 
 pub use event_hub::EventHub;
-pub(crate) use server::cancel_inactive_pane_graphics_streams;
 pub use server::{start_server, start_server_with_capabilities, ServerHandle};
 pub use status::{read_runtime_status_at, RuntimeStatus};
 
@@ -62,6 +61,7 @@ pub(crate) fn request_changes_ui(request: &Request) -> bool {
             | Method::PaneGraphicsClear(_)
             | Method::PaneGraphicsStream(_)
             | Method::PaneGraphicsStreamSet(_)
+            | Method::PaneGraphicsStreamDirect(_)
             | Method::PaneGraphicsStreamOpen(_)
             | Method::PaneGraphicsStreamClose(_)
             | Method::PaneReportAgent(_)
@@ -84,6 +84,7 @@ pub struct ApiRequestMessage {
     pub request: Request,
     pub respond_to: std::sync::mpsc::Sender<String>,
     pub response_write_complete: Option<std::sync::mpsc::Receiver<()>>,
+    pub stream_active: Option<std::sync::Arc<std::sync::atomic::AtomicBool>>,
 }
 
 pub type ApiRequestSender = mpsc::UnboundedSender<ApiRequestMessage>;

src/api/schema.rs

@@ -187,6 +187,9 @@ pub enum Method {
     PaneGraphicsStreamSet(PaneGraphicsSetParams),
     #[serde(skip)]
     #[schemars(skip)]
+    PaneGraphicsStreamDirect(PaneGraphicsDirectParams),
+    #[serde(skip)]
+    #[schemars(skip)]
     PaneGraphicsStreamOpen(PaneGraphicsStreamParams),
     #[serde(skip)]
     #[schemars(skip)]

src/api/schema/panes.rs

@@ -4,6 +4,10 @@ use serde::{Deserialize, Serialize};
 
 pub(crate) const PANE_GRAPHICS_SET_MAX_BYTES: usize = 512 * 1024;
 pub(crate) const PANE_GRAPHICS_STREAM_MAX_BYTES: usize = 16 * 1024 * 1024;
+pub(crate) const PANE_GRAPHICS_MAX_LAYERS_PER_PANE: usize = 16;
+pub(crate) const PANE_GRAPHICS_MAX_LAYERS_TOTAL: usize = 64;
+pub(crate) const PANE_GRAPHICS_MAX_INLINE_BYTES_TOTAL: usize = 64 * 1024 * 1024;
+pub(crate) const PANE_GRAPHICS_PRIMARY_LAYER_ID: &str = "primary";
 
 use super::agents::AgentSessionInfo;
 use super::common::{AgentStatus, PaneAgentState, ReadFormat, ReadSource, SplitDirection};
@@ -287,11 +291,16 @@ pub enum PaneGraphicsFormat {
     Png,
     Rgb,
     Rgba,
+    Bgra,
 }
 
 #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, schemars::JsonSchema)]
 pub struct PaneGraphicsSetParams {
     pub pane_id: String,
+    #[serde(default, skip_serializing_if = "Option::is_none")]
+    pub layer_id: Option<String>,
+    #[serde(default)]
+    pub z_index: i32,
     #[serde(skip)]
     #[schemars(skip)]
     pub owner: String,
@@ -323,11 +332,32 @@ pub struct PaneGraphicsPlacementParams {
 #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, schemars::JsonSchema)]
 pub struct PaneGraphicsClearParams {
     pub pane_id: String,
+    #[serde(default, skip_serializing_if = "Option::is_none")]
+    pub layer_id: Option<String>,
+}
+
+#[derive(Debug, Clone, PartialEq, Eq)]
+pub struct PaneGraphicsDirectParams {
+    pub pane_id: String,
+    pub layer_id: Option<String>,
+    pub z_index: i32,
+    pub owner: String,
+    pub image_width: u32,
+    pub image_height: u32,
+    pub format: PaneGraphicsFormat,
+    pub path: String,
+    pub sequence: u64,
+    pub revision: u64,
+    pub placement: PaneGraphicsPlacementParams,
 }
 
 #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, schemars::JsonSchema)]
 pub struct PaneGraphicsStreamParams {
     pub pane_id: String,
+    #[serde(default, skip_serializing_if = "Option::is_none")]
+    pub layer_id: Option<String>,
+    #[serde(default)]
+    pub z_index: i32,
     #[serde(skip)]
     #[schemars(skip)]
     pub owner: String,

src/api/schema/response.rs

@@ -162,9 +162,30 @@ pub enum ResponseResult {
     PaneRead {
         read: PaneReadResult,
     },
+    PaneGraphicsFrameAck {
+        sequence: u64,
+        revision: u64,
+    },
     PaneGraphicsInfo {
         cell_width_px: u32,
         cell_height_px: u32,
+        /// True only when this pane is on the currently rendered terminal surface.
+        pane_visible: bool,
+        #[serde(default, skip_serializing_if = "Option::is_none")]
+        file_frame_directory: Option<String>,
+        #[serde(default, skip_serializing_if = "Vec::is_empty")]
+        file_frame_formats: Vec<String>,
+        #[serde(default, skip_serializing_if = "Option::is_none")]
+        file_frame_max_bytes: Option<usize>,
+        /// Accepts damage metadata while still consuming a complete canonical file.
+        #[serde(default)]
+        file_frame_damage: bool,
+        #[serde(default)]
+        max_layers_per_pane: usize,
+        #[serde(default)]
+        pixel_mouse: bool,
+        #[serde(default, skip_serializing_if = "Option::is_none")]
+        file_frame_transport: Option<String>,
     },
     AgentExplain {
         explain: serde_json::Value,