fix(graphics): support high-dpi direct file frames

docs/next/api/herdr-api.schema.json

@@ -9295,6 +9295,14 @@
                   "description": "Accepts damage metadata while still consuming a complete canonical file.",
                   "type": "boolean"
                 },
+                "file_frame_direct_max_bytes": {
+                  "format": "uint",
+                  "minimum": 0,
+                  "type": [
+                    "integer",
+                    "null"
+                  ]
+                },
                 "file_frame_directory": {
                   "type": [
                     "string",

docs/next/website/src/content/docs/socket-api.mdx

@@ -221,6 +221,13 @@ acknowledging source reuse. Monolithic `--no-session` mode advertises neither
 fast file transport nor exact pixel mouse and remains on owned inline fallback.
 
 Direct files are always complete canonical `width * height * 4` RGBA frames.
+`file_frame_max_bytes` is the limit that remains eligible for owned inline fallback.
+Primary-layer RGBA files may use the larger `file_frame_direct_max_bytes` limit when
+`file_frame_transport` is available. Frames above the fallback limit are acknowledged
+only when the terminal accepts the direct transfer; rejection closes the stream. If a
+frame cannot use owned inline fallback while its pane is temporarily hidden or cannot be
+placed during a redraw, Herdr uploads the image without displaying it and replays its
+placement when the pane becomes visible again.
 `file_frame_damage: true` means Herdr accepts optional damage metadata for
 producer-side canonical-ring efficiency; it still copies or presents the full file.
 Resize and full redraw replay placements without retransmitting pixels.

src/api/schema/panes.rs

@@ -4,6 +4,7 @@ use serde::{Deserialize, Serialize};
 
 pub(crate) const PANE_GRAPHICS_SET_MAX_BYTES: usize = 512 * 1024;
 pub(crate) const PANE_GRAPHICS_STREAM_MAX_BYTES: usize = 16 * 1024 * 1024;
+pub(crate) const PANE_GRAPHICS_DIRECT_FILE_MAX_BYTES: usize = 400 * 1024 * 1024;
 pub(crate) const PANE_GRAPHICS_MAX_LAYERS_PER_PANE: usize = 16;
 pub(crate) const PANE_GRAPHICS_MAX_LAYERS_TOTAL: usize = 64;
 pub(crate) const PANE_GRAPHICS_MAX_INLINE_BYTES_TOTAL: usize = 64 * 1024 * 1024;

src/api/schema/response.rs

@@ -177,6 +177,8 @@ pub enum ResponseResult {
         file_frame_formats: Vec<String>,
         #[serde(default, skip_serializing_if = "Option::is_none")]
         file_frame_max_bytes: Option<usize>,
+        #[serde(default, skip_serializing_if = "Option::is_none")]
+        file_frame_direct_max_bytes: Option<usize>,
         /// Accepts damage metadata while still consuming a complete canonical file.
         #[serde(default)]
         file_frame_damage: bool,

src/app/api/pane_graphics.rs

@@ -2,8 +2,8 @@ use base64::Engine;
 
 use crate::api::schema::{
     PaneGraphicsClearParams, PaneGraphicsSetParams, PaneGraphicsStreamParams, ResponseResult,
-    PANE_GRAPHICS_MAX_LAYERS_PER_PANE, PANE_GRAPHICS_PRIMARY_LAYER_ID, PANE_GRAPHICS_SET_MAX_BYTES,
-    PANE_GRAPHICS_STREAM_MAX_BYTES,
+    PANE_GRAPHICS_DIRECT_FILE_MAX_BYTES, PANE_GRAPHICS_MAX_LAYERS_PER_PANE,
+    PANE_GRAPHICS_PRIMARY_LAYER_ID, PANE_GRAPHICS_SET_MAX_BYTES, PANE_GRAPHICS_STREAM_MAX_BYTES,
 };
 use crate::app::pane_graphics::{Key as PaneGraphicsKey, Layer, Slot};
 use crate::app::App;
@@ -65,6 +65,7 @@ impl App {
                     Vec::new()
                 },
                 file_frame_max_bytes: direct.then_some(PANE_GRAPHICS_STREAM_MAX_BYTES),
+                file_frame_direct_max_bytes: direct.then_some(PANE_GRAPHICS_DIRECT_FILE_MAX_BYTES),
                 // Damage metadata never changes the complete canonical frame contract.
                 file_frame_damage: true,
                 max_layers_per_pane: PANE_GRAPHICS_MAX_LAYERS_PER_PANE,
@@ -318,9 +319,18 @@ impl App {
         ) {
             return encode_error(id, "invalid_image", "direct frames require rgba or bgra");
         }
+        let primary = key.1 == PANE_GRAPHICS_PRIMARY_LAYER_ID;
+        let direct = self.direct_graphics_available
+            && primary
+            && params.format == crate::api::schema::PaneGraphicsFormat::Rgba;
+        let max_bytes = if direct {
+            PANE_GRAPHICS_DIRECT_FILE_MAX_BYTES
+        } else {
+            PANE_GRAPHICS_STREAM_MAX_BYTES
+        };
         let expected_len =
             match expected_len(params.format, params.image_width, params.image_height) {
-                Ok(Some(len)) if len <= PANE_GRAPHICS_STREAM_MAX_BYTES => len,
+                Ok(Some(len)) if len <= max_bytes => len,
                 _ => return encode_error(id, "invalid_image", "invalid direct RGBA dimensions"),
             };
         let lease = match self
@@ -330,10 +340,6 @@ impl App {
             Ok(lease) => lease,
             Err(err) => return encode_error(id, "invalid_frame_file", err.to_string()),
         };
-        let primary = key.1 == PANE_GRAPHICS_PRIMARY_LAYER_ID;
-        let direct = self.direct_graphics_available
-            && primary
-            && params.format == crate::api::schema::PaneGraphicsFormat::Rgba;
         if !direct && !self.pane_graphics.can_store_inline(&key, expected_len) {
             return encode_error(
                 id,
@@ -699,6 +705,14 @@ mod tests {
             value["result"]["file_frame_formats"],
             serde_json::json!(["rgba", "bgra"])
         );
+        assert_eq!(
+            value["result"]["file_frame_max_bytes"],
+            PANE_GRAPHICS_STREAM_MAX_BYTES
+        );
+        assert_eq!(
+            value["result"]["file_frame_direct_max_bytes"],
+            PANE_GRAPHICS_DIRECT_FILE_MAX_BYTES
+        );
         assert_eq!(value["result"]["file_frame_damage"], true);
         assert_eq!(value["result"]["file_frame_transport"], "direct-kitty");
     }
@@ -1038,6 +1052,24 @@ mod tests {
         path.to_string_lossy().into_owned()
     }
 
+    #[cfg(unix)]
+    fn sparse_direct_file(app: &App, name: &str, len: usize) -> String {
+        use std::os::unix::fs::OpenOptionsExt as _;
+        let path = app
+            .pane_graphics_files
+            .source_directory()
+            .unwrap()
+            .join(name);
+        let file = std::fs::OpenOptions::new()
+            .write(true)
+            .create_new(true)
+            .mode(0o600)
+            .open(&path)
+            .unwrap();
+        file.set_len(len as u64).unwrap();
+        path.to_string_lossy().into_owned()
+    }
+
     #[cfg(unix)]
     fn direct_params(
         pane_id: String,
@@ -1145,6 +1177,46 @@ mod tests {
         );
     }
 
+    #[cfg(unix)]
+    #[test]
+    fn direct_primary_rgba_accepts_fullscreen_retina_frame() {
+        let (mut app, pane_id) = app();
+        app.direct_graphics_available = true;
+        app.handle_pane_graphics_stream_open(
+            "open".into(),
+            PaneGraphicsStreamParams {
+                pane_id: pane_id.clone(),
+                layer_id: None,
+                z_index: 0,
+                owner: "owner".into(),
+            },
+        );
+        let (image_width, image_height) = (3456, 2234);
+        let len = image_width * image_height * 4;
+        let path = sparse_direct_file(&app, "retina-frame", len as usize);
+        let response = app.handle_pane_graphics_stream_direct(
+            "frame".into(),
+            crate::api::schema::PaneGraphicsDirectParams {
+                image_width,
+                image_height,
+                ..direct_params(pane_id, "owner", path)
+            },
+        );
+
+        assert!(serde_json::from_str::<SuccessResponse>(&response).is_ok());
+        assert!(app
+            .pane_graphics
+            .slots
+            .values()
+            .next()
+            .unwrap()
+            .layer
+            .as_ref()
+            .unwrap()
+            .direct_lease()
+            .is_some());
+    }
+
     #[cfg(unix)]
     #[test]
     fn bgra_and_secondary_file_frames_are_canonical_owned_rgba() {

src/client/direct_graphics.rs

@@ -212,42 +212,47 @@ fn matching_response_controls(bytes: &[u8], expected: u32) -> bool {
     matched
 }
 
-pub(super) fn valid_control(control: &str, image_id: u32) -> bool {
+pub(super) fn valid_control(control: &str, image_id: u32, expected_len: usize) -> bool {
     if control.len() > 1024 || control.contains([';', '\x1b']) {
         return false;
     }
-    let mut action = false;
+    let mut seen = 0_u32;
+    let mut action = None;
     let mut format = false;
     let mut image = false;
     let mut quiet = false;
-    let mut cursor = false;
+    let mut width = None;
+    let mut height = None;
+    let mut placement = [false; 5];
+    let mut has_placement_controls = false;
     for field in control.split(',') {
         let Some((key, value)) = field.split_once('=') else {
             return false;
         };
-        if key == "t"
-            || !matches!(
-                key,
-                "a" | "f"
-                    | "s"
-                    | "v"
-                    | "i"
-                    | "p"
-                    | "c"
-                    | "r"
-                    | "z"
-                    | "C"
-                    | "q"
-                    | "x"
-                    | "y"
-                    | "w"
-                    | "h"
-                    | "X"
-                    | "Y"
-            )
-        {
+        let key_bit = match key {
+            "a" => 1 << 0,
+            "f" => 1 << 1,
+            "s" => 1 << 2,
+            "v" => 1 << 3,
+            "i" => 1 << 4,
+            "p" => 1 << 5,
+            "c" => 1 << 6,
+            "r" => 1 << 7,
+            "z" => 1 << 8,
+            "C" => 1 << 9,
+            "q" => 1 << 10,
+            "x" => 1 << 11,
+            "y" => 1 << 12,
+            "w" => 1 << 13,
+            "h" => 1 << 14,
+            "X" => 1 << 15,
+            "Y" => 1 << 16,
+            _ => return false,
+        };
+        if seen & key_bit != 0 {
             return false;
         }
+        seen |= key_bit;
         let numeric = value
             .strip_prefix('-')
             .unwrap_or(value)
@@ -258,15 +263,47 @@ pub(super) fn valid_control(control: &str, image_id: u32) -> bool {
             return false;
         }
         match key {
-            "a" => action = value == "T",
+            "a" if matches!(value, "T" | "t") => action = Some(value),
+            "a" => return false,
             "f" => format = value == "32",
+            "s" => width = value.parse::<usize>().ok().filter(|value| *value > 0),
+            "v" => height = value.parse::<usize>().ok().filter(|value| *value > 0),
             "i" => image = value.parse() == Ok(image_id),
             "q" => quiet = value == "0",
-            "C" => cursor = value == "1",
+            "p" => {
+                placement[0] = true;
+                has_placement_controls = true;
+            }
+            "c" => {
+                placement[1] = true;
+                has_placement_controls = true;
+            }
+            "r" => {
+                placement[2] = true;
+                has_placement_controls = true;
+            }
+            "z" => {
+                placement[3] = true;
+                has_placement_controls = true;
+            }
+            "C" => {
+                placement[4] = value == "1";
+                has_placement_controls = true;
+            }
+            "x" | "y" | "w" | "h" | "X" | "Y" => has_placement_controls = true,
             _ => {}
         }
     }
-    action && format && image && quiet && cursor
+    let dimensions_match = width
+        .zip(height)
+        .and_then(|(width, height)| width.checked_mul(height)?.checked_mul(4))
+        == Some(expected_len);
+    let profile_matches = match action {
+        Some("T") => placement.into_iter().all(|present| present),
+        Some("t") => !has_placement_controls,
+        _ => false,
+    };
+    format && image && quiet && dimensions_match && profile_matches
 }
 
 #[cfg(test)]
@@ -313,18 +350,25 @@ mod tests {
     }
 
     #[test]
-    fn validated_control_is_one_owned_rgba_transmit_and_display() {
+    fn validated_control_accepts_only_owned_rgba_direct_profiles() {
         assert!(valid_control(
             "a=T,f=32,s=10,v=20,i=42,p=7,c=5,r=6,z=-1,C=1,q=0,x=2",
-            42
+            42,
+            800,
         ));
+        assert!(valid_control("a=t,f=32,s=10,v=20,i=42,q=0", 42, 800,));
         for invalid in [
-            "a=T,f=24,i=42,C=1,q=0",
-            "a=T,f=32,i=41,C=1,q=0",
-            "a=T,t=f,f=32,i=42,C=1,q=0",
-            "a=p,f=32,i=42,C=1,q=0",
+            "a=T,f=24,s=10,v=20,i=42,p=7,c=5,r=6,z=-1,C=1,q=0",
+            "a=T,f=32,s=10,v=20,i=41,p=7,c=5,r=6,z=-1,C=1,q=0",
+            "a=T,t=f,f=32,s=10,v=20,i=42,p=7,c=5,r=6,z=-1,C=1,q=0",
+            "a=p,f=32,s=10,v=20,i=42,q=0",
+            "a=t,f=32,s=10,v=20,i=42,C=1,q=0",
+            "a=t,f=32,s=10,v=20,i=42,p=7,q=0",
+            "a=t,f=32,s=10,v=19,i=42,q=0",
+            "a=t,f=32,s=10,i=42,q=0",
+            "a=t,f=32,s=10,s=10,v=20,i=42,q=0",
         ] {
-            assert!(!valid_control(invalid, 42), "{invalid}");
+            assert!(!valid_control(invalid, 42, 800), "{invalid}");
         }
     }
 

src/client/mod.rs

@@ -1721,8 +1721,8 @@ async fn run_client_loop(
                                     len,
                                 )
                                 .is_ok()
+                                    && direct_graphics::valid_control(&control, image_id, len)
                             })
-                            && direct_graphics::valid_control(&control, image_id)
                             && state
                                 .direct_graphics_response
                                 .lock()

src/ghostty/mod.rs

@@ -3488,6 +3488,43 @@ mod tests {
         let _ = std::fs::remove_dir_all(dir);
     }
 
+    #[cfg(unix)]
+    #[test]
+    fn kitty_graphics_file_upload_can_be_placed_later() {
+        let dir = std::env::temp_dir().join(format!(
+            "herdr-kitty-file-upload-test-{}",
+            std::process::id()
+        ));
+        std::fs::create_dir_all(&dir).unwrap();
+        let path = dir.join("pixel.rgba");
+        std::fs::write(&path, [255, 0, 0, 255]).unwrap();
+
+        let mut terminal = Terminal::new(10, 5, 0).unwrap();
+        terminal.enable_kitty_graphics().unwrap();
+        terminal.resize(10, 5, 8, 16).unwrap();
+        let mut upload = Vec::new();
+        crate::kitty_graphics::encode_kitty_regular_file(
+            &mut upload,
+            &[],
+            "a=t,f=32,s=1,v=1,i=10,q=0",
+            path.to_str().unwrap(),
+        );
+        terminal.write(&upload);
+        assert!(terminal.kitty_image_placements().unwrap().is_empty());
+
+        terminal.write(b"\x1b_Ga=p,i=10,p=5,c=10,r=5,C=1,q=2\x1b\\");
+        let placements = terminal.kitty_image_placements().unwrap();
+        assert_eq!(placements.len(), 1);
+        assert_eq!(placements[0].image_id, 10);
+        assert_eq!(placements[0].placement_id, 5);
+        assert_eq!(placements[0].image_width, 1);
+        assert_eq!(placements[0].image_height, 1);
+        assert_eq!(placements[0].format, KittyImageFormat::Rgba);
+        assert_eq!(placements[0].data, [255, 0, 0, 255]);
+
+        let _ = std::fs::remove_dir_all(dir);
+    }
+
     #[test]
     fn kitty_graphics_unicode_placeholder_placement_is_queryable() {
         let mut terminal = Terminal::new(10, 5, 0).unwrap();