From 20af1c83378b68ba9180a9e48d3482539e1c0fd0 Mon Sep 17 00:00:00 2001 From: Pekka Enberg Date: Tue, 3 Feb 2026 15:14:55 +0200 Subject: [PATCH 1/2] sdk/rust: Skip ensure_parent_dirs when parent is already in delta layer When creating files/directories in the overlay filesystem, we previously always called ensure_parent_dirs() to walk the entire path and verify each ancestor exists in the delta layer. This was expensive because it checked each path component against both delta and base layers. Now we skip this walk when the parent inode is already in the delta layer, since a directory can only be in delta if all its ancestors are too. This optimization eliminates the ensure_parent_dirs overhead for the common case of creating many files in the same directory. --- sdk/rust/src/filesystem/overlayfs.rs | 53 ++++++++++++++++++++-------- 1 file changed, 38 insertions(+), 15 deletions(-) diff --git a/sdk/rust/src/filesystem/overlayfs.rs b/sdk/rust/src/filesystem/overlayfs.rs index 136235f5..43e6c941 100644 --- a/sdk/rust/src/filesystem/overlayfs.rs +++ b/sdk/rust/src/filesystem/overlayfs.rs @@ -909,14 +909,18 @@ impl FileSystem for OverlayFS { // Remove whiteout if exists self.remove_whiteout(&path).await?; - // Ensure parent dirs exist in delta - self.ensure_parent_dirs(&path, uid, gid).await?; - - // Get delta parent inode + // Resolve the delta parent inode. When the parent is already in the delta layer, + // we can use it directly - all its ancestors are guaranteed to exist in delta + // (a directory can only be in delta if its parents are too). This avoids the + // expensive ensure_parent_dirs() walk which would otherwise check each path + // component against both delta and base layers. let delta_parent_ino = if parent_info.layer == Layer::Delta { parent_info.underlying_ino } else { - // Walk delta to find parent + // Parent is in base layer - we need to create the directory hierarchy in + // delta before we can create the new entry. This walks the path and creates + // any missing directories. + self.ensure_parent_dirs(&path, uid, gid).await?; let mut ino: i64 = 1; for comp in parent_info.path.split('/').filter(|s| !s.is_empty()) { if let Some(s) = FileSystem::lookup(&self.delta, ino, comp).await? { @@ -954,13 +958,18 @@ impl FileSystem for OverlayFS { // Remove whiteout if exists self.remove_whiteout(&path).await?; - // Ensure parent dirs exist in delta - self.ensure_parent_dirs(&path, uid, gid).await?; - - // Get delta parent inode + // Resolve the delta parent inode. When the parent is already in the delta layer, + // we can use it directly - all its ancestors are guaranteed to exist in delta + // (a directory can only be in delta if its parents are too). This avoids the + // expensive ensure_parent_dirs() walk which would otherwise check each path + // component against both delta and base layers. let delta_parent_ino = if parent_info.layer == Layer::Delta { parent_info.underlying_ino } else { + // Parent is in base layer - we need to create the directory hierarchy in + // delta before we can create the new file. This walks the path and creates + // any missing directories. + self.ensure_parent_dirs(&path, uid, gid).await?; let mut ino: i64 = 1; for comp in parent_info.path.split('/').filter(|s| !s.is_empty()) { if let Some(s) = FileSystem::lookup(&self.delta, ino, comp).await? { @@ -993,11 +1002,15 @@ impl FileSystem for OverlayFS { let path = self.build_path(parent_ino, name)?; self.remove_whiteout(&path).await?; - self.ensure_parent_dirs(&path, uid, gid).await?; + // Resolve the delta parent inode. When the parent is already in the delta layer, + // we can use it directly - all its ancestors are guaranteed to exist in delta. + // This avoids the expensive ensure_parent_dirs() walk. let delta_parent_ino = if parent_info.layer == Layer::Delta { parent_info.underlying_ino } else { + // Parent is in base layer - create the directory hierarchy in delta first. + self.ensure_parent_dirs(&path, uid, gid).await?; let mut ino: i64 = 1; for comp in parent_info.path.split('/').filter(|s| !s.is_empty()) { if let Some(s) = FileSystem::lookup(&self.delta, ino, comp).await? { @@ -1034,11 +1047,15 @@ impl FileSystem for OverlayFS { let path = self.build_path(parent_ino, name)?; self.remove_whiteout(&path).await?; - self.ensure_parent_dirs(&path, uid, gid).await?; + // Resolve the delta parent inode. When the parent is already in the delta layer, + // we can use it directly - all its ancestors are guaranteed to exist in delta. + // This avoids the expensive ensure_parent_dirs() walk. let delta_parent_ino = if parent_info.layer == Layer::Delta { parent_info.underlying_ino } else { + // Parent is in base layer - create the directory hierarchy in delta first. + self.ensure_parent_dirs(&path, uid, gid).await?; let mut ino: i64 = 1; for comp in parent_info.path.split('/').filter(|s| !s.is_empty()) { if let Some(s) = FileSystem::lookup(&self.delta, ino, comp).await? { @@ -1172,12 +1189,15 @@ impl FileSystem for OverlayFS { }; self.remove_whiteout(&new_path).await?; - self.ensure_parent_dirs(&new_path, 0, 0).await?; - // Get delta parent + // Resolve the delta parent inode for the new link location. When the parent is + // already in the delta layer, we can use it directly - all its ancestors are + // guaranteed to exist in delta. This avoids the expensive ensure_parent_dirs() walk. let delta_parent_ino = if parent_info.layer == Layer::Delta { parent_info.underlying_ino } else { + // Parent is in base layer - create the directory hierarchy in delta first. + self.ensure_parent_dirs(&new_path, 0, 0).await?; let mut ino: i64 = 1; for comp in parent_info.path.split('/').filter(|s| !s.is_empty()) { if let Some(s) = FileSystem::lookup(&self.delta, ino, comp).await? { @@ -1246,12 +1266,15 @@ impl FileSystem for OverlayFS { // Remove whiteout at destination self.remove_whiteout(&new_path).await?; - self.ensure_parent_dirs(&new_path, 0, 0).await?; - // Get delta destination parent + // Resolve the delta destination parent inode. When the parent is already in the + // delta layer, we can use it directly - all its ancestors are guaranteed to exist + // in delta. This avoids the expensive ensure_parent_dirs() walk. let delta_dst_parent_ino = if new_parent_info.layer == Layer::Delta { new_parent_info.underlying_ino } else { + // Destination parent is in base layer - create the directory hierarchy in delta first. + self.ensure_parent_dirs(&new_path, 0, 0).await?; let mut ino: i64 = 1; for comp in new_parent_info.path.split('/').filter(|s| !s.is_empty()) { if let Some(s) = FileSystem::lookup(&self.delta, ino, comp).await? { From 29cd8b7528267d2b266c347a8ebb4407d202f388 Mon Sep 17 00:00:00 2001 From: Pekka Enberg Date: Tue, 3 Feb 2026 15:31:59 +0200 Subject: [PATCH 2/2] sdk/rust: Skip base lookup for pure delta directories When looking up a child in a directory that was created purely in delta (not copied from base), skip the base layer lookup entirely. Such directories can't have children in base, so the stat() syscall is wasted. This is detected by checking if the parent's delta inode has an origin mapping - if not, it was created in delta and never existed in base. Common case: npm creates node_modules/ in delta, then creates thousands of packages inside it. Previously each lookup would fall through to HostFS even though nothing could possibly exist there. --- sdk/rust/src/filesystem/overlayfs.rs | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/sdk/rust/src/filesystem/overlayfs.rs b/sdk/rust/src/filesystem/overlayfs.rs index 43e6c941..089e6e3c 100644 --- a/sdk/rust/src/filesystem/overlayfs.rs +++ b/sdk/rust/src/filesystem/overlayfs.rs @@ -626,7 +626,18 @@ impl FileSystem for OverlayFS { return Ok(Some(stats)); } - // Try base + // If the parent is a "pure delta" directory (created in delta, not copied from base), + // there's no point checking the base layer - children can't exist there. This avoids + // unnecessary stat() syscalls when looking up entries in directories like node_modules/ + // that were created entirely within the delta layer. + if let Some(delta_parent) = delta_parent_ino { + if self.get_origin_ino(delta_parent).is_none() { + // Parent exists in delta with no base origin - it's a pure delta directory + return Ok(None); + } + } + + // Try base - parent either exists in base or was copied from base let base_parent_ino = if parent_info.layer == Layer::Base { parent_info.underlying_ino } else {