Proof of concept: task eviction after snapshot for turbo-tasks-backend

crates/next-api/src/project.rs

@@ -625,17 +625,21 @@
                     .start_watching_with_invalidation_reason(watch.poll_interval)
                     .await?;
             } else {
-                project_fs.invalidate_with_reason(|path| invalidation::Initialize {
-                    // this path is just used for display purposes
-                    path: RcStr::from(path.to_string_lossy()),
-                });
+                project_fs
+                    .invalidate_with_reason(|path| invalidation::Initialize {
+                        // this path is just used for display purposes
+                        path: RcStr::from(path.to_string_lossy()),
+                    })
+                    .await?;
             }
             let output_fs = output_fs_operation(project)
                 .read_strongly_consistent()
                 .await?;
-            output_fs.invalidate_with_reason(|path| invalidation::Initialize {
-                path: RcStr::from(path.to_string_lossy()),
-            });
+            output_fs
+                .invalidate_with_reason(|path| invalidation::Initialize {
+                    path: RcStr::from(path.to_string_lossy()),
+                })
+                .await?;
             Ok(())
         }
         .instrument(span_clone)
@@ -770,16 +774,20 @@
                         .start_watching_with_invalidation_reason(watch.poll_interval)
                         .await?;
                 } else {
-                    project_fs.invalidate_with_reason(|path| invalidation::Initialize {
-                        // this path is just used for display purposes
-                        path: RcStr::from(path.to_string_lossy()),
-                    });
+                    project_fs
+                        .invalidate_with_reason(|path| invalidation::Initialize {
+                            // this path is just used for display purposes
+                            path: RcStr::from(path.to_string_lossy()),
+                        })
+                        .await?;
                 }
             }
             if !ReadRef::ptr_eq(&prev_output_fs, &output_fs) {
-                prev_output_fs.invalidate_with_reason(|path| invalidation::Initialize {
-                    path: RcStr::from(path.to_string_lossy()),
-                });
+                prev_output_fs
+                    .invalidate_with_reason(|path| invalidation::Initialize {
+                        path: RcStr::from(path.to_string_lossy()),
+                    })
+                    .await?;
             }
 
             Ok(())
@@ -1495,7 +1503,7 @@
 
     /// Computes the whole app module graph without dropping issues.
     ///
     /// Use this instead of [`whole_app_module_graphs`] when you need to collect issues from the
     /// computation (e.g. for the `get_compilation_issues` MCP tool).
     #[turbo_tasks::function]
     pub async fn whole_app_module_graphs_without_dropping_issues(

crates/next-napi-bindings/src/next_api/project.rs

@@ -1247,15 +1247,19 @@ async fn invalidate_deferred_entry_source_dirs_after_callback(
 
     if paths_to_invalidate.is_empty() {
         // Fallback to full invalidation when app dir paths are unavailable.
-        project_fs.invalidate_with_reason(|path| invalidation::Initialize {
-            path: RcStr::from(path.to_string_lossy()),
-        });
-    } else {
-        project_fs.invalidate_path_and_children_with_reason(paths_to_invalidate, |path| {
-            invalidation::Initialize {
+        project_fs
+            .invalidate_with_reason(|path| invalidation::Initialize {
                 path: RcStr::from(path.to_string_lossy()),
-            }
-        });
+            })
+            .await?;
+    } else {
+        project_fs
+            .invalidate_path_and_children_with_reason(paths_to_invalidate, |path| {
+                invalidation::Initialize {
+                    path: RcStr::from(path.to_string_lossy()),
+                }
+            })
+            .await?;
     }
 
     Ok(())

crates/next-napi-bindings/src/next_api/turbopack_ctx.rs

@@ -249,6 +249,8 @@ pub fn create_turbo_tasks(
                 }),
                 dependency_tracking,
                 num_workers: Some(tokio::runtime::Handle::current().metrics().num_workers()),
+                evict_after_snapshot: std::env::var("TURBO_ENGINE_EVICT_AFTER_SNAPSHOT")
+                    .is_ok_and(|v| v == "1" || v == "true"),
                 ..Default::default()
             },
             Either::Left(backing_storage),

test/e2e/filesystem-cache/evict-after-snapshot.test.ts

@@ -0,0 +1,91 @@
+import { nextTestSetup, isNextDev } from 'e2e-utils'
+import { retry, waitFor } from 'next-test-utils'
+
+// Eviction requires the dev server (HMR) and persistent caching (Turbopack).
+// Skip entirely in prod/start mode.
+;(isNextDev ? describe : describe.skip)('evict-after-snapshot', () => {
+  const envVars = [
+    'ENABLE_CACHING=1',
+    'TURBO_ENGINE_IGNORE_DIRTY=1',
+    'TURBO_ENGINE_SNAPSHOT_IDLE_TIMEOUT_MILLIS=1000',
+    'TURBO_ENGINE_EVICT_AFTER_SNAPSHOT=1',
+  ].join(' ')
+
+  const { skipped, next } = nextTestSetup({
+    files: __dirname,
+    skipDeployment: true,
+    packageJson: {
+      scripts: {
+        dev: `${envVars} next dev`,
+      },
+    },
+    installCommand: 'npm i',
+    startCommand: 'npm run dev',
+  })
+
+  if (skipped) {
+    return
+  }
+
+  async function waitForSnapshotAndEviction() {
+    // The idle timeout is 1s, give extra time for snapshot + eviction to complete
+    await waitFor(5000)
+  }
+
+  // Turbopack-only: eviction requires persistent caching
+  ;(process.env.IS_TURBOPACK_TEST ? it : it.skip)(
+    'should serve correct content after eviction and HMR',
+    async () => {
+      const browser = await next.browser('/')
+      await retry(async () => {
+        expect(await browser.elementByCss('p').text()).toBe('hello world')
+      })
+
+      let currentContent = 'hello world'
+      for (let cycle = 1; cycle <= 3; cycle++) {
+        await waitForSnapshotAndEviction()
+
+        const prevContent = currentContent
+        const nextContent = `cycle ${cycle}`
+        await next.patchFile('app/page.tsx', (content) =>
+          content.replace(prevContent, nextContent)
+        )
+        currentContent = nextContent
+
+        const expected = currentContent
+        await retry(async () => {
+          expect(await browser.elementByCss('p').text()).toBe(expected)
+        }, 10000)
+      }
+
+      await browser.close()
+    },
+    90000
+  )
+  ;(process.env.IS_TURBOPACK_TEST ? it : it.skip)(
+    'should handle client component HMR after eviction',
+    async () => {
+      const browser = await next.browser('/client')
+      await retry(async () => {
+        expect(await browser.elementByCss('p').text()).toBe('hello world')
+      })
+
+      await waitForSnapshotAndEviction()
+
+      await next.patchFile(
+        'app/client/page.tsx',
+        (content) => content.replace('hello world', 'hello eviction'),
+        async () => {
+          await retry(async () => {
+            expect(await browser.elementByCss('p').text()).toBe(
+              'hello eviction'
+            )
+          }, 10000)
+        }
+      )
+
+      await browser.close()
+    },
+    90000
+  )
+})

turbopack/crates/turbo-persistence/src/db.rs

@@ -619,8 +619,7 @@ impl<S: ParallelScheduler, const FAMILIES: usize> TurboPersistence<S, FAMILIES>
 
     /// Clears all caches of the database.
     pub fn clear_cache(&self) {
-        self.key_block_cache.clear();
-        self.value_block_cache.clear();
+        self.clear_block_caches();
         for meta in self.inner.write().meta_files.iter_mut() {
             meta.clear_cache();
         }

turbopack/crates/turbo-tasks-auto-hash-map/src/set.rs

@@ -97,6 +97,14 @@ impl<K: Hash + Eq, H: BuildHasher + Default, const I: usize> AutoSet<K, H, I> {
     pub fn contains(&self, key: &K) -> bool {
         self.map.contains_key(key)
     }
+
+    /// see [HashSet::retain](https://doc.rust-lang.org/std/collections/hash_set/struct.HashSet.html#method.retain)
+    pub fn retain<F>(&mut self, mut f: F)
+    where
+        F: FnMut(&K) -> bool,
+    {
+        self.map.retain(|k, _| f(k));
+    }
 }
 
 impl<K, H, const I: usize> AutoSet<K, H, I> {

turbopack/crates/turbo-tasks-backend/Cargo.toml

@@ -60,6 +60,7 @@ turbo-persistence = { workspace = true }
 turbo-rcstr = { workspace = true }
 turbo-tasks = { workspace = true }
 turbo-tasks-hash = { workspace = true }
+turbo-tasks-malloc = { workspace = true, default-features = false }
 thread_local = { workspace = true }
 
 [dev-dependencies]
@@ -68,6 +69,7 @@ futures = { workspace = true }
 indoc = { workspace = true }
 regex = { workspace = true }
 tempfile = { workspace = true }
+triomphe = { workspace = true }
 turbo-tasks-malloc = { workspace = true }
 rstest = { workspace = true }
 turbo-tasks-testing = { workspace = true }