Replace generation bundle chains with content-addressed pack storage

.agents/skills/testing-git-cache-runtime/SKILL.md

@@ -39,7 +39,7 @@ Create a local config file with:
 - `allowed_upstream_hosts = ["github.com"]`
 - `[disk] min_free_bytes = 0` for small local test fixtures
 - `[git_remote] enabled = true` if testing HTTP git routes
-- `[compaction] chain_depth_threshold = 2` when you need a three-generation chain to compact quickly
+- `[compaction] chain_depth_threshold = 2` when you need a three-pack generation head to compact quickly (the threshold counts packs referenced by the head generation manifest)
 - `[compaction] inline = false` unless inline compaction itself is the feature under test
 
 Set `GIT_CACHE_CONFIG=/path/to/config.toml` for both CLI and API commands.
@@ -64,7 +64,7 @@ Set `GIT_CACHE_CONFIG=/path/to/config.toml` for both CLI and API commands.
 4. Inspect object-store JSON before compaction. For `github.com/acme/repo`, useful paths are:
 
    - `objects/repos/github.com/acme/repo/generations/<generation>/manifest.json`
-   - `objects/repos/github.com/acme/repo/generations/<generation>/base.bundle`
+   - `objects/repos/github.com/acme/repo/packs/pack-<sha256>.pack` (listed in each generation manifest's `packs` array)
    - `objects/repos/github.com/acme/repo/manifests/generation-head.json`
    - `objects/repos/github.com/acme/repo/manifests/refs/heads/main.json`
    - `objects/repos/github.com/acme/repo/manifests/refs/heads/default.json`
@@ -75,19 +75,19 @@ Set `GIT_CACHE_CONFIG=/path/to/config.toml` for both CLI and API commands.
    target/debug/git-cache compact --repo github.com/acme/repo --dry-run
    ```
 
-   Assert the report has `old_chain_depth: 3` and the generation head still points to the pre-compaction head.
+   Assert the report has `old_pack_count: 3` and the generation head still points to the pre-compaction head.
 
 6. Run real compaction:
 
    ```bash
    target/debug/git-cache compact --repo github.com/acme/repo
    ```
 
-   Assert the report has `old_chain_depth: 3`, three `old_generations`, a non-empty `new_generation`, and `bytes_reclaimed > 0`.
+   Assert the report has `old_pack_count: 3`, three `old_generations`, a non-empty `new_generation`, and `bytes_reclaimed > 0`.
 
-7. Assert each old generation from the report no longer has either `manifest.json` or `base.bundle` in the object store.
+7. Assert each old generation from the report no longer has a `manifest.json`, and that pack keys referenced only by old generations were deleted from `packs/`.
 
-8. Assert the new generation manifest has `parent_generation: null` and contains exactly commits `[A, B, C]`.
+8. Assert the new generation manifest has a single entry in `packs`, a non-null `verified_at`, and contains exactly commits `[A, B, C]`.
 
 9. Assert branch ref manifests, including `refs/heads/default` when applicable, point to the new compacted generation.
 

crates/git-cache-api/src/lib.rs

@@ -1643,7 +1643,6 @@ mod tests {
             git_remote: Default::default(),
             compaction: Default::default(),
             max_concurrent_git_processes: git_cache_core::default_max_concurrent_git_processes(),
-            max_concurrent_generation_verifications: 1,
             async_materialize_concurrency: git_cache_core::default_async_materialize_concurrency(),
             use_gitoxide: true,
         };

crates/git-cache-api/tests/correctness.rs

@@ -110,7 +110,7 @@ mod tests {
         }
 
         fn object_store_root(&self) -> PathBuf {
-            self.tmp.path().join("objects-v2")
+            self.tmp.path().join("objects-v3")
         }
     }
 

crates/git-cache-api/tests/runtime_cache_recovery.rs

@@ -88,7 +88,7 @@ mod tests {
         }
 
         fn object_store_root(&self) -> PathBuf {
-            self.tmp.path().join("objects-v2")
+            self.tmp.path().join("objects-v3")
         }
 
         fn head_commit(&self) -> String {
@@ -129,19 +129,13 @@ mod tests {
 
         let second = server.commit_and_push("second");
         let second_manifest = materialize_branch_and_wait(&server, &client, &second).await;
-        assert_generation_parent(
-            &server,
-            &second_manifest.generation,
-            Some(&first_manifest.generation),
-        );
+        assert!(second_manifest.generation != first_manifest.generation);
+        assert_generation_pack_count(&server, &second_manifest.generation, 2);
 
         let third = server.commit_and_push("third");
         let third_manifest = materialize_branch_and_wait(&server, &client, &third).await;
-        assert_generation_parent(
-            &server,
-            &third_manifest.generation,
-            Some(&second_manifest.generation),
-        );
+        assert!(third_manifest.generation != second_manifest.generation);
+        assert_generation_pack_count(&server, &third_manifest.generation, 3);
 
         fs::remove_dir_all(server.cache_repo_dir()).unwrap();
 
@@ -249,9 +243,13 @@ mod tests {
             if let Ok(raw) = fs::read_to_string(&path) {
                 let json: Value = serde_json::from_str(&raw).unwrap();
                 let generation = json["generation"].as_str().unwrap().to_string();
-                let verified = server.object_store_root().join(format!(
-                    "repos/{REPO}/generations/{generation}/verified.json"
+                let manifest_path = server.object_store_root().join(format!(
+                    "repos/{REPO}/generations/{generation}/manifest.json"
                 ));
+                let verified = fs::read_to_string(&manifest_path)
+                    .ok()
+                    .and_then(|raw| serde_json::from_str::<Value>(&raw).ok())
+                    .is_some_and(|json| json["verified_at"].is_string());
                 let generation_head = server
                     .object_store_root()
                     .join(format!("repos/{REPO}/manifests/generation-head.json"));
@@ -261,7 +259,7 @@ mod tests {
                     .and_then(|json| json["generation"].as_str().map(str::to_string))
                     .as_deref()
                     == Some(generation.as_str());
-                if verified.exists() && head_matches {
+                if verified && head_matches {
                     return CommitManifest { generation };
                 }
             }
@@ -283,20 +281,16 @@ mod tests {
         ))
     }
 
-    fn assert_generation_parent(
-        server: &TestServer,
-        generation: &str,
-        expected_parent: Option<&str>,
-    ) {
+    fn assert_generation_pack_count(server: &TestServer, generation: &str, expected: usize) {
         let path = server.object_store_root().join(format!(
             "repos/{REPO}/generations/{generation}/manifest.json"
         ));
         let raw = fs::read_to_string(&path).unwrap();
         let json: Value = serde_json::from_str(&raw).unwrap();
         assert_eq!(
-            json["parent_generation"].as_str(),
-            expected_parent,
-            "unexpected parent for generation {generation}"
+            json["packs"].as_array().map(Vec::len),
+            Some(expected),
+            "unexpected pack count for generation {generation}"
         );
     }
 

crates/git-cache-api/tests/support/mod.rs

@@ -37,7 +37,6 @@ pub fn test_config_with_upstream(
         },
         compaction: Default::default(),
         max_concurrent_git_processes: git_cache_core::default_max_concurrent_git_processes(),
-        max_concurrent_generation_verifications: 1,
         async_materialize_concurrency: git_cache_core::default_async_materialize_concurrency(),
         use_gitoxide: true,
     }

crates/git-cache-core/src/config.rs

@@ -30,8 +30,6 @@ pub struct AppConfig {
     pub compaction: CompactionConfig,
     #[serde(default = "default_max_concurrent_git_processes")]
     pub max_concurrent_git_processes: usize,
-    #[serde(default = "default_max_concurrent_generation_verifications")]
-    pub max_concurrent_generation_verifications: usize,
     #[serde(default = "default_async_materialize_concurrency")]
     pub async_materialize_concurrency: usize,
     /// Use in-process gitoxide for local read-only Git operations instead of
@@ -118,10 +116,6 @@ impl AppConfig {
                 "GIT_CACHE_MAX_CONCURRENT_GIT_PROCESSES",
                 default_max_concurrent_git_processes(),
             )?,
-            max_concurrent_generation_verifications: parse_env(
-                "GIT_CACHE_MAX_CONCURRENT_GENERATION_VERIFICATIONS",
-                default_max_concurrent_generation_verifications(),
-            )?,
             async_materialize_concurrency: parse_env(
                 "GIT_CACHE_ASYNC_MATERIALIZE_CONCURRENCY",
                 default_async_materialize_concurrency(),
@@ -269,10 +263,6 @@ pub fn default_max_concurrent_git_processes() -> usize {
     64
 }
 
-pub fn default_max_concurrent_generation_verifications() -> usize {
-    1
-}
-
 pub fn default_async_materialize_concurrency() -> usize {
     2
 }
@@ -358,7 +348,6 @@ mod tests {
         "GIT_CACHE_COMPACTION_CHAIN_DEPTH_THRESHOLD",
         "GIT_CACHE_COMPACTION_INLINE",
         "GIT_CACHE_MAX_CONCURRENT_GIT_PROCESSES",
-        "GIT_CACHE_MAX_CONCURRENT_GENERATION_VERIFICATIONS",
         "GIT_CACHE_ASYNC_MATERIALIZE_CONCURRENCY",
     ];
 
@@ -453,7 +442,6 @@ min_free_bytes = 100000
         assert_eq!(config.max_git_output_bytes, 16 * 1024 * 1024);
         assert_eq!(config.git_remote, GitRemoteConfig::default());
         assert_eq!(config.compaction, CompactionConfig::default());
-        assert_eq!(config.max_concurrent_generation_verifications, 1);
         assert_eq!(config.async_materialize_concurrency, 2);
     }
 
@@ -502,7 +490,6 @@ min_free_bytes = 100000
             ("GIT_CACHE_GIT_REMOTE_PROXY_ON_MISS_BY_DEFAULT", "off"),
             ("GIT_CACHE_COMPACTION_CHAIN_DEPTH_THRESHOLD", "4"),
             ("GIT_CACHE_COMPACTION_INLINE", "yes"),
-            ("GIT_CACHE_MAX_CONCURRENT_GENERATION_VERIFICATIONS", "3"),
         ]);
 
         let config = AppConfig::from_env().unwrap();
@@ -518,7 +505,6 @@ min_free_bytes = 100000
         assert!(!config.git_remote.proxy_on_miss_by_default);
         assert_eq!(config.compaction.chain_depth_threshold, 4);
         assert!(config.compaction.inline);
-        assert_eq!(config.max_concurrent_generation_verifications, 3);
 
         match config.object_store {
             ObjectStoreConfig::S3 {

crates/git-cache-core/src/lib.rs

@@ -16,8 +16,8 @@ pub use config::{
 };
 pub use error::{GitCacheError, Result};
 pub use manifest::{
-    CommitManifest, GenerationId, GenerationManifest, RefManifest, RepoGenerationHead,
-    VerifiedGenerationManifest,
+    CommitManifest, GenerationId, GenerationManifest, PackInfo, PackKind, RefManifest,
+    RepoGenerationHead,
 };
 pub use repo::{CommitSha, RepoKey, ShortCommitSha};
 pub use selector::{BranchName, Selector};

crates/git-cache-core/src/manifest.rs

@@ -25,37 +25,36 @@ impl fmt::Display for GenerationId {
     }
 }
 
+#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
+#[serde(rename_all = "lowercase")]
+pub enum PackKind {
+    Base,
+    Delta,
+}
+
 #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
-pub struct GenerationManifest {
-    pub repo: RepoKey,
-    pub generation: GenerationId,
-    pub bundle_key: String,
-    #[serde(default)]
-    pub parent_generation: Option<GenerationId>,
-    pub created_at: DateTime<Utc>,
-    #[serde(default)]
-    pub commits: Vec<CommitSha>,
+pub struct PackInfo {
+    pub key: String,
+    pub len: u64,
+    pub sha256: String,
+    pub kind: PackKind,
 }
 
 #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
-pub struct VerifiedGenerationManifest {
-    pub schema_version: u32,
+pub struct GenerationManifest {
     pub repo: RepoKey,
     pub generation: GenerationId,
-    pub bundle_key: String,
-    pub bundle_len: u64,
-    pub bundle_sha256: String,
-    #[serde(default)]
-    pub parent_generation: Option<GenerationId>,
     pub created_at: DateTime<Utc>,
-    pub verified_at: DateTime<Utc>,
-    pub verifier_version: u32,
-    pub git_version: String,
-    pub fsck_mode: String,
     #[serde(default)]
-    pub commits: Vec<CommitSha>,
+    pub verified_at: Option<DateTime<Utc>>,
     #[serde(default)]
-    pub tip_commits: Vec<CommitSha>,
+    pub packs: Vec<PackInfo>,
+    #[serde(default)]
+    pub refs: std::collections::BTreeMap<String, CommitSha>,
+    #[serde(default)]
+    pub head_ref: Option<String>,
+    #[serde(default)]
+    pub commits: Vec<CommitSha>,
 }
 
 #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]

crates/git-cache-core/tests/correctness.rs

@@ -498,7 +498,7 @@ mod tests {
 
     #[test]
     fn generation_manifest_missing_repo_field() {
-        let json = r#"{"generation":"550e8400-e29b-41d4-a716-446655440000","bundle_key":"k","created_at":"2026-01-01T00:00:00Z"}"#;
+        let json = r#"{"generation":"550e8400-e29b-41d4-a716-446655440000","created_at":"2026-01-01T00:00:00Z"}"#;
         assert!(serde_json::from_str::<GenerationManifest>(json).is_err());
     }
 

crates/git-cache-domain/src/materializer.rs

@@ -4,20 +4,18 @@ use bytes::Bytes;
 use chrono::Utc;
 use git_cache_core::{
     BranchName, CommitManifest, CommitSha, GenerationId, GenerationManifest, GitCacheError,
-    MaterializeRequest, MaterializeResponse, MaterializeSource, RefManifest, RepoGenerationHead,
-    RepoKey, ResolveResponse, Result as CoreResult, Selector, ShortCommitSha, UpstreamAuth,
-    VerifiedGenerationManifest,
+    MaterializeRequest, MaterializeResponse, MaterializeSource, PackInfo, PackKind, RefManifest,
+    RepoGenerationHead, RepoKey, ResolveResponse, Result as CoreResult, Selector, ShortCommitSha,
+    UpstreamAuth,
 };
 use git_cache_core::{UpdateExecutor, UpdateRequest, UpdateTarget};
 use git_cache_disk::RepoLock;
 pub use git_cache_git::UploadPackProcess;
 use git_cache_objectstore::{
-    generation_manifest_key, pending_generation_publish_key, read_commit_manifest,
-    read_generation_manifest, read_json, read_pending_generation_publish,
-    read_repo_generation_head, read_verified_generation_manifest, verified_generation_manifest_key,
-    write_commit_manifest, write_json, write_ref_manifest, write_repo_generation_head,
-    write_verified_generation_manifest_if_absent_or_matches, GenerationPublish,
-    PendingGenerationPublish, PublishManifests,
+    generation_manifest_key, generation_manifest_prefix, pack_key, read_commit_manifest,
+    read_generation_manifest, read_json, read_repo_generation_head, write_commit_manifest,
+    write_json, write_ref_manifest, write_repo_generation_head, GenerationPublish,
+    PublishManifests,
 };
 use serde::Serialize;
 use std::collections::{HashMap, HashSet};
@@ -41,7 +39,7 @@ pub use direct_git::{
     frame_ref_advertisement, synthesize_ref_advertisement, UpstreamRefComparison,
 };
 pub use executor::MaterializerExecutor;
-pub use generations::{bundle_key, default_manifest_key, CompactionReport};
+pub use generations::{default_manifest_key, CompactionReport};
 pub use repo::repo_from_git_path;
 
 #[derive(Clone)]