From 59c86db1f60fb0539d36324db70d6a26ce4c9350 Mon Sep 17 00:00:00 2001 From: Martin Zihlmann Date: Mon, 6 Jul 2026 08:54:48 +0100 Subject: [PATCH 1/2] add single build cache key collision test --- .../dockerfiles/Dockerfile_test_issue_mz873 | 14 ++++++++++++++ integration/images.go | 2 ++ 2 files changed, 16 insertions(+) create mode 100644 integration/dockerfiles/Dockerfile_test_issue_mz873 diff --git a/integration/dockerfiles/Dockerfile_test_issue_mz873 b/integration/dockerfiles/Dockerfile_test_issue_mz873 new file mode 100644 index 000000000..67c6999f2 --- /dev/null +++ b/integration/dockerfiles/Dockerfile_test_issue_mz873 @@ -0,0 +1,14 @@ +# mz873: the composite cache key joined its parts with '-', so distinct part +# sequences can join to the same text and collide. Stage one's single RUN +# spells out the joined key text of stage two's two RUNs behind a '#' shell +# comment. Serves swapped layers for /one and /two on v1.28.0. +FROM busybox AS one +RUN echo a > /out #-|1-PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin-RUN echo b >> /out + +FROM busybox AS two +RUN echo a > /out # +RUN echo b >> /out + +FROM busybox +COPY --from=one /out /one +COPY --from=two /out /two diff --git a/integration/images.go b/integration/images.go index d562468ee..b4b3b36ac 100644 --- a/integration/images.go +++ b/integration/images.go @@ -199,6 +199,7 @@ var additionalKanikoFlagsMap = map[string][]string{ "Dockerfile_test_cache_copy_oci": {"--cache-copy-layers=true"}, "Dockerfile_test_issue_add": {"--cache-copy-layers=true"}, "Dockerfile_test_issue_mz655": {"--cache-copy-layers=true"}, + "Dockerfile_test_issue_mz873": {"--reproducible"}, "Dockerfile_test_issue_mz774": {"--cache-copy-layers=true"}, "Dockerfile_test_volume_3": {"--skip-unused-stages=false"}, "Dockerfile_test_multistage": {"--skip-unused-stages=false"}, @@ -485,6 +486,7 @@ func NewDockerFileBuilder() *DockerFileBuilder { "Dockerfile_test_issue_mz774": {}, "Dockerfile_test_issue_mz775": {}, "Dockerfile_test_issue_mz782": {}, + "Dockerfile_test_issue_mz873": {}, "Dockerfile_test_issue_mz793": {}, } d.TestOCICacheDockerfiles = map[string]struct{}{ From f0d91d69e64af10a4f0df155e9f9d2f74608f88f Mon Sep 17 00:00:00 2001 From: Martin Zihlmann Date: Sun, 5 Jul 2026 22:25:03 +0100 Subject: [PATCH 2/2] hash composite cache keys recursively behind FF_KANIKO_ROLLING_CACHE_KEY --- README.md | 9 ++++++ integration/images.go | 1 + pkg/executor/build.go | 10 +++---- pkg/executor/composite_cache.go | 53 ++++++++++++++++++++++++++++----- 4 files changed, 60 insertions(+), 13 deletions(-) diff --git a/README.md b/README.md index 70064046c..230940817 100644 --- a/README.md +++ b/README.md @@ -138,6 +138,7 @@ expect - see [Known Issues](#known-issues). - [Flag `FF_KANIKO_BUILDKIT_ARG_ENV_PRECEDENCE`](#flag-ff_kaniko_buildkit_arg_env_precedence) - [Flag `FF_KANIKO_INFER_CROSS_STAGE_CACHE_KEY`](#flag-ff_kaniko_infer_cross_stage_cache_key) - [Flag `FF_KANIKO_CACHE_LOOKAHEAD`](#flag-ff_kaniko_cache_lookahead) + - [Flag `FF_KANIKO_ROLLING_CACHE_KEY`](#flag-ff_kaniko_rolling_cache_key) - [Flag `FF_KANIKO_CACHE_PROBE_AFTER_MISS`](#flag-ff_kaniko_cache_probe_after_miss) - [Flag `FF_KANIKO_WARMER_CACHE_LOCK`](#flag-ff_kaniko_warmer_cache_lock) - [Flag `FF_KANIKO_PRESERVE_MOUNTED_PATHS`](#flag-ff_kaniko_preserve_mounted_paths) @@ -1241,6 +1242,14 @@ Becomes default in `v1.29.0`. Set this flag to `true` to run a precompute pass before the build loop that derives each stage's final cache key ahead of time. The build loop still recomputes each key during its own `optimize()` call and asserts that it matches the precomputed value. This is a developer assertion to verify the new precompute pass is correct, there is no benefit to enabling it in production. Defaults to `false`. +#### Flag `FF_KANIKO_ROLLING_CACHE_KEY` + +By default the composite cache joins all inputs with `-` and hashes the result. Since `-` is a legal input too, different sequences can join to the same text and collide, silently picking up the wrong cache layer. +This flag switches to a rolling hash, think git, where `state = SHA256(state || input)`. This makes boundaries unambiguous and prevents collisions. It also prevents the hash input from growing, so key computation gets marginally cheaper. Toggling it changes every cache key, forcing a rebuild from scratch. +Set this flag to `true` to enable the rolling cache key. +Defaults to `false`. +Becomes default in `v1.29.0`. + #### Flag `FF_KANIKO_CACHE_PROBE_AFTER_MISS` By default, a single layer cache miss within a stage disables every subsequent cache lookup in that same stage. A 30-step Dockerfile with one transient miss at step 3 will rebuild the remaining 27 layers locally even when later layers are still cached in the registry. diff --git a/integration/images.go b/integration/images.go index b4b3b36ac..49aef7b13 100644 --- a/integration/images.go +++ b/integration/images.go @@ -110,6 +110,7 @@ var KanikoEnv = []string{ "FF_KANIKO_CACHE_LOOKAHEAD=1", "FF_KANIKO_SCOPED_DOCKERIGNORE=1", "FF_KANIKO_RESOLVE_CACHE_KEY=1", + "FF_KANIKO_ROLLING_CACHE_KEY=1", "FF_KANIKO_UNTAR_SKIP_ROOT=1", "FF_KANIKO_REPRODUCIBLE_PRESERVE_BASE_LAYERS=1", "KANIKO_PRINT_PLAN=1", diff --git a/pkg/executor/build.go b/pkg/executor/build.go index e428535b4..071a3947b 100644 --- a/pkg/executor/build.go +++ b/pkg/executor/build.go @@ -285,7 +285,7 @@ func redirectCacheKey(inferredKey CompositeCache, layerCache cache.LayerCache) ( if !ok { return nil, fmt.Errorf("failed resolving cache pointer") } - return NewCompositeCache(rawKey), nil + return ResumeCompositeCache(rawKey), nil } func (s *stageBuilder) optimize(compositeKeyPtr *CompositeCache, cfg v1.Config, args *dockerfile.BuildArgs, opts *config.KanikoOptions, fileContext util.FileContext, layerCache cache.LayerCache, stageFinalCacheKeys map[int]string, externalImageDigests map[string]string, hasContext bool) (string, *stageCacheInfo, v1.Config, error) { @@ -624,8 +624,8 @@ func (s *stageBuilder) build(compositeKey CompositeCache, opts *config.KanikoOpt // subsequent optimize pass can find the content key and continue // the cache chain without unpacking the source stage. if inferredCacheKey != "" && inferredCacheKey != ck { - rawKey := compositeKey.Key() - h, err := NewCompositeCache(rawKey).Hash() + rawKey := compositeKey.State() + h, err := ResumeCompositeCache(rawKey).Hash() if err != nil { return err } @@ -1106,7 +1106,7 @@ func DoBuild(opts *config.KanikoOptions) (image v1.Image, retErr error) { var compositeKey *CompositeCache if stage.BaseImageStoredLocally { if cacheKey, ok := stageFinalCacheKeys[stage.BaseImageIndex]; ok { - compositeKey = NewCompositeCache(cacheKey) + compositeKey = ResumeCompositeCache(cacheKey) } } else { compositeKey = NewCompositeCache(sb.baseImageDigest) @@ -1218,7 +1218,7 @@ func DoBuild(opts *config.KanikoOptions) (image v1.Image, retErr error) { var compositeKey *CompositeCache if stage.BaseImageStoredLocally { if cacheKey, ok := stageFinalCacheKeys[stage.BaseImageIndex]; ok { - compositeKey = NewCompositeCache(cacheKey) + compositeKey = ResumeCompositeCache(cacheKey) } } if compositeKey == nil { diff --git a/pkg/executor/composite_cache.go b/pkg/executor/composite_cache.go index 0d22257a4..d0fa56c42 100644 --- a/pkg/executor/composite_cache.go +++ b/pkg/executor/composite_cache.go @@ -25,29 +25,49 @@ import ( "path/filepath" "strings" + "github.com/osscontainertools/kaniko/pkg/config" "github.com/osscontainertools/kaniko/pkg/util" ) +const emptyState = "0000000000000000000000000000000000000000000000000000000000000000" + // NewCompositeCache returns an initialized composite cache object. func NewCompositeCache(initial ...string) *CompositeCache { - c := CompositeCache{ - keys: initial, - } + c := CompositeCache{} + c.AddKey(initial...) return &c } +func ResumeCompositeCache(state string) *CompositeCache { + if !config.EnvBool("FF_KANIKO_ROLLING_CACHE_KEY") { + return NewCompositeCache(state) + } + return &CompositeCache{state: state, keys: []string{state}} +} + // CompositeCache is a type that generates a cache key from a series of keys. type CompositeCache struct { - keys []string + state string + keys []string } // Clone returns an independent copy of the CompositeCache with its own backing array. func (s CompositeCache) Clone() CompositeCache { - return CompositeCache{keys: append([]string(nil), s.keys...)} + return CompositeCache{state: s.state, keys: append([]string(nil), s.keys...)} } // AddKey adds the specified key to the sequence. func (s *CompositeCache) AddKey(k ...string) { + if config.EnvBool("FF_KANIKO_ROLLING_CACHE_KEY") { + for _, key := range k { + state := s.state + if state == "" { + state = emptyState + } + digest := sha256.Sum256([]byte(state + key)) + s.state = hex.EncodeToString(digest[:]) + } + } s.keys = append(s.keys, k...) } @@ -56,9 +76,26 @@ func (s *CompositeCache) Key() string { return strings.Join(s.keys, "-") } +// State returns the representation that ResumeCompositeCache resumes from. +func (s *CompositeCache) State() string { + if !config.EnvBool("FF_KANIKO_ROLLING_CACHE_KEY") { + return s.Key() + } + if s.state == "" { + return emptyState + } + return s.state +} + // Hash returns the composite key in a string SHA256 format. func (s *CompositeCache) Hash() (string, error) { - return util.SHA256(strings.NewReader(s.Key())) + if !config.EnvBool("FF_KANIKO_ROLLING_CACHE_KEY") { + return util.SHA256(strings.NewReader(s.Key())) + } + if s.state == "" { + return emptyState, nil + } + return s.state, nil } func (s *CompositeCache) AddPath(p string, context util.FileContext) error { @@ -77,7 +114,7 @@ func (s *CompositeCache) AddPath(p string, context util.FileContext) error { // Only add the hash of this directory to the key // if there is any ignored content. if !empty || !context.ExcludesFile(p) { - s.keys = append(s.keys, k) + s.AddKey(k) } return nil } @@ -93,7 +130,7 @@ func (s *CompositeCache) AddPath(p string, context util.FileContext) error { return err } - s.keys = append(s.keys, hex.EncodeToString(sha.Sum(nil))) + s.AddKey(hex.EncodeToString(sha.Sum(nil))) return nil }