diff --git a/.claude/commands/review-roundup.md b/.claude/commands/review-roundup.md new file mode 100644 index 000000000..792602d78 --- /dev/null +++ b/.claude/commands/review-roundup.md @@ -0,0 +1,24 @@ +--- +model: sonnet +effort: medium +--- + +Categorize open non-draft PRs into a Slack-ready review-nudge message, prioritizing user-reported bugs. Usage: `/review-roundup` + +1. List PRs: + ```bash + gh pr list --state open --draft=false --limit 100 \ + --json number,title,reviewRequests,additions,deletions,createdAt \ + --jq '.[] | "\(.number)\t+\(.additions)/-\(.deletions)\t[\(.reviewRequests|map(.login // .name)|join(","))]\t\(.title)"' + ``` +2. Linked issue per PR: `gh pr view --json body --jq '.body[:600]'` β†’ `Closes`/`Fixes #N`. +3. Issue author+labels: `gh issue view --json author,title,labels --jq '.author.login + " [" + (.labels|map(.name)|join(",")) + "] " + .title'` + - `mzihlmann` is the maintainer; his issues (typically `fuzz`/internal) are NOT user-reported and never go in πŸ”₯/πŸ™‹, whatever the labels. + - Multi-issue PR = user-reported only if β‰₯1 linked issue has an external author. All-`mzihlmann` β†’ internal (πŸ›/🧹). Check every linked issue. + - A PR continuing a user's contribution (PR body cites a prior user PR) credits that contributor (`continues @user's #N`) and rises out of plain 🧹. +4. Output the Slack message in a code block for the WYSIWYG composer: `*bold*` headers (single asterisk, not `**`), `- ` bullets, `[#N](url)` links. PR url: `https://github.com/osscontainertools/kaniko/pull/`. No `---` separators, no prose around the block. + - Header: `πŸ“‹ *Open PRs needing review*`. No reviewer @mentions β€” pasted mentions don't notify and highlight inconsistently. + - Groups in priority order, drop empty: πŸ”₯ Top priority (user bug `bug`/`regression`) Β· πŸ™‹ User-requested feature (user `enhancement`) Β· πŸ› Crash fixes (fuzz/internal) Β· ⚑ Caching Β· ♻️ Reproducible builds Β· 🧹 Maintenance (dead code, dep bumps, deprecation flags). + - Line: `- [#](url) β€” β€” closes # by @ \`+A/-B\``. `by @` only when user-reported. `⚠️ no reviewers` when `reviewRequests` empty. Tag `easy review` for low-risk PRs (dead code, coverage). + - Never warn `large`. The `+A/-B` shown is the reviewable diff: exclude `vendor/`, `go.mod`/`go.sum`/`modules.txt`, and `golden/`/`testdata/` snapshots. When raw total is inflated, show the code count and note `(rest vendored)` / `(rest golden snapshots)`. Compute via: `gh pr view --json files --jq '[.files[]|select(.path|test("^vendor/|go\\.(mod|sum)|modules\\.txt|golden/|testdata/")|not)]|"+\(map(.additions)|add)/-\(map(.deletions)|add)"'`. +5. If any PR lacks reviewers, flag and ask before assigning the shared set. Never assign without confirmation. diff --git a/cmd/executor/cmd/bake.go b/cmd/executor/cmd/bake.go new file mode 100644 index 000000000..5298d9bf2 --- /dev/null +++ b/cmd/executor/cmd/bake.go @@ -0,0 +1,99 @@ +/* +Copyright 2026 OSS Container Tools + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package cmd + +import ( + "fmt" + "strings" + + "github.com/osscontainertools/kaniko/pkg/bake" + "github.com/osscontainertools/kaniko/pkg/config" + "github.com/osscontainertools/kaniko/pkg/logging" + "github.com/spf13/cobra" +) + +var bakeSet []string + +func init() { + AddBakeFlags(bakeCmd, opts, &bakeSet) + addHiddenFlags(bakeCmd) + RootCmd.AddCommand(bakeCmd) +} + +func AddBakeFlags(cmd *cobra.Command, opts *config.KanikoOptions, set *[]string) { + AddSharedBuildFlags(cmd, opts) + cmd.Flags().StringArrayVar(set, "set", nil, "Override a bakefile target field: .=. Set it repeatedly for multiple overrides.") +} + +func ConfigureFromBakefile(opts *config.KanikoOptions, path string, selection, set []string) error { + bakefile, err := bake.Parse(path) + if err != nil { + return err + } + targets, err := bakefile.Resolve(selection) + if err != nil { + return err + } + overrides := make([]bake.Override, 0, len(set)) + for _, s := range set { + o, err := bake.ParseOverride(s) + if err != nil { + return err + } + overrides = append(overrides, o) + } + if err := bake.ApplyOverrides(targets, overrides); err != nil { + return err + } + if len(targets) != 1 { + ids := make([]string, len(targets)) + for i, t := range targets { + ids[i] = t.ID + } + return fmt.Errorf("bakefile defines multiple targets, name one to build: %s", strings.Join(ids, ", ")) + } + target := targets[0] + if !opts.NoPush && len(target.Destination) == 0 { + return fmt.Errorf("target %q has no destination, set one in the bakefile or use --no-push", target.ID) + } + opts.Target = []string{target.Stage} + for _, d := range target.Destination { + if err := opts.Destinations.Set(d); err != nil { + return err + } + } + return nil +} + +var bakeCmd = &cobra.Command{ + Use: "bake [target]", + Short: "Build a target defined in a JSON bakefile", + Long: `Build a target defined in a JSON bakefile. The bakefile may define several +targets; name the one to build (it may be omitted when there is only one). The +target's stage and push destination come from the bakefile. Context, dockerfile, +build args and other settings come from the usual flags.`, + Args: cobra.RangeArgs(1, 2), + RunE: func(_ *cobra.Command, args []string) error { + if err := logging.Configure(logLevel, logFormat, logTimestamp); err != nil { + return err + } + if err := ConfigureFromBakefile(opts, args[0], args[1:], bakeSet); err != nil { + return err + } + return runBuild(opts) + }, +} diff --git a/cmd/executor/cmd/root.go b/cmd/executor/cmd/root.go index 46b4696e2..1ee4ed150 100644 --- a/cmd/executor/cmd/root.go +++ b/cmd/executor/cmd/root.go @@ -114,158 +114,179 @@ func ValidateFlags(opts *config.KanikoOptions) { } } +func setupBuild() error { + ValidateFlags(opts) + + // mz661: resolveSecrets must run before moveKanikoDir so that secret src= + // paths pointing into the original kaniko dir are still valid when copied. + if err := resolveSecrets(); err != nil { + return fmt.Errorf("error resolving secrets: %w", err) + } + + // Command line flag takes precedence over the KANIKO_DIR environment variable. + dir := config.KanikoDir + if opts.KanikoDir != "" { + dir = opts.KanikoDir + } + + if dir != config.KanikoExeDir { + err := moveKanikoDir(config.KanikoExeDir, dir) + if err != nil { + return err + } + } + + resolveEnvironmentBuildArgs(opts.BuildArgs, os.Getenv) + + if err := cacheFlagsValid(); err != nil { + return fmt.Errorf("cache flags invalid: %w", err) + } + if err := resolveSourceContext(); err != nil { + return fmt.Errorf("error resolving source context: %w", err) + } + if err := resolveDockerfilePath(); err != nil { + return fmt.Errorf("error resolving dockerfile path: %w", err) + } + // Update ignored paths + if opts.IgnoreVarRun { + // /var/run is a special case. It's common to mount in /var/run/docker.sock + // or something similar which leads to a special mount on the /var/run/docker.sock + // file itself, but the directory to exist in the image with no way to tell if it came + // from the base image or not. + logrus.Trace("Adding /var/run to default ignore list") + util.AddToDefaultIgnoreList(util.IgnoreListEntry{ + Path: "/var/run", + PrefixMatchOnly: false, + }) + } + for _, p := range opts.IgnorePaths { + util.AddToDefaultIgnoreList(util.IgnoreListEntry{ + Path: p, + PrefixMatchOnly: false, + }) + } + return nil +} + // RootCmd is the kaniko command that is run var RootCmd = &cobra.Command{ Use: "executor", PersistentPreRunE: func(cmd *cobra.Command, args []string) error { if cmd.Use == "executor" { - if err := logging.Configure(logLevel, logFormat, logTimestamp); err != nil { return err } - - ValidateFlags(opts) - - // mz661: resolveSecrets must run before moveKanikoDir so that secret src= - // paths pointing into the original kaniko dir are still valid when copied. - if err := resolveSecrets(); err != nil { - return fmt.Errorf("error resolving secrets: %w", err) - } - - // Command line flag takes precedence over the KANIKO_DIR environment variable. - dir := config.KanikoDir - if opts.KanikoDir != "" { - dir = opts.KanikoDir - } - - if dir != config.KanikoExeDir { - err := moveKanikoDir(config.KanikoExeDir, dir) - if err != nil { - return err - } - } - - resolveEnvironmentBuildArgs(opts.BuildArgs, os.Getenv) - if !opts.NoPush && len(opts.Destinations) == 0 { return errors.New("you must provide --destination, or use --no-push") } - if err := cacheFlagsValid(); err != nil { - return fmt.Errorf("cache flags invalid: %w", err) - } - if err := resolveSourceContext(); err != nil { - return fmt.Errorf("error resolving source context: %w", err) - } - if err := resolveDockerfilePath(); err != nil { - return fmt.Errorf("error resolving dockerfile path: %w", err) - } if len(opts.Destinations) == 0 && opts.ImageNameDigestFile != "" { return errors.New("you must provide --destination if setting ImageNameDigestFile") } if len(opts.Destinations) == 0 && opts.ImageNameTagDigestFile != "" { return errors.New("you must provide --destination if setting ImageNameTagDigestFile") } - // Update ignored paths - if opts.IgnoreVarRun { - // /var/run is a special case. It's common to mount in /var/run/docker.sock - // or something similar which leads to a special mount on the /var/run/docker.sock - // file itself, but the directory to exist in the image with no way to tell if it came - // from the base image or not. - logrus.Trace("Adding /var/run to default ignore list") - util.AddToDefaultIgnoreList(util.IgnoreListEntry{ - Path: "/var/run", - PrefixMatchOnly: false, - }) - } - for _, p := range opts.IgnorePaths { - util.AddToDefaultIgnoreList(util.IgnoreListEntry{ - Path: p, - PrefixMatchOnly: false, - }) - } } return nil }, Run: func(cmd *cobra.Command, args []string) { - if !checkContained() { - if !force { - exit(errors.New("kaniko should only be run inside of a container, run with the --force flag if you are sure you want to continue")) - } - logrus.Warn("Kaniko is being run outside of a container. This can have dangerous effects on your system") - } - if !opts.NoPush || opts.CacheRepo != "" { - if err := executor.CheckPushPermissions(opts); err != nil { - logrus.Warnf("make sure you entered the correct tag name, that you are authenticated correctly, and try again.") - // mz280: remind users that DOCKER_AUTH_CONFIG gets prioritized by docker-cli - // https://github.com/docker/cli/pull/6171 - _, ok := os.LookupEnv("DOCKER_AUTH_CONFIG") - if ok { - logrus.Warnf("note that your DOCKER_AUTH_CONFIG env variable can shadow credentials from configfile") - logrus.Warnf("see https://github.com/osscontainertools/kaniko/issues/280#issuecomment-3498449955") - } - exit(fmt.Errorf("error checking push permissions: %w", err)) - } - } - if err := resolveRelativePaths(); err != nil { - exit(fmt.Errorf("error resolving relative paths to absolute paths: %w", err)) - } - if err := os.Chdir("/"); err != nil { - exit(fmt.Errorf("error changing to root dir: %w", err)) - } - if opts.Cleanup && config.EnvBoolDefault("FF_KANIKO_CLEAN_KANIKO_DIR", true) { - defer func() { - if err := config.Cleanup(); err != nil { - logrus.Warnf("error cleaning kaniko dir: %v", err) - } - }() - } - image, err := executor.DoBuild(opts) - if err != nil { - exit(fmt.Errorf("error building image: %w", err)) - } - if err := executor.DoPush(image, opts); err != nil { - exit(fmt.Errorf("error pushing image: %w", err)) + if err := runBuild(opts); err != nil { + exit(err) } + }, +} - benchmarkFile := os.Getenv("BENCHMARK_FILE") - // false is a keyword for integration tests to turn off benchmarking - if benchmarkFile != "" && benchmarkFile != "false" { - s, err := timing.JSON() - if err != nil { - logrus.Warnf("Unable to write benchmark file: %s", err) - return +func runBuild(opts *config.KanikoOptions) error { + if err := setupBuild(); err != nil { + return err + } + if !checkContained() { + if !force { + return errors.New("kaniko should only be run inside of a container, run with the --force flag if you are sure you want to continue") + } + logrus.Warn("Kaniko is being run outside of a container. This can have dangerous effects on your system") + } + if !opts.NoPush || opts.CacheRepo != "" { + if err := executor.CheckPushPermissions(opts); err != nil { + logrus.Warnf("make sure you entered the correct tag name, that you are authenticated correctly, and try again.") + // mz280: remind users that DOCKER_AUTH_CONFIG gets prioritized by docker-cli + // https://github.com/docker/cli/pull/6171 + _, ok := os.LookupEnv("DOCKER_AUTH_CONFIG") + if ok { + logrus.Warnf("note that your DOCKER_AUTH_CONFIG env variable can shadow credentials from configfile") + logrus.Warnf("see https://github.com/osscontainertools/kaniko/issues/280#issuecomment-3498449955") } - if strings.HasPrefix(benchmarkFile, "gs://") { - logrus.Info("Uploading to gcs") - if err := buildcontext.UploadToBucket(strings.NewReader(s), benchmarkFile); err != nil { - logrus.Infof("Unable to upload %s due to %v", benchmarkFile, err) - } - logrus.Infof("Benchmark file written at %s", benchmarkFile) - } else { - f, err := os.Create(benchmarkFile) - if err != nil { - logrus.Warnf("Unable to create benchmarking file %s: %s", benchmarkFile, err) - return - } - defer f.Close() - _, err = f.WriteString(s) - if err != nil { - logrus.Warnf("Unable to write to benchmarking file %s: %s", benchmarkFile, err) - return - } - logrus.Infof("Benchmark file written at %s", benchmarkFile) + return fmt.Errorf("error checking push permissions: %w", err) + } + } + if err := resolveRelativePaths(); err != nil { + return fmt.Errorf("error resolving relative paths to absolute paths: %w", err) + } + if err := os.Chdir("/"); err != nil { + return fmt.Errorf("error changing to root dir: %w", err) + } + if opts.Cleanup && config.EnvBoolDefault("FF_KANIKO_CLEAN_KANIKO_DIR", true) { + defer func() { + if err := config.Cleanup(); err != nil { + logrus.Warnf("error cleaning kaniko dir: %v", err) } + }() + } + image, err := executor.DoBuild(opts) + if err != nil { + return fmt.Errorf("error building image: %w", err) + } + if err := executor.DoPush(image, opts); err != nil { + return fmt.Errorf("error pushing image: %w", err) + } + writeBenchmark() + return nil +} + +func writeBenchmark() { + benchmarkFile := os.Getenv("BENCHMARK_FILE") + // false is a keyword for integration tests to turn off benchmarking + if benchmarkFile == "" || benchmarkFile == "false" { + return + } + s, err := timing.JSON() + if err != nil { + logrus.Warnf("Unable to write benchmark file: %s", err) + return + } + if strings.HasPrefix(benchmarkFile, "gs://") { + logrus.Info("Uploading to gcs") + if err := buildcontext.UploadToBucket(strings.NewReader(s), benchmarkFile); err != nil { + logrus.Infof("Unable to upload %s due to %v", benchmarkFile, err) } - }, + logrus.Infof("Benchmark file written at %s", benchmarkFile) + return + } + f, err := os.Create(benchmarkFile) + if err != nil { + logrus.Warnf("Unable to create benchmarking file %s: %s", benchmarkFile, err) + return + } + defer f.Close() + _, err = f.WriteString(s) + if err != nil { + logrus.Warnf("Unable to write to benchmarking file %s: %s", benchmarkFile, err) + return + } + logrus.Infof("Benchmark file written at %s", benchmarkFile) } // addKanikoOptionsFlags configures opts func AddKanikoOptionsFlags(cmd *cobra.Command, opts *config.KanikoOptions) { + AddSharedBuildFlags(cmd, opts) + cmd.Flags().VarP(&opts.Destinations, "destination", "d", "Registry the final image should be pushed to. Set it repeatedly for multiple destinations.") + cmd.Flags().StringSliceVarP(&opts.Target, "target", "", []string{}, "Set the target stages to build, the first in the list denotes the stage to be pushed") +} + +func AddSharedBuildFlags(cmd *cobra.Command, opts *config.KanikoOptions) { cmd.Flags().StringVarP(&opts.DockerfilePath, "dockerfile", "f", "Dockerfile", "Path to the dockerfile to be built.") cmd.Flags().StringVarP(&opts.SrcContext, "context", "c", "/workspace/", "Path to the dockerfile build context.") cmd.Flags().StringVarP(&ctxSubPath, "context-sub-path", "", "", "Sub path within the given context.") cmd.Flags().StringVarP(&opts.Bucket, "bucket", "b", "", "Name of the GCS bucket from which to access build context as tarball.") - cmd.Flags().VarP(&opts.Destinations, "destination", "d", "Registry the final image should be pushed to. Set it repeatedly for multiple destinations.") cmd.Flags().StringVarP(&opts.SnapshotMode, "snapshot-mode", "", "full", "Change the file attributes inspected during snapshotting") cmd.Flags().StringVarP(&opts.CustomPlatform, "custom-platform", "", "", "Specify the build platform if different from the current host") cmd.Flags().VarP(&opts.BuildArgs, "build-arg", "", "This flag allows you to pass in ARG values at build time. Set it repeatedly for multiple values.") @@ -274,7 +295,6 @@ func AddKanikoOptionsFlags(cmd *cobra.Command, opts *config.KanikoOptions) { cmd.Flags().StringVarP(&opts.TarPath, "tar-path", "", "", "Path to save the image in as a tarball instead of pushing") cmd.Flags().BoolVarP(&opts.SingleSnapshot, "single-snapshot", "", false, "Take a single snapshot at the end of the build.") cmd.Flags().BoolVarP(&opts.Reproducible, "reproducible", "", false, "Strip timestamps out of the image to make it reproducible") - cmd.Flags().StringSliceVarP(&opts.Target, "target", "", []string{}, "Set the target stages to build, the first in the list denotes the stage to be pushed") cmd.Flags().BoolVarP(&opts.NoPush, "no-push", "", false, "Do not push the image to the registry") cmd.Flags().BoolVarP(&opts.NoPushCache, "no-push-cache", "", false, "Do not push the cache layers to the registry") cmd.Flags().StringVarP(&opts.CacheRepo, "cache-repo", "", "", "Specify a repository to use as a cache, otherwise one will be inferred from the destination provided; when prefixed with 'oci:' the repository will be written in OCI image layout format at the path provided") diff --git a/docs/design_proposals/bug-2594-hardlinks-lost-in-copy-from.md b/docs/design_proposals/bug-2594-hardlinks-lost-in-copy-from.md new file mode 100644 index 000000000..b13aa8476 --- /dev/null +++ b/docs/design_proposals/bug-2594-hardlinks-lost-in-copy-from.md @@ -0,0 +1,115 @@ +# Bug #2594: Hardlinks lost during COPY --from + +## Status +Confirmed bug present in `main`. + +## Summary + +`COPY --from=` silently breaks hardlinks. Files that shared an inode in the +source stage become independent regular files in the output image, inflating image size. +Docker/BuildKit preserves hardlinks correctly. + +Real-world impact: `bitnami/git` has 141 hardlinks under `/opt/bitnami/git/libexec/git-core/`, +causing kaniko to produce a 720 MB image vs Docker's 83 MB. + +## Reproduction + +`integration/dockerfiles/Dockerfile_test_issue_2594` β€” self-contained reproducer. +Run with: `DOCKERFILE_TEST_FILTER=Dockerfile_test_issue_2594 bash scripts/integration-test.sh` + +## Root Cause + +### The break point: `CopyDir` in `pkg/util/fs_util.go:673` + +`CopyDir` classifies each file as one of: + +``` +file == "." β†’ MkdirAll +fi.IsDir() β†’ MkdirAll +IsSymlink() β†’ CopySymlink +else β†’ CopyFile ← hardlinks land here +``` + +`os.Lstat` returns hardlinks as regular files β€” there is no mode bit distinguishing them. +`CopyFile` (`fs_util.go:813`) opens the source and writes a new independent file, giving +it its own inode. The hardlink relationship is destroyed. + +### Why the snapshot pass can't recover it + +`checkHardlink` in `tar_util.go:194` detects hardlinks at snapshot time via `Nlink > 1`. +But `CopyFile` already created each file with a fresh inode (`Nlink == 1`), so +`checkHardlink` finds nothing and writes every file as a regular tar entry. + +### Why Docker works + +BuildKit processes `COPY --from` through tar streams end-to-end. `tar.TypeLink` entries +from the source layer are reproduced verbatim in the output β€” inodes are never +materialised, so the relationship is never lost. + +### What already works + +`ExtractFile` in `fs_util.go:376` correctly handles `tar.TypeLink` when extracting source +images to the inter-stage directory (`os.Link` is called for each hardlink entry). The +problem is only in the subsequent `CopyDir` step. + +## Fix + +### One change: add inode tracking to `CopyDir` + +Add a `map[uint64]string` (inode β†’ first dest path) before the file loop β€” the same +pattern `Tar.hardlinks` uses in `tar_util.go:40`. + +In the `else` branch (`fs_util.go:732`), before calling `CopyFile`, extract +`syscall.Stat_t` via `getSyscallStatT` (already in `tar_util.go:214`, same `util` +package, no move needed): + +```go +// first occurrence of this inode: copy normally, record dest +// later occurrences: create a hardlink to the first dest, skip CopyFile +hardlinksSeen := make(map[uint64]string) +``` + +```go +} else { + if linked, linkDst := checkCopyHardlink(fi, destPath, hardlinksSeen); linked { + if err := os.Link(linkDst, destPath); err != nil { + return nil, err + } + } else { + if _, err := CopyFile(fullPath, destPath, context, uid, gid, mode, useDefaultChmod); err != nil { + return nil, err + } + } +} +``` + +```go +// checkCopyHardlink returns (true, existingDest) for the second and later occurrences +// of an inode. On first occurrence it records inodeβ†’dest and returns (false, ""). +func checkCopyHardlink(fi os.FileInfo, dest string, seen map[uint64]string) (bool, string) { + stat := getSyscallStatT(fi) + if stat == nil || stat.Nlink <= 1 { + return false, "" + } + if existing, ok := seen[stat.Ino]; ok { + return true, existing + } + seen[stat.Ino] = dest + return false, "" +} +``` + +### No other changes needed + +- `ExtractFile` β€” already correct. +- `Tar.checkHardlink` β€” will automatically detect the now-preserved hardlinks + (`Nlink > 1` again on the filesystem) and write correct `tar.TypeLink` entries. +- `CopyFile` β€” no change needed. + +## Affected locations + +| File | Lines | Change | +|------|-------|--------| +| `pkg/util/fs_util.go` | 673–754 | `CopyDir` β€” add `hardlinksSeen` map and hardlink branch | +| `pkg/util/fs_util.go` | β€” | add `checkCopyHardlink` helper | +| `pkg/util/tar_util.go` | 194–221 | `checkHardlink`, `getSyscallStatT` β€” no change | diff --git a/docs/design_proposals/bug-677-alpine-tls-auth-test-coverage.md b/docs/design_proposals/bug-677-alpine-tls-auth-test-coverage.md new file mode 100644 index 000000000..d9f119024 --- /dev/null +++ b/docs/design_proposals/bug-677-alpine-tls-auth-test-coverage.md @@ -0,0 +1,171 @@ +# Bug #677: kaniko-alpine missing DOCKER_CONFIG and SSL_CERT_DIR β€” test coverage plan + +* Author: Martin Zihlmann +* Date: 2026-05-05 +* Status: Under implementation + +## Background + +`kaniko-alpine` (introduced in #647) uses a bare `FROM alpine` stage that does not inherit from `kaniko-base-slim` or `kaniko-base`. Those base stages set `ENV DOCKER_CONFIG=/kaniko/.docker/` and `ENV SSL_CERT_DIR=/kaniko/ssl/certs` and copy the CA bundle to `/kaniko/ssl/certs/`. The alpine stage sets neither, which causes two independent failures for users of that image: + +**DOCKER_CONFIG not set.** go-containerregistry resolves push credentials by looking up `DOCKER_CONFIG`. When unset it falls back to `~/.docker/config.json`, never finding credentials written to `/kaniko/.docker/config.json`. Pushes fail with `UNAUTHORIZED`. Pulls work because the base image is public. + +**SSL_CERT_DIR not set + certs wiped by KANIKO_PRE_CLEANUP.** `KANIKO_PRE_CLEANUP=1` calls `util.DeleteFilesystem()` before the first build stage. That function preserves everything under `/kaniko/` (the hardcoded ignore-list entry) but wipes `/etc/ssl/certs/`. The CA bundle installed by `apk add ca-certificates` lives at `/etc/ssl/certs/ca-certificates.crt` and is therefore gone before the first TLS connection. Pulling any base image from Docker Hub (or any HTTPS registry) then fails with `x509: certificate signed by unknown authority`. + +**Why the current integration tests did not catch this.** `buildKanikoImage` calls `addServiceAccountFlags` for every executor image. That function always mounts the host docker config at `/root/.docker/config.json` and passes `-e DOCKER_CONFIG=/root/.docker`, overriding the image's own (missing) env var. The local test mode uses `localhost:5000` (plain HTTP) for push, so TLS is never required for the outbound push. Together these two shortcuts masked both bugs completely. + +## Fix to deploy/Dockerfile + +The fix to the alpine stage is straightforward and not the focus of this document: + +- `COPY --from=builder /kaniko/.docker /kaniko/.docker` +- `COPY --from=certs /etc/ssl/certs/ca-certificates.crt /kaniko/ssl/certs/` +- Add `DOCKER_CONFIG=/kaniko/.docker/`, `DOCKER_CREDENTIAL_GCR_CONFIG=...`, and `SSL_CERT_DIR=/kaniko/ssl/certs` to the ENV block + +The rest of this document is the test plan. + +## Integration test plan + +The goal is a test that fails if either env var is removed from the alpine image. That requires two things the current test infrastructure does not have: + +1. A registry that rejects unauthenticated pushes, to exercise DOCKER_CONFIG. +2. A registry that requires TLS, to exercise SSL_CERT_DIR. + +A single local `registry:2` instance with both basic auth and TLS covers both in one. + +### Secure local registry + +Add `start_local_secure_registry` to `scripts/integration-test.sh`. It is started alongside the existing plain registry when `LOCAL=1` and listens on port 5001. + +The function needs a TLS cert/key pair and an htpasswd file. Both are generated by a small Go helper (see below) and written to temp paths before the docker run. + +```bash +function start_local_secure_registry { + docker start secure-registry 2>/dev/null || docker run -d --name secure-registry \ + -p 5001:5000 \ + -v "${SECURE_REGISTRY_CERT}":/certs/tls.crt:ro \ + -v "${SECURE_REGISTRY_KEY}":/certs/tls.key:ro \ + -v "${SECURE_REGISTRY_HTPASSWD}":/auth/htpasswd:ro \ + -e REGISTRY_HTTP_TLS_CERTIFICATE=/certs/tls.crt \ + -e REGISTRY_HTTP_TLS_KEY=/certs/tls.key \ + -e REGISTRY_AUTH=htpasswd \ + -e REGISTRY_AUTH_HTPASSWD_REALM="Test Registry" \ + -e REGISTRY_AUTH_HTPASSWD_PATH=/auth/htpasswd \ + registry:2 +} +``` + +`SECURE_REGISTRY_CERT`, `SECURE_REGISTRY_KEY`, and `SECURE_REGISTRY_HTPASSWD` are set by the Go setup helper before the test binary starts (see `TestMain` below). + +### TLS cert generation + +Use Go's `crypto/x509` + `crypto/rsa` in `integration/integration_test.go`'s `TestMain`. No openssl binary needed. + +The cert must have `SubjectAltName` covering `127.0.0.1` and `localhost` so Go's TLS verifier accepts it. The generated PEM files are written to `os.MkdirTemp` paths and exported to env vars so `integration-test.sh` can pick them up, and so the cert path can be mounted into the kaniko container. + +```go +func generateSelfSignedCert(dir string) (certPath, keyPath string, err error) { + key, _ := rsa.GenerateKey(rand.Reader, 2048) + tmpl := &x509.Certificate{ + SerialNumber: big.NewInt(1), + Subject: pkix.Name{CommonName: "localhost"}, + NotBefore: time.Now().Add(-time.Hour), + NotAfter: time.Now().Add(10 * 365 * 24 * time.Hour), + KeyUsage: x509.KeyUsageKeyEncipherment | x509.KeyUsageCertSign, + ExtKeyUsage: []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth}, + IPAddresses: []net.IP{net.ParseIP("127.0.0.1")}, + DNSNames: []string{"localhost"}, + IsCA: true, + } + certDER, _ := x509.CreateCertificate(rand.Reader, tmpl, tmpl, &key.PublicKey, key) + // write PEM files to dir, return paths +} +``` + +### htpasswd generation + +Use `golang.org/x/crypto/bcrypt` (already vendored) to hash a fixed test password and write a standard htpasswd file. Credentials: `kanikotest` / `kanikotest`. No external `htpasswd` binary needed. + +```go +hash, _ := bcrypt.GenerateFromPassword([]byte("kanikotest"), bcrypt.MinCost) +htpasswd := fmt.Sprintf("kanikotest:%s\n", hash) +``` + +### docker credentials file + +Generate a `config.json` for `localhost:5001` and write it to a temp file. This file will be mounted into the kaniko container at `/kaniko/.docker/config.json`. + +```go +auth := base64.StdEncoding.EncodeToString([]byte("kanikotest:kanikotest")) +config := fmt.Sprintf(`{"auths":{"localhost:5001":{"auth":%q}}}`, auth) +``` + +### Credential mounting for the alpine executor + +Add `addAlpineServiceAccountFlags` to `integration/images.go`. It differs from `addServiceAccountFlags` in two ways: + +- Credentials are mounted at `/kaniko/.docker/config.json`, not `/root/.docker/config.json`. +- `-e DOCKER_CONFIG` is **not** passed. The test relies on the image's own `ENV DOCKER_CONFIG=/kaniko/.docker/`. + +```go +func addAlpineServiceAccountFlags(flags []string, serviceAccount, dockerConfigPath string) []string { + if len(serviceAccount) > 0 { + flags = append(flags, "-e", + "GOOGLE_APPLICATION_CREDENTIALS=/secret/"+filepath.Base(serviceAccount), + "-v", filepath.Dir(serviceAccount)+":/secret/") + return flags + } + if dockerConfigPath != "" { + flags = append(flags, "-v", dockerConfigPath+":/kaniko/.docker/config.json:ro") + } + return flags +} +``` + +`dockerConfigPath` comes from the `KANIKO_ALPINE_DOCKER_CONFIG` env var set by `TestMain` when running locally, or from the regular host config path when running against GCR. + +### CA cert mounting for the alpine executor + +The self-signed cert for `localhost:5001` must be trusted by the kaniko executor. Mount it as an additional file under `/kaniko/ssl/certs/`, which `SSL_CERT_DIR` already points at after the deploy/Dockerfile fix: + +```go +"-v", certPath+":/kaniko/ssl/certs/test-registry-ca.crt:ro" +``` + +This also validates that `SSL_CERT_DIR=/kaniko/ssl/certs` is set: if it is not, Go's TLS stack will not look in that directory and the self-signed cert will be rejected. The Docker Hub pull of `FROM debian` (which uses the CA bundle also under `/kaniko/ssl/certs/`) provides a second independent TLS check. + +### Test Dockerfile + +`integration/dockerfiles/Dockerfile_test_issue_mz677` is a minimal Dockerfile that exercises push to the secure local registry. It does not need to produce any particular output β€” the test assertion is that the build and push succeed without error. + +```dockerfile +FROM debian +RUN echo "mz677: kaniko-alpine auth+tls smoke test" +``` + +Wire it in `integration/images.go`: + +```go +// executorImages +"Dockerfile_test_issue_mz677": AlpineImage, +``` + +The destination image for this test is `localhost:5001/...` when running locally, falling back to the normal `imageRepo` in CI. + +### What the test exercises + +| Scenario | Failure mode if broken | +|---|---| +| Push to localhost:5001 (basic auth) | `UNAUTHORIZED` β€” catches missing `DOCKER_CONFIG` | +| TLS handshake with localhost:5001 (self-signed cert) | `x509: certificate signed by unknown authority` β€” catches missing `SSL_CERT_DIR` or missing cert in `/kaniko/ssl/certs/` | +| Pull of `FROM debian` from Docker Hub | `x509: certificate signed by unknown authority` β€” second check for `SSL_CERT_DIR` pointing to the CA bundle | + +### Open questions + +**Should the secure registry also be used by existing alpine tests?** + +`Dockerfile_test_issue_mz595` currently pushes to `localhost:5000`. Redirecting it to `localhost:5001` would give existing alpine tests auth+TLS coverage too, but complicates the diffoci comparison step which also needs to pull the result. Probably easiest to leave mz595 on 5000 and add mz677 as a dedicated smoke test. + +**Cert and htpasswd generation in TestMain vs. integration-test.sh.** + +The design above generates files in Go's `TestMain` and exports paths via env vars. An alternative is to do it all in `integration-test.sh` using `openssl` and `docker run --entrypoint htpasswd httpd:2`. The Go approach avoids the external tool dependency and keeps the setup co-located with the test code. Prefer Go. diff --git a/docs/design_proposals/bug-maintainer-nil-command-index-skew.md b/docs/design_proposals/bug-maintainer-nil-command-index-skew.md new file mode 100644 index 000000000..d52959070 --- /dev/null +++ b/docs/design_proposals/bug-maintainer-nil-command-index-skew.md @@ -0,0 +1,133 @@ +# Bug: MAINTAINER instruction causes index skew between stage.Commands and stageBuilder.cmds + +## Status +Confirmed latent bug present in `main`. Triggered by any Dockerfile containing a `MAINTAINER` instruction. + +## Summary + +`GetCommand` in `pkg/commands/commands.go:107` returns `nil, nil` for `MAINTAINER` +(the instruction is deprecated and intentionally skipped). `newStageBuilder` filters these +nil results out, so `stageBuilder.cmds` is shorter than `stage.Commands` whenever a +`MAINTAINER` instruction is present. Any code that indexes into a slice built from +`s.cmds` by position, then reads that slice using a position from `stage.Commands`, will +read the wrong entry or panic with an out-of-bounds access. + +## Concrete impact: dryrun cache annotation output + +`stageCacheInfo` (`build.go:83`) stores per-command cache keys and hit flags. It is +populated in `optimize` using index `i` over `s.cmds`, then consumed in `RenderStages` +using index `jdx` over `s.Commands`. + +Example with `MAINTAINER` at position 0: + +``` +stage.Commands = [MAINTAINER, RUN touch /a, RUN touch /b] len=3 +stageBuilder.cmds = [RUN touch /a, RUN touch /b] len=2 + +ci.cacheKeys[0] = key_for_RUN_touch_a (i=0 in optimize) +ci.cacheKeys[1] = key_for_RUN_touch_b (i=1 in optimize) + +RenderStages: + jdx=0 β†’ MAINTAINER β†’ ci.cacheKeys[0] = key_for_RUN_touch_a ← wrong command + jdx=1 β†’ RUN touch /a β†’ ci.cacheKeys[1] = key_for_RUN_touch_b ← wrong key + jdx=2 β†’ RUN touch /b β†’ ci.cacheKeys[2] ← out-of-bounds panic +``` + +The panic only occurs when `opts.Cache && FF_KANIKO_CACHE_LOOKAHEAD` are both set. +Without the cache-lookahead feature flag the `cacheInfo` slice is all nils and +`RenderStages` never reads it. + +## Root cause + +`newStageBuilder` (`build.go:147`) maps `stage.Commands β†’ s.cmds` with a gap wherever +`GetCommand` returns nil: + +```go +for _, cmd := range stage.Commands { + command, err := commands.GetCommand(cmd, ...) + if command == nil { + continue // ← gap introduced here + } + s.cmds = append(s.cmds, command) +} +``` + +There is no record of which position in `stage.Commands` each `s.cmds[i]` came from, so +callers cannot reconstruct the mapping. + +## Other potentially affected sites + +These sites iterate `stage.Commands` and could diverge from `s.cmds`-indexed data in +future work: + +| File | Location | Risk | +|------|-----------|------| +| `build.go:893` | `RenderStages` β€” `for jdx, c := range s.Commands` | Active: reads `ci.cacheKeys[jdx]` (see above) | +| `build.go:1321` | stage dependency scan β€” `for _, c := range stage.Commands` | Low: reads only, no parallel `s.cmds` index | +| `build.go:1413` | cross-stage dep resolution β€” `for _, c := range stage.Commands` | Low: reads only | + +## Proposed fix + +Track the raw index of each command in `stageBuilder`, then use it when writing into +`stageCacheInfo`. + +### 1. Add mapping fields to `stageBuilder` + +```go +type stageBuilder struct { + ... + cmds []commands.DockerCommand + cmdRawIdxs []int // cmdRawIdxs[i] = index in stage.Commands for s.cmds[i] + numRawCmds int // len(stage.Commands) + ... +} +``` + +### 2. Populate them in `newStageBuilder` + +```go +s.numRawCmds = len(stage.Commands) +for rawIdx, cmd := range stage.Commands { + command, err := commands.GetCommand(cmd, ...) + if command == nil { + continue + } + s.cmds = append(s.cmds, command) + s.cmdRawIdxs = append(s.cmdRawIdxs, rawIdx) +} +``` + +### 3. Size and index `stageCacheInfo` by raw position in `optimize` + +```go +ci := &stageCacheInfo{ + redirectKeys: make([]string, s.numRawCmds), + redirectHits: make([]bool, s.numRawCmds), + cacheKeys: make([]string, s.numRawCmds), + cacheHits: make([]bool, s.numRawCmds), +} +... +for i, command := range s.cmds { + raw := s.cmdRawIdxs[i] + ci.cacheKeys[raw] = ck + ci.redirectKeys[raw] = inferredCK + ci.redirectHits[raw] = true / false + ci.cacheHits[raw] = true / false +} +``` + +`RenderStages` then reads `ci.cacheKeys[jdx]` with `jdx` from `s.Commands` iteration β€” +which now correctly aligns. + +## Test coverage needed + +1. **Golden test with MAINTAINER**: add a Dockerfile variant for an existing golden test + that prepends `MAINTAINER deprecated@example.com` before the first `RUN`. With + `FF_KANIKO_CACHE_LOOKAHEAD=1` the plan output must be identical to the variant without + `MAINTAINER` (the instruction is a no-op). + +2. **Unit test for `newStageBuilder`**: assert that `cmdRawIdxs` is populated correctly + and `numRawCmds` equals `len(stage.Commands)` even when one entry is nil. + +3. **`optimize` unit test**: verify that `stageCacheInfo` entries land at the correct raw + index when `s.cmds` is shorter than `stage.Commands`. diff --git a/docs/design_proposals/bug-oci-layer-annotations-divergence.md b/docs/design_proposals/bug-oci-layer-annotations-divergence.md new file mode 100644 index 000000000..451df4060 --- /dev/null +++ b/docs/design_proposals/bug-oci-layer-annotations-divergence.md @@ -0,0 +1,47 @@ +# Bug: OCI layer descriptor annotations missing from kaniko output + +## Status + +Hotfixed via `--extra-ignore-annotations` in `TestBuildWithAnnotations` and pinning `FROM ubuntu` to Noble in affected test Dockerfiles (#680). Proper fix is open. + +## Summary + +When building from an OCI base image, Docker BuildKit adds `ci.umo.uncompressed_blob_size` annotations to every layer descriptor in the output OCI manifest. Kaniko strips base-image layer annotations via `withoutAnnotations()` and does not add them to layers it creates. The outputs therefore diverge structurally even when the image contents are identical. + +A second related gap: the `--annotation` flag in kaniko does not reliably propagate to OCI manifest annotations when the base image is OCI. Docker BuildKit does propagate them. This was previously masked because the test (`TestBuildWithAnnotations`) compared against Docker's full manifest diff; when both sides had no manifest annotations the diff was empty. + +## Background + +### What triggered it + +`ubuntu:latest` moved to Ubuntu 26.04 "Resolute" around 2026-05-04. Resolute's OCI manifest carries `ci.umo.uncompressed_blob_size` annotations on its layer descriptors. Docker BuildKit propagates these to the output manifest and also adds them to any new layers it creates (e.g. from `RUN` instructions). Kaniko strips them. CI tests comparing Docker vs kaniko output with `diffoci` then failed. + +Noble (ubuntu:24.04) has no layer annotations, so tests that pin to Noble are unaffected by the annotation divergence β€” but `TestBuildWithAnnotations` fetches its Dockerfile from the `main` branch at test time (git-URL build context always resolves to `#main`), so a Dockerfile pin in a PR branch can't take effect until merged. + +### Affected code + +`pkg/executor/build.go` β€” `withoutAnnotations` is called at two points (~line 965 and ~line 1051) to strip annotations from the base image before building. This is correct for preventing stale base-image annotations from leaking into output, but kaniko then never re-adds the annotations that BuildKit would produce for the final layers. + +`pkg/executor/build.go:1138` β€” `mutate.Annotations(sourceImage, opts.Annotations)` applies user-supplied `--annotation` flags. The go-containerregistry `mutate.Annotations` call sets the OCI manifest-level `annotations` field. However, subsequent mutations (e.g. `mutate.AppendLayers`) return a new `v1.Image` that does not carry forward manifest-level annotations, so the annotation may silently disappear depending on call order. + +## Proper fix + +### Layer descriptor annotations + +When kaniko creates or re-emits an OCI manifest, compute `ci.umo.uncompressed_blob_size` for each layer (the uncompressed size is available from `layer.Uncompressed()`) and set it as an annotation on the layer descriptor. This should only apply when the output format is OCI (`application/vnd.oci.image.manifest.v1+json`). + +Alternatively, verify whether `diffoci --extra-ignore-annotations` is the right long-term stance (i.e. treat layer descriptor annotations as noise in comparisons). That depends on whether any tooling relies on `ci.umo.uncompressed_blob_size` for correctness. + +### `--annotation` propagation + +Audit the call order in `pkg/executor/build.go`. The `mutate.Annotations` call must happen **after** all `mutate.AppendLayers` / `mutate.Config` calls, because those calls return a new `image` object that drops manifest-level annotations. The fix is to move the annotation application to the very end of the image construction pipeline, after all layer and config mutations are complete. + +### Test + +Restore the `getImageManifestAnnotations` check that was removed in commit `7675a2db3`. The test should: + +1. Build with `--annotation myannotation=myvalue` via both Docker and kaniko. +2. Fetch the manifest for the kaniko-built image from the registry. +3. Assert `manifest.Annotations["myannotation"] == "myvalue"`. + +This is independent of Docker's output format and will catch regressions without being sensitive to BuildKit-specific OCI extensions. diff --git a/docs/design_proposals/bug-volume-cache-inconsistency.md b/docs/design_proposals/bug-volume-cache-inconsistency.md new file mode 100644 index 000000000..feca5135f --- /dev/null +++ b/docs/design_proposals/bug-volume-cache-inconsistency.md @@ -0,0 +1,40 @@ +# Bug: VOLUME + cache produces inconsistent builds in multistage Dockerfiles + +This is the same root cause as the `WORKDIR` cache bug at https://github.com/GoogleContainerTools/kaniko/issues/3340. + +**Actual behavior** + +When `VOLUME /vol` is declared, kaniko creates the directory implicitly via `os.MkdirAll`. But the directory is not snapshotted (`FilesToSnapshot` returns `[]`), so the timestamp change is never included in a layer. the cache key for the `VOLUME` instruction itself is unaffected, so this goes unnoticed in single stage builds. + +In a **multistage** build the fresh `mtime` on `/vol` is visible to later stages that copy from this stage, causing a guaranteed cache miss on every subsequent run even when nothing in the Dockerfile has changed. + +**Expected behavior** + +The directory timestamp of a `VOLUME`-declared path should be stable across builds so that dependent cache keys are not invalidated. Two consecutive kaniko builds of the same multistage Dockerfile with `--cache` should both get a full cache hit after the first build. + +**To Reproduce** + +```dockerfile +FROM busybox AS base +VOLUME /vol +RUN echo "blubb" > /vol/b + +FROM busybox +COPY --from=base /vol /vol +``` + +1. Build the image with kaniko and `--cache` (first run, cache miss) β€” succeeds. +2. Build the image again with the same `--cache` repo β€” the `COPY --from=base` stage gets a cache miss because `/vol`'s `mtime` changed when `VOLUME` recreated it in step 1. + +**WORKAROUND** + +Explicitly create the directory prior to calling `VOLUME` + +```Dockerfile +RUN mkdir /vol +VOLUME /vol +``` + +**Fix** + +`FF_KANIKO_SKIP_VOLUME_MKDIR=true` avoids the problem entirely by not creating the directory at all, matching Docker/BuildKit behaviour where `VOLUME` only declares a mountpoint in the image config without touching the filesystem, basically enforcing the workaround. diff --git a/docs/design_proposals/bug-warmer-shared-volume-race.md b/docs/design_proposals/bug-warmer-shared-volume-race.md new file mode 100644 index 000000000..b57881fea --- /dev/null +++ b/docs/design_proposals/bug-warmer-shared-volume-race.md @@ -0,0 +1,105 @@ +# Plan: warmer concurrency lock, gated behind `FF_KANIKO_WARMER_CACHE_LOCK` + +## Context + +Issue #364 reports a TOCTOU race in the cache warmer: two warmer processes sharing the same cache volume can both decide an image is missing, both download it, and then race on the final rename-into-place. With the legacy tarball cache the loser silently overwrites the winner (wasted bandwidth). With the new ocilayout cache (`FF_KANIKO_OCI_WARMER`) the loser's `os.Rename` of a directory onto a non-empty destination fails with `ENOTEMPTY` and the warmer exits non-zero β€” the user-visible failure mode that motivated the bug. + +PR #705 (external contribution) proposes a per-digest `flock(2)` on `cacheDir/.warmer-locks/.lock` taken around the recheck + rename step. The idea is correct. Two issues to address before merge: + +1. **No integration test exercises the race.** The existing `TestWarmerTwice` is sequential and (since commit 9f4021ae) gives each subtest its own tmp cache dir, deliberately working around the race. The PR's unit tests cover the lock primitive itself but not the end-to-end behavior with two warmer containers contending on a shared volume. We need a test that fails on `main` without the fix and passes with it. +2. **No feature flag.** The PR argues the fix falls under the "behavior so broken no working workflow could depend on it" exception in `docs/releases.md`. That's defensible for the `ENOTEMPTY` failure under `FF_KANIKO_OCI_WARMER`, but the same code path also changes legacy tarball behavior (silent overwrite β†’ "drop our copy, keep theirs" with a new log line). Per release policy when in doubt, gate it. Picking a flag also makes the integration test easy: run the same test with the FF on (expect success) and the FF being default-off means existing users see no behavior change. + +Outcome: warmer is safe under concurrent shared-volume usage when `FF_KANIKO_WARMER_CACHE_LOCK=true`, with an integration test that proves both directions (race reproduces with FF off in the OCI path; FF on fixes it). + +## Approach + +Three pieces of work, in this order: + +### 1. Integration test: revert 9f4021ae and see if `TestWarmerTwice` reproduces the race naturally + +Commit 9f4021ae gives each `TestWarmerTwice` subtest its own `tmpDir` precisely to dodge this bug. Reverting it puts all 3 subtests back on a shared volume: + +| Subtest | Reference | Final cache key | +|---|---|---| +| 1 | `debian:trixie-slim` | digest A | +| 2 | `debian:12.10@sha256:264982…` (image-index) | digest M (resolved to per-arch manifest via `warm.go:202-244`) | +| 3 | `debian:12.10@sha256:6bc30d…` (image-manifest) | digest M (direct) | + +Subtests 2 and 3 both end up writing to `cacheDir/`, so the revert gives a free 2-way same-digest race; subtest 1 adds cross-digest noise (validates the lock granularity is per-digest, not global). + +**Plan:** + +1. Revert 9f4021ae on `main` (no fix, no FF wiring). +2. Run `TestWarmerTwice` in OCI mode several times (5–10), count `ENOTEMPTY` triggers. +3. Branch on result: + - **β‰₯80% reproduce** β€” revert alone is sufficient. Skip to step 2 (lock impl). + - **50–80% reproduce** β€” add a 4th subtest that targets digest M via yet another reference (e.g. by tag) to widen the race surface. + - **<50% reproduce** β€” write a dedicated `TestWarmerConcurrent` that launches K=4 warmer containers in parallel against the same image via `sync.WaitGroup`. Independent of `t.Parallel` scheduling. + +Once the test reliably reproduces the bug, wire `FF_KANIKO_WARMER_CACHE_LOCK=1` into `WarmerEnv` in `integration/images.go:128` (after the fix lands) so all warmer integration tests exercise the new path. + +### 2. Lock primitive + wiring + +New file `pkg/warmer/lock.go` with `acquireCacheLock(cacheDir, key string) (release func(), err error)`: + +- `os.MkdirAll(cacheDir/.warmer-locks, 0o755)` +- `os.OpenFile(cacheDir/.warmer-locks/.lock, O_RDWR|O_CREATE, 0o644)` +- `unix.Flock(fd, LOCK_EX)` +- Return a `release` closure that flock-unlocks and closes (idempotent). + +This is essentially the PR's primitive. `golang.org/x/sys/unix` is already a transitive dep β€” no new modules. + +Wire into both warm paths in `pkg/warmer/warm.go`: + +- `warmToFile` (legacy tarball, line ~88): after `cw.Warm(...)` succeeds and `finalCachePath` is computed (line 121), gate the lock + recheck + rename block on `config.EnvBool("FF_KANIKO_WARMER_CACHE_LOCK")`. When off: keep existing `os.Rename` for both file and manifest (legacy behavior preserved bit-for-bit). When on: acquire lock, re-stat `finalCachePath`; if present log "Image %v became available in cache while warming; keeping existing copy" and return nil, else rename file then manifest. +- `ociWarmToFile` (line ~139): same shape. Gate the lock + recheck on the FF. The recheck is what prevents the `ENOTEMPTY` even with the lock held β€” the second warmer must skip the rename, not retry it. + +Per `feedback_comments.md`: no new explanatory comments in the wiring sites unless they document a non-obvious invariant. The PR's verbose comments can be trimmed; the FF name + the conditional structure read clearly enough. + +### 3. Documentation + +In `README.md`: + +- Add a ToC entry at the bottom of the feature-flag list (line ~142, after `FF_KANIKO_CACHE_PROBE_AFTER_MISS`). +- Add a section body in the feature-flag descriptions block (after the `FF_KANIKO_CACHE_PROBE_AFTER_MISS` section, line ~1219). Follow the existing pattern: 1 paragraph problem statement, 1 sentence "Set this flag to `true` to ...", `Defaults to false.`, `Becomes default in vX.Y.Z.` once a target minor is picked. +- Update the existing `FF_KANIKO_OCI_WARMER` section (line 1134): replace "Note that currently there is no mutex lock mechanism yet, so it does not support multiple parallel writes." with a reference to the new flag. + +No `docs/releases.md` change needed β€” the policy doc already covers feature-flag lifecycle. + +## Critical files + +- `pkg/warmer/warm.go` β€” add FF-gated lock + recheck blocks in `warmToFile` and `ociWarmToFile` +- `pkg/warmer/lock.go` β€” new, contains `acquireCacheLock` +- `pkg/warmer/lock_test.go` β€” new, unit tests for the primitive (mutual exclusion, different keys don't block, release is idempotent). Adapted from PR #705's tests. +- `integration/integration_test.go` β€” new `TestWarmerConcurrent`; optionally revert the per-subtest tmpDir in `TestWarmerTwice` +- `integration/images.go` β€” add `FF_KANIKO_WARMER_CACHE_LOCK=1` to `WarmerEnv` (line 128) +- `README.md` β€” ToC entry + new section body + edit of the `FF_KANIKO_OCI_WARMER` note + +## Reused functions + +- `config.EnvBool(key string) bool` from `pkg/config/options.go:196` β€” gates the new path, same pattern as the existing `FF_KANIKO_OCI_WARMER` check at `pkg/warmer/warm.go:62`. +- `golang.org/x/sys/unix.Flock` β€” already in vendor (used by go-billy transitively, per PR description). +- Existing `WarmerEnv` array in `integration/images.go:128` β€” the integration test framework already propagates this into `docker run -e` flags for warmer containers (see `integration_test.go:883`). + +## Verification + +**Reproduce the race on main first** (proves the test catches the bug we're fixing): + +1. Check out `main` at e28a9a0db. +2. Add only the new `TestWarmerConcurrent` test (no fix yet), with `FF_KANIKO_OCI_WARMER=1` from `WarmerEnv`. +3. Build warmer image and run the test inside the sandbox. Expect: at least one container fails with `ENOTEMPTY`. + +**Verify the fix:** + +1. Apply lock + wiring + FF. +2. Re-run `TestWarmerConcurrent` with `FF_KANIKO_WARMER_CACHE_LOCK=1`. Expect: all containers exit zero, exactly one `/` directory present, "Image ... became available in cache while warming" appears in at least N-1 of the logs. +3. Re-run `TestWarmerConcurrent` with `FF_KANIKO_WARMER_CACHE_LOCK=0`. Expect: same failure as the main-baseline run, confirming the FF gates the new behavior. +4. Run existing `TestWarmer` and `TestWarmerTwice` with the FF on. Expect: pass (no regression in the sequential paths). + +**Unit-level:** + +- `go test ./pkg/warmer/... -race` β€” passes, including the new `lock_test.go` (mutual-exclusion goroutine test, idempotent release). + +**Build / lint:** + +- Build executor + warmer images inside the sandbox per `feedback_kaniko_sandbox.md` β€” never run `out/executor` or `out/warmer` directly on the host. diff --git a/docs/design_proposals/bug-workdir-missing-from-optimize-precompute.md b/docs/design_proposals/bug-workdir-missing-from-optimize-precompute.md new file mode 100644 index 000000000..57e0a7c3c --- /dev/null +++ b/docs/design_proposals/bug-workdir-missing-from-optimize-precompute.md @@ -0,0 +1,69 @@ +# Bug: WORKDIR not applied during optimize precompute pass + +## Status +Known bug present in `main`. Fix should be done separately from the cache-lookahead work. + +## Summary + +`WorkdirCommand.MetadataOnly()` returns `false` for any path other than `/`, because +WORKDIR has a filesystem side effect (it creates the directory). As a result, `optimize` +never calls `ExecuteCommand` for WORKDIR, so `cfg.WorkingDir` is never updated during +the precompute pass. + +## Impact + +1. **Wrong cache keys for COPY after WORKDIR.** `populateCompositeKey` calls + `command.FilesUsedFromContext(&cfg, s.args)`, which resolves relative COPY source + paths against `cfg.WorkingDir`. If WorkingDir is stale (empty or from the base + image), relative paths resolve incorrectly and the computed cache key differs from + the one produced during the actual build. + +2. **Wrong `stageFinalConfigs` for locally-stored stages.** The config propagated to a + dependent stage via `stageFinalConfigs` carries a stale `WorkingDir`, so any + command in that stage that resolves paths relative to the working directory (e.g. + a subsequent COPY) will use the wrong base path when computing its cache key. + +## Root cause + +`WorkdirCommand.ExecuteCommand` does two things: +1. Resolves and sets `config.WorkingDir` β€” a pure config mutation, safe anywhere. +2. Creates the directory on the filesystem via `mkdirAllWithPermissions` β€” a side + effect that must not run during a precompute pass (the rootfs is not yet unpacked). + +Because both concerns live in the same method, and because `MetadataOnly()` is the +only signal available to `optimize`, WORKDIR is excluded from the precompute entirely. + +## Proposed fix + +Split the two concerns by introducing a new interface method (e.g. `ApplyConfig`) that +performs only the config mutation without any filesystem effects. `optimize` calls +`ApplyConfig` for every command, while the actual build calls `ExecuteCommand` as +before. + +```go +// ApplyConfig updates v1.Config to reflect this command's effect on image metadata +// (e.g. WorkingDir, Env, Labels). It must not touch the filesystem. +// The default implementation in BaseCommand is a no-op. +ApplyConfig(config *v1.Config, buildArgs *dockerfile.BuildArgs) error +``` + +Commands that currently return `MetadataOnly() == true` (ARG, ENV, LABEL, USER, EXPOSE, +VOLUME, CMD, ENTRYPOINT, SHELL, STOPSIGNAL) can delegate `ApplyConfig` to their +existing `ExecuteCommand` since they have no filesystem effects. + +`WorkdirCommand.ApplyConfig` would be: + +```go +func (w *WorkdirCommand) ApplyConfig(config *v1.Config, buildArgs *dockerfile.BuildArgs) error { + replacementEnvs := buildArgs.ReplacementEnvs(config.Env) + resolved, err := util.ResolveEnvironmentReplacement(w.cmd.Path, replacementEnvs, true) + if err != nil { + return err + } + config.WorkingDir = ToAbsPath(resolved, config.WorkingDir) + return nil +} +``` + +`optimize` would then replace the `if command.MetadataOnly()` check with +`command.ApplyConfig(&cfg, s.args)`. diff --git a/docs/design_proposals/ci-git-context-uses-main.md b/docs/design_proposals/ci-git-context-uses-main.md new file mode 100644 index 000000000..e2c3292c9 --- /dev/null +++ b/docs/design_proposals/ci-git-context-uses-main.md @@ -0,0 +1,66 @@ +# CI: git-context tests always fetch from main on PRs + +## Status + +Known issue. No fix in place. + +## Problem + +`getBranchCommitAndURL()` in `integration/integration_test.go` hardcodes `branch = "main"` whenever `GITHUB_HEAD_REF` is set (i.e. whenever the run is triggered by a pull request): + +```go +if _, isPR := os.LookupEnv("GITHUB_HEAD_REF"); isPR { + branch = "main" +} +``` + +Any test that passes this branch to `DockerGitRepo` or `KanikoGitRepo` as the build context will clone `main` β€” not the PR branch β€” during CI. The PR branch's file changes are invisible to those tests until after merge. + +Affected tests (12 at time of writing): + +- `TestBuildWithAnnotations` +- `TestBuildWithLabels` +- `TestBuildWithHTTPError` +- `TestBuildSkipFallback` +- `TestBuildViaRegistryMirrors` +- `TestBuildViaRegistryMap` +- `TestGitBuildcontext` +- `TestGitBuildcontextNoRef` +- `TestGitBuildcontextSubPath` +- `TestGitBuildcontextExplicitCommit` +- `TestKanikoDir` +- `TestExpectError` + +The `TestGitBuildcontext*` family is intentionally testing kaniko's git-context feature and must use a real git URL, so the impact there is different (they test the mechanic, not the Dockerfile content). The others (`TestBuildWithAnnotations`, `TestBuildWithLabels`, etc.) use a git URL purely as a convenience to supply a build context β€” the Dockerfile content is what matters, and it is silently sourced from main. + +This caused a real incident: pinning `Dockerfile.trivial` to `ubuntu:24.04` in PR #680 had no effect on `TestBuildWithAnnotations` during CI because the test kept fetching the un-pinned file from main, requiring `--extra-ignore-annotations` as a workaround. + +## Fix + +For tests that use a git URL only as a build context (not to test the git-context feature itself), replace the git URL with the local checkout. `runtime.Caller(0)` is already used elsewhere in the integration suite to get the local source directory. + +```go +// Before +DockerGitRepo(url, "", branch) // fetches github.com/.../kaniko.git#main +KanikoGitRepo(url, "", branch) // fetches git://github.com/.../kaniko.git#refs/heads/main + +// After +cwd // local path, already used in BuildImage / BuildImageWithContext +``` + +For kaniko the local path would be passed as `-c ` (the directory context flag), which is the same pattern used by the main `TestRun` loop via `buildContextPath`. + +The `TestGitBuildcontext*` tests must keep using a git URL because they are specifically exercising that feature. Those should be updated to use `GITHUB_SHA` (the merge commit) rather than branch, so the exact commit under test is always fetched: + +```go +// GITHUB_SHA for a PR is the ephemeral merge commit GitHub creates, +// which contains the PR branch changes merged into main. +_, commit, url := getBranchCommitAndURL() +KanikoGitRepo(url, commit, "") // fetches the exact merge commit +``` + +## Open questions + +**Should `getBranchCommitAndURL` be removed entirely?** + +Once the non-git-context tests switch to local paths, the function is only needed by the `TestGitBuildcontext*` family. It could be simplified or inlined there. diff --git a/docs/design_proposals/ci-hardening.md b/docs/design_proposals/ci-hardening.md new file mode 100644 index 000000000..10c300027 --- /dev/null +++ b/docs/design_proposals/ci-hardening.md @@ -0,0 +1,22 @@ +# CI supply chain hardening + +* Author: Martin Zihlmann +* Date: 2026-04-26 +* Status: Under implementation + +## Background + +GitHub Actions workflows are a common supply chain attack surface. A compromised action, a floating script download, or overly broad permissions can allow an attacker to exfiltrate secrets, push malicious images, or inject code into the build. This document tracks the remaining hardening work for kaniko's four workflows: `images.yaml`, `integration-tests.yaml`, `unit-tests.yaml`, and `nightly-vulnerability-scan.yml`. + +## Open items + +**1. Switch `integration-tests.yaml` harden-runner from `audit` to `block`** +Needs a main-branch run with harden-runner present to collect the egress domain list. Once that data is available, derive the allowlist and flip the policy. + +**2. Add harden-runner to `coverage-merge` job** (`integration-tests.yaml`) +The `coverage-merge` job does not yet have a `harden-runner` step. + +## Implementation order + +1. Item 1 β€” merge the ci-hardening branch so integration tests run on main with harden-runner in audit mode; collect the domain list from those logs, then switch to `block`. +2. Item 2 can be done independently. diff --git a/docs/design_proposals/cleanup-further-code-elimination.md b/docs/design_proposals/cleanup-further-code-elimination.md new file mode 100644 index 000000000..20dc55c30 --- /dev/null +++ b/docs/design_proposals/cleanup-further-code-elimination.md @@ -0,0 +1,199 @@ +# Cleanup: further code elimination opportunities + +## Status +Survey. Companion to [cleanup-test-only-production-exports.md](cleanup-test-only-production-exports.md), which handled test-only exports and test-fixture files. This doc lists the remaining classes of removable code in rough order of ROI. + +## Background + +After the test-only-exports cleanup, `staticcheck` with its substantive (non-style) checks returns zero hits β€” no unused variables, no unreachable branches, no deprecated stdlib APIs. The codebase is healthy on that axis. The remaining elimination opportunities are at coarser granularities: deprecated CLI flags, vestigial hidden flags, integration scaffolding for code paths CI doesn't exercise, and dependency audit. + +## Overview + +| Class | Approx lines | Risk | Effort | +|---|---|---|---| +| [Deprecated CLI flags](#1-deprecated-cli-flags) | ~50 | Low (major-version breaking) | Mechanical | +| [Hidden but possibly vestigial flags](#2-hidden-flags) | ~10–20 | Need investigation first | Low-medium | +| [GCP integration scaffolding](#3-gcp-integration-scaffolding) | ~150 | Low (test-only) | Medium; deferred β€” see [cleanup-test-only-production-exports.md](cleanup-test-only-production-exports.md) | +| [`buildKanikoImage` unused return value](#4-unused-return-value-in-buildkanikoimage) | ~3 | None | Trivial | +| [Parallel cleanup for AWS/Azure](#5-aws-and-azure-integration-paths) | TBD | Need investigation | Medium | +| [Dependency audit](#6-dependency-audit) | indirect (vendor tree) | None | Low | +| [CLI subcommand/flag audit](#7-cli-subcommand-and-flag-audit) | TBD | Need investigation | Medium | + +## 1. Deprecated CLI flags + +Five `executor` flags (and one `warmer` flag) emit `logrus.Warn("Flag --X is deprecated. ...")` and translate to the new flag at runtime. Each one has: + +- A field in `KanikoOptions` (`pkg/config/options.go`) β€” typically named `*Deprecated` +- A `flag.StringVarP` / `BoolVarP` registration in `cmd/executor/cmd/root.go` around lines 328–332 +- A translation block in `checkNoDeprecatedFlags` (`cmd/executor/cmd/root.go:375+`) + +Candidates: + +| Flag | Replacement | Notes | +|---|---|---| +| `--snapshotMode` | `--snapshot-mode` | Naming-only alias | +| `--customPlatform` | `--custom-platform` | Naming-only alias | +| `--tarPath` | `--tar-path` | Naming-only alias | +| `--force-build-metadata` | n/a | Now the default behavior | +| `--skip-unused-stages` | n/a | Now the default behavior | +| `--whitelist-var-run` | `--ignore-var-run` | Renamed | + +These are user-visible CLI changes, so they should land in a major version. Before removing, check `git blame` for how long each has been deprecated β€” anything deprecated for several releases (say, β‰₯3) is fair game; anything recent should wait. + +**Effort:** mechanical, ~50 lines deleted across `pkg/config/options.go` and `cmd/executor/cmd/root.go`. One commit per flag for easy revert, or one bundled commit with a clear "Removed: ..." entry in the changelog. + +## 2. Hidden flags + +Two flags are registered but hidden from `--help`: + +- `--azure-container-registry-config` (executor + warmer) +- `--bucket` (executor; comment says "Name of the GCS bucket from which to access build context as tarball") + +Hidden flags are typically one of: (a) deprecated but kept for backwards-compat, (b) still functional but for internal use, (c) leftover registrations that no longer do anything. + +**Investigation first:** trace each flag from its `StringVar` binding to actual consumption. If a flag's bound field has no production readers, it's vestigial and the registration can go. If it's still wired up, the question becomes whether to publicly re-expose it or formally deprecate. + +`--bucket` is particularly suspicious given the GCP-integration-tests-not-exercised situation β€” needs to be paired with that audit. + +## 3. GCP integration scaffolding + +Covered in [cleanup-test-only-production-exports.md](cleanup-test-only-production-exports.md). Roughly ~150 lines removable in `integration/` plus `pkg/util/bucket.Delete`, conditional on the bigger question of whether to keep the production GCS code path at all. The production `pkg/buildcontext/gcs.go` is reachable from `gs://` source context and would stay regardless of CI exercise. + +## 4. Unused return value in `buildKanikoImage` + +`unparam` flags: + +``` +integration/images.go:893:4: buildKanikoImage - result 0 (string) is never used +``` + +The function returns `(string, error)` but every caller discards the string. Two-line fix: change signature to return `error` only, drop the value from every `return` and call site. + +## 5. AWS and Azure integration paths + +The same audit that applies to GCS should apply to S3 and Azure Blob: + +- Are CI integration tests exercising `s3://` and Azure Blob source contexts? +- If not, the same "we don't validate this" status applies to S3-/Azure-specific code in `pkg/buildcontext/s3.go` and `pkg/buildcontext/azureblob.go`. +- Unlike GCP, these don't have a separate test-utility package equivalent to `pkg/util/bucket`, so the cleanup surface is smaller β€” mostly just the question of whether the production fetchers are exercised. + +**Effort:** investigation first. If S3 and Azure are also not exercised, decide whether they're real user features kept on faith or candidates for deprecation. + +## 6. Dependency audit + +Vendor directory is currently **71M** (337 files in `vendor/github.com/envoyproxy/go-control-plane` alone). Of the 33 direct deps in `go.mod`, all are reachable from production or test code β€” there are no outright-unused direct dependencies. The interesting findings are about transitive cost. + +### Heaviest vendored subtrees + +| Subtree | Size | Pulled in by | +|---|---|---| +| `github.com/aws/` | 11M | `pkg/buildcontext/s3.go`, `pkg/util/bucket` | +| `github.com/envoyproxy/go-control-plane/` | **9.6M** | **TEST-only transitive chain** β€” see below | +| `cloud.google.com/go/` | 6.5M | `pkg/buildcontext/gcs.go`, `pkg/util/bucket`, `integration/` | +| `google.golang.org/grpc/` | 4.0M | gRPC, pulled via cloud.google.com | +| `go.opentelemetry.io/otel/` | 3.1M | Pulled via `integration β†’ cloud.google.com/go/storage` | +| `github.com/Azure/` | 2.7M | `pkg/buildcontext/azureblob.go` | +| `google.golang.org/protobuf/` | 2.0M | Indirect; protobuf runtime | +| `github.com/go-git/` | 2.0M | `pkg/buildcontext/git.go` (production git context) | +| `github.com/google/` | 1.9M | go-containerregistry mostly | +| `google.golang.org/api/` | 1.3M | Cloud GCS SDK | + +### The envoyproxy oddity + +`github.com/envoyproxy/go-control-plane` is 9.6M of vendored code. `go mod why` initially makes this look like a test-only chain (it shows `.test` and `xds/e2e` in the trace), but **that's misleading** β€” the actual import path is through production grpc code: + +``` +pkg/util/bucket + β†’ cloud.google.com/go/storage + β†’ cloud.google.com/go/storage/grpc_dp.go ← production file + β†’ google.golang.org/grpc/xds/googledirectpath ← production file + β†’ google.golang.org/grpc/xds/* ← production + β†’ github.com/envoyproxy/go-control-plane/envoy/* (data types for Envoy) +``` + +`grpc_dp.go` is a 22-line file in the Google Cloud SDK whose entire content is two anonymous imports: + +```go +//go:build !disable_grpc_modules + +package storage + +import ( + _ "google.golang.org/grpc/balancer/rls" + _ "google.golang.org/grpc/xds/googledirectpath" +) +``` + +These register two gRPC mechanisms used for GCP-internal performance optimizations: **Direct Path** (low-latency GCS access from GCP VMs via the GCP-internal network) and **RLS** (route lookup service load balancing). Neither is needed for kaniko's use case β€” pulling a build context tarball once per build. The Google Cloud SDK gracefully falls back to plain HTTPS when these aren't available. + +### The fix: `-tags=disable_grpc_modules` + +The build tag is upstream-provided exactly to exclude these imports. Adding `-tags=disable_grpc_modules` to the executor and warmer build commands in `Makefile`: + +```diff +- GOARCH=$(GOARCH) GOOS=$(GOOS) CGO_ENABLED=0 go build $(if $(COVER),-cover) -ldflags $(GO_LDFLAGS) -o $@ $(EXECUTOR_PACKAGE) ++ GOARCH=$(GOARCH) GOOS=$(GOOS) CGO_ENABLED=0 go build -tags=disable_grpc_modules $(if $(COVER),-cover) -ldflags $(GO_LDFLAGS) -o $@ $(EXECUTOR_PACKAGE) +``` + +Measured impact (`go build ./cmd/executor`, default `-ldflags '-w -s'`): + +| Build | Binary size | +|---|---| +| Default | 72 MB | +| `-tags=disable_grpc_modules` | 54 MB | + +**βˆ’18 MB, ~25% reduction** in the shipped executor binary. Same approximate reduction expected for the warmer. The linker drops `envoyproxy/go-control-plane/envoy/*`, `grpc/xds/*`, `grpc/balancer/rls`, and several transitively-reachable subpackages. + +### What this does NOT change + +- **Vendor directory stays at 71 MB.** `go mod vendor` always includes all build-tag variants β€” it can't know which tags a downstream build will use. The win is purely at link time. +- **Production GCS source context (`gs://`) still works.** Direct Path is an optimization; HTTPS is the always-supported fallback that the SDK uses transparently. Users see no functional difference. +- **`go.mod` is unchanged.** The dependencies are still declared; they just don't end up in the final binary. + +### Trade-off + +Kaniko users running inside a GCP VM that fetches build contexts from a same-project GCS bucket would lose the Direct Path optimization. For a typical build-context tarball (under ~100 MB) and the rare GCP-VM-to-GCS-in-same-project topology, the latency/throughput difference is negligible. For all other users (CI runners, Kubernetes clusters not on GCP, etc.) there is zero difference. + +### Marginal direct-dep candidates + +| Dep | Usage | Note | +|---|---|---| +| `github.com/google/slowjam` | One line in `cmd/executor/main.go`: `stacklog.MustStartFromEnv("STACKLOG_PATH")` β€” emits a stack trace on signal if the env var is set | If nobody on the team actively uses `STACKLOG_PATH` for diagnostics, drop it. Saves a small vendored module and one stray dep. | +| `github.com/golang/mock` | Only used by `pkg/util/mock_layer_test.go` (the `gomock`-generated `MockLayer` we just folded in) | Keep. gomock is the standard mock framework; replacing it would mean hand-writing a `MockLayer`. | + +### Conditional cleanup tied to bigger questions + +These deps come and go with bigger decisions: + +- **Drop GCS entirely** (production + integration): removes `cloud.google.com/go/storage` (6.5M vendored) and most of the grpc/otel/envoyproxy baggage from the binary too. Risk: real users may rely on `gs://` source context. +- **Drop S3 entirely**: removes `aws/aws-sdk-go-v2/*` (11M vendored). Same risk question. +- **Drop Azure entirely**: removes `Azure/azure-sdk-for-go` (2.7M vendored). Same risk question. + +These are product/user questions, not code-quality questions. + +### Recommended action + +1. **Add `-tags=disable_grpc_modules` to the Makefile build lines.** Cheapest, highest-impact change β€” 18 MB off the executor binary with no functional regression for kaniko's GCS use case. See the envoyproxy section above for the full analysis. +2. Check whether `STACKLOG_PATH` (slowjam) is anyone's actual diagnostic tool; remove if not. +3. Don't chase the vendor-size goal in isolation β€” vendor mode includes everything regardless of tags; the meaningful target is the linked binary, and most of its weight is justified by actual product features. + +## 7. CLI subcommand and flag audit + +Kaniko ships two binaries (`executor`, `warmer`) with many flags. Each flag is a contract β€” and contracts have maintenance cost. + +- Walk every flag in `executor` and `warmer` and ask: is it documented in `README.md`? Is it used by any of the integration tests? When was its bound option field last read by production code? +- Surfaces likely candidates: flags that were added for one-off use cases, flags that overlap with environment-variable equivalents, flags whose entire purpose is GCP/AWS/Azure-specific behavior. + +This is a more involved audit than the others β€” best done as a one-time pass with output captured in a markdown checklist. + +## Out of scope + +These came up during the analysis but are NOT recommended actions: + +- **Style/naming fixes** (`ST1000`, `ST1003`, `ST1016`, `ST1020`, `ST1022`). Many findings, all cosmetic. Touching them would create churn without removing anything. Leave for future opportunistic cleanup as files are edited for other reasons. +- **Unused struct fields**. Go's tooling can't reliably detect these without custom analysis. The signal-to-noise ratio is poor. +- **Refactoring "looks weird" code**. Code that works but isn't pretty is not the same as dead code. Out of scope here. + +## Risk + +Low overall. The CLI changes (sections 1–2) are user-visible and warrant a major-version bump and a clear changelog entry. The rest is internal-only. diff --git a/docs/design_proposals/cleanup-test-only-production-exports.md b/docs/design_proposals/cleanup-test-only-production-exports.md new file mode 100644 index 000000000..5e4e0012e --- /dev/null +++ b/docs/design_proposals/cleanup-test-only-production-exports.md @@ -0,0 +1,52 @@ +# Cleanup: production symbols only reachable from tests + +## Status +Survey. Each row is a separate candidate; the recommendation column is the suggested action, not a commitment. + +## Background + +`deadcode ./...` (`golang.org/x/tools/cmd/deadcode`) on the kaniko main binary lists functions unreachable from `main`. Filtering out `_test.go`, fakes/mocks, `testutil/`, and the `integration/` helper package leaves nine candidates. All of them compile into production binaries today but are only exercised by test code, an integration helper, or godoc. + +The prior cleanup of `util.ParentDirectoriesWithoutLeadingSlash` (commit `c8dffc9b`) is the same pattern. + +## Already done + +- `warmer.ExampleWarmer_Warm` β€” deleted (commit `e06b100e`). +- `executor.ResolveCrossStageInstructions` β€” deleted along with four no-op call sites and the orphan test (commit `5bc9e946`). +- `snapshot.filesWithLinks` + `util.GetSymLink` β€” deleted along with `TestFileWithLinks` and the orphaned `setupSymlink`/`sortAndCompareFilepaths` test helpers (commit `e11bc167`). Function had been dead since `a675ad998` (2020-02-20). +- `util.GetInputFrom` β€” deleted along with `TestGetInputFrom` (commit `f5481bde`). One-line wrapper around `io.ReadAll`, dead since `2ea368dde` (2021-12-26) when stdin reading switched to streaming via `gzip.NewReader`. +- `util.CreateTarballOfDirectory` β€” moved into `integration/tar.go` as private `tarballOfDirectory` (commit `4c161701`). Introduced in `679c71c90` (2022-06-14) as an integration-test helper and never had a production caller. Its unit test (`Test_CreateTarballOfDirectory`) and `pkg/commands/add_test.go` (the latter's coverage is duplicated by `Dockerfile_test_add` and friends in the integration suite) were dropped at the same time. + +## Decided to leave alone + +| Symbol | Location | Reason | +|---|---|---| +| `util.OSFS` (type + `Open`) | `pkg/util/fs_util.go:1419` | Mechanical rename across 6 sites for a 6-line type. Not worth the churn. | +| `bucket.Delete` | `pkg/util/bucket/bucket_util.go:46` | Marginal cleanup. Also: GCP integration tests aren't actually exercised in CI, so the whole GCS path (including `bucket.Delete`'s single integration caller) is in a "we don't validate this" state β€” a bigger question than where to put one function. | +| `timing.Summary` / `timing.TimedRun.Summary` | `pkg/timing/timing.go:77,86` | Small, coherent public API of `pkg/timing`; revisit if `pkg/timing` ever shrinks further. | + +## Future consideration + +GCP integration tests are not run in CI. That means anything reachable only through GCS code paths (bucket cleanup, GCS context fetch in `cmd/executor` if untouched by other tests, etc.) is effectively unvalidated. Worth a separate pass to identify which GCS-related code is truly used by anyone vs. carry-over from upstream that we should drop. + +## Detail by candidate + +### `bucket.Delete` β€” move to `integration/` + +`pkg/util/bucket/bucket_util.go:46`. Single caller: `integration/integration_test.go:102` (GCS cleanup between integration runs). The function does not belong in `pkg/util/bucket` if the only consumer is the integration test. + +Plan: move the function (and any test for it) into the `integration/` package. `pkg/util/bucket` keeps the upload-side functions that production code actually uses. + +### `util.OSFS` β€” move to `testutil` + +`pkg/util/fs_util.go:1419` defines `OSFS` as a `fs.FS` implementation that calls `os.Open`. Production code uses `NoAtimeFS` exclusively (`FSys fs.FS = NoAtimeFS{}` at line 48). `OSFS` exists so tests can swap `FSys` to a regular reader; it is referenced from: + +- `pkg/commands/copy_test.go` (2 sites) +- `pkg/util/command_util_test.go` +- `pkg/util/fs_util_test.go` (3 sites) + +Plan: move the type to `testutil` (e.g. `testutil.OSFS`) since it's a cross-package fixture. Mechanical rename in the call sites. + +## Risk + +Low across the board for the deletions that were done. The `executor` and `util` packages are not formally part of the kaniko library API even though they are under `pkg/`. A changelog note under "Removed API" covers the exported deletions for any out-of-tree consumer. diff --git a/docs/design_proposals/integration-test-coverage.md b/docs/design_proposals/integration-test-coverage.md new file mode 100644 index 000000000..dfba8ec29 --- /dev/null +++ b/docs/design_proposals/integration-test-coverage.md @@ -0,0 +1,45 @@ +# Integration test coverage for untested CLI flags + +* Author: Martin Zihlmann +* Date: 2026-05-01 +* Status: Under implementation (PR #669) + +## Background + +The kaniko executor exposes many CLI flags that affect build behaviour, output format, caching strategy, snapshot mode, and filesystem handling. Coverage on `pkg/executor/build.go` shows a large number of option-gated branches that are never exercised by the integration test suite. This document identifies those gaps and proposes concrete integration tests for each one. + +The analysis is based on comparing every field in `pkg/config/options.go` against the flags and Dockerfiles wired up in `integration/images.go` and `integration/integration_test.go`. + +## Gap analysis + +The following flags have zero integration test coverage. + +| Flag | Config field | build.go reference | +|---|---|---| +| `--tar-path` | `TarPath` | image push path | +| `--oci-layout-path` | `OCILayoutPath` | image push path | +| `--materialize` | `Materialize` | line 406 | +| `--ignore-var-run=false` | `IgnoreVarRun` | snapshot exclusion | + +Already covered: `--compression=zstd`, `--compression-level`, `--compressed-caching=false` (all three wired onto `Dockerfile_test_cache` via `additionalKanikoFlagsMap`; covers `pushLayerToCache` zstd path in `push.go`, `CompressionLevel > 0` and `CompressedCaching` false branch in `getLayerOptionFromOpts`; note: the zstd path in `saveSnapshotToLayer` at `build.go:639` requires an OCI-format base image and remains uncovered), `--digest-file`, `--image-name-with-digest-file`, `--image-name-tag-with-digest-file` (all three passed as `--digest-file=/dev/stdout` etc. in every `buildKanikoImage` call, output goes to captured stdout), `--reproducible`, `--single-snapshot`, `--snapshot-mode=redo`, `--use-new-run`, `--target`, `--cache`, `--no-push-cache`, `--cache-copy-layers`, `--secret`, `--cleanup`, `--skip-unused-stages`, `--annotation`, `--label`, `--registry-mirror`, `--registry-map`, `--skip-default-registry-fallback`, `--kaniko-dir`, `--build-arg`, `--git`, `--pre-cleanup` (baked as `KANIKO_PRE_CLEANUP=1` into `kaniko-alpine`, exercised by `mz595`), `--preserve-context` (baked as `KANIKO_PRESERVE_CONTEXT=1` into `kaniko-alpine`, exercised by `mz595`; additionally has a dedicated behavioural test via `Dockerfile_test_preserve_context`), `--ignore-path` (`Dockerfile_test_ignore_path` with `--ignore-path=/kaniko-extra-file --ignore-path=/kaniko-extra-dir` in `additionalKanikoFlagsMap`, layer-length-mismatch disabled so absence of the path is verifiable), `--cache-run-layers=false` (hardcoded in `buildKanikoImage` at `images.go:750`, exercised by every build including all cache builds), `--snapshot-mode=time` (`TestSnapshotModes` builds `Dockerfile_test_run` with `full`, `redo`, and `time` and compares all three), `--custom-platform` (`Dockerfile_test_cross_compile`: two-stage arm64 build with cross-stage `COPY`, no `RUN`; wired via `additionalKanikoFlagsMap` with `--custom-platform=linux/arm64` and `additionalDockerFlagsMap` with `--platform=linux/arm64`; `platformMap` ensures `docker pull` and diffoci both use `linux/arm64`), `FF_KANIKO_CACHE_LOOKAHEAD=1` (in global `envVars` at `images.go:122`, exercised by every cache build), `KANIKO_PRINT_PLAN=1` (in global `KanikoEnv`, `RenderStages` exercised on every build). + +## Proposed tests + +### Group 1: Tar and OCI layout output β€” `TestTarPath`, `TestOCILayoutPath` + +**File:** `integration/integration_test.go` (two new functions) + +Both tests skip pushing to a registry. Mount a host temp dir into the kaniko container and point the flag at it. + +`TestTarPath`: pass `--tar-path=/out/image.tar --no-push`. Assert the output file is a valid tar archive containing `manifest.json` (Docker image layout). + +`TestOCILayoutPath`: pass `--oci-layout-path=/out/image --no-push`. Assert the output directory contains `oci-layout` and `index.json` (OCI image layout spec). + +Reuse `Dockerfile_test_env` for both. No new Dockerfiles needed. + +## Prioritised implementation order + +| Priority | Group | Effort | Primary coverage gain | +|---|---|---|---| +| 1 | Group 1 β€” tar / OCI output | Low | `TarPath`, `OCILayoutPath` push paths | +| 2 | `--materialize` | High | requires pre-warmed cache setup | diff --git a/docs/design_proposals/mz334-infer-cache-key-external-source.md b/docs/design_proposals/mz334-infer-cache-key-external-source.md new file mode 100644 index 000000000..1236c1503 --- /dev/null +++ b/docs/design_proposals/mz334-infer-cache-key-external-source.md @@ -0,0 +1,75 @@ +# mz334: infer cross-stage cache key β€” part 2 (external image sources) + +## Where we are + +PR [#618](https://github.com/osscontainertools/kaniko/pull/618) introduced `FF_KANIKO_INFER_CROSS_STAGE_CACHE_KEY`. When set together with `--cache-copy-layers`, a `COPY --from=` emits **two** cache entries side by side: + +- the existing content-addressed entry, keyed by hashes of the copied files; +- a new cachekey-addressed entry, keyed by the source stage's precomputed `finalCacheKey` (a pointer that resolves to the content key). + +The pointer entry is what lets cache-lookahead skip the filesystem scan: when we already trust the source stage's identity via its final cache key, the COPY layer's cache key can be derived without reading any files. The content-addressed entry is retained because it remains more stable in the opposite direction β€” file contents can be unchanged while the source stage's `finalCacheKey` shifts (e.g. metadata changes upstream). Both writes happen on the build pass; the optimize pass can hit on either. + +The shortcut today only fires for **local stage references**: `crossStageCacheKey` (`pkg/executor/build.go:202`) parses `copyCmd.From()` as a numeric stage index via `strconv.Atoi` and returns false on failure. `resolveCrossStageCommands` (`pkg/dockerfile/dockerfile.go:265`) rewrites `COPY --from=` to the numeric form only when the name matches a known stage; external image references like `COPY --from=alpine:3.18` stay as image refs. + +So external-image `COPY --from` still walks the file-hashing path: kaniko pulls and unpacks the external image (via `fetchExtraStages` machinery) and `populateCompositeKey` hashes every file the COPY reads. That's expensive β€” large external images make the cache key computation itself a significant cost β€” and it's avoidable, because the external image already has a globally stable content-addressable identifier: its digest. + +## What this part adds + +Extend the inferred-key shortcut to external-image `COPY --from` sources by using the source image's digest as the stand-in identifier. Functionally: + +- For `COPY --from=`: behaviour unchanged (use `stageFinalCacheKeys[fromIdx]`). +- For `COPY --from=`: resolve the image's digest once, use it as the inferred-key contribution. + +The composite cache key then no longer depends on the file content scan of an external image β€” only on its registry-resolved digest, the COPY command string, and any args. The redirect-pointer machinery (`pushPointer` / `redirectCacheKey`) extends naturally to this case. + +This is **strictly an optimization**: the file-hashing path remains correct. The flag continues to gate the optimization so we can validate equality before flipping the default. + +## Why it belongs separate from cache-lookahead + +PR #618 framed the two features as something to "activate together" β€” that's still the rollout plan for lookahead, because lookahead has nothing to use the inferred keys for if they aren't emitted. But the *extension to external sources* is a strict superset of what part 1 does and stands on its own: it doesn't depend on lookahead, doesn't depend on stage elimination, and improves cost in the standalone infer-cache-key path. Bundling it into the cache-lookahead workstream would conflate two rollouts and force users to opt into both behaviours together; treating it as part 2 of infer-cache-key keeps the flag matrix clean. + +## Code change sketch + +`crossStageCacheKey` becomes a two-branch lookup: + +```go +func crossStageCacheKey(command commands.DockerCommand, stageFinalCacheKeys map[int]string, externalImageDigests map[string]string) (string, bool) { + copyCmd, ok := commands.CastAbstractCopyCommand(command) + if !ok || copyCmd.From() == "" { + return "", false + } + if fromIdx, err := strconv.Atoi(copyCmd.From()); err == nil { + cacheKey, ok := stageFinalCacheKeys[fromIdx] + return cacheKey, ok + } + digest, ok := externalImageDigests[copyCmd.From()] + return digest, ok +} +``` + +`externalImageDigests` is populated upstream from the same place that fetches external `COPY --from` images today (`fetchExtraStages` and its callers). We resolve each external image once, cache its `Image.Digest()`, and thread the map down through `optimize` / `populateCompositeKey` alongside `stageFinalCacheKeys`. + +The optimize-side block (`pkg/executor/build.go:333`) does not need new branches β€” it just calls into `populateCompositeKey` with the augmented signature. + +The redirect-pointer write (`pkg/executor/build.go:576`) works unchanged: a pointer keyed by the inferred key resolving to the content key already covers the external-image case the moment the inferred key starts being computed. + +## Stability vs. PR #618's local-source path + +PR #618 highlighted that the inferred key for a local stage can be *less* stable than file-hashing in some scenarios β€” upstream metadata changes can shift `finalCacheKey` without changing the files the COPY touches. That argument doesn't translate to external image sources: the manifest digest is the registry-canonical content identifier, so the inferred key for `COPY --from=alpine@sha256:...` is at least as stable as file-hashing, and often more so (no transient FS attributes leak into the hash). + +For unpinned tag references (`COPY --from=alpine:3.18`), the digest can shift between builds when the tag is republished β€” but that's the same caveat external base images already carry (`FROM alpine:3.18` has identical behaviour). The inferred key inherits this; it does not make caching less safe than the file-hashing path, since both are computed *after* the image has been resolved. + +Like part 1, both cache entries (content-addressed + cachekey-addressed) are emitted, so a downstream consumer that hashes files identically gets a cache hit either way. + +## Scope + +Strictly: when `FF_KANIKO_INFER_CROSS_STAGE_CACHE_KEY=1`, the existing redirect-pointer logic now also fires for `COPY --from=`. No new flag, no change to the flag default, no change to the local-stage path. A consumer that has the flag off sees no behavioural difference; a consumer with the flag on simply gets the optimization on more `COPY --from` instructions. + +## Test plan + +A golden test under `golden/testdata/test_issue_mz334_external/` covering `COPY --from=` with `--cache --cache-copy-layers` and `FF_KANIKO_INFER_CROSS_STAGE_CACHE_KEY=1`. Pin the source by digest (`alpine@sha256:...`) so the golden output stays stable, and assert the plan emits `CACHE REDIRECT HIT/MISS` on the COPY just as local-source tests do today. + +## Open questions + +- Where does the `externalImageDigests` map get built? Cleanest is at `fetchExtraStages` time, since that's where we already pull the external image. The map needs to be in scope wherever `populateCompositeKey` is called for a COPY command. +- Image digest vs. config digest vs. manifest digest: `Image.Digest()` returns the manifest digest. That's the right choice β€” it's what registry deduplication uses and what `FROM image@` references. diff --git a/docs/design_proposals/mz334-stage-elimination.md b/docs/design_proposals/mz334-stage-elimination.md new file mode 100644 index 000000000..8468542aa --- /dev/null +++ b/docs/design_proposals/mz334-stage-elimination.md @@ -0,0 +1,167 @@ +# mz334: stage elimination using the cache-lookahead precompute + +## Where we are + +`FF_KANIKO_CACHE_LOOKAHEAD` (introduced in part-2b) adds a precompute pass before the real build: for each `KanikoStage` we instantiate a `stageBuilder` and call `optimize(..., hasContext=false)`. That populates a `stageCacheInfo` (per-command `cacheKeys`/`cacheHits` and redirect keys), and the stage's `finalCacheKey` lands in `stageFinalCacheKeys[stage.Index]`. + +Today we only use those precomputed keys to: + +- Render `CACHE HIT` / `CACHE MISS` annotations in the dryrun plan (`RenderStages`). +- Assert in the real build loop that the precomputed `finalCacheKey` matches the one computed during `optimize(..., hasContext=true)` β€” the correctness fence (`pkg/executor/build.go:1156`). + +Static stage elimination (skip-unused-stages, `FF_KANIKO_SQUASH_STAGES`) already runs upstream in `dockerfile.MakeKanikoStages`. It only looks at the dockerfile graph (FROM / COPY --from edges), not at the cache. + +The next step is to feed precompute results back into the build plan and **dynamically drop stages whose results are already in the cache**. The historical attempt (commit `ad9a05ae0`, reverted by `2719b7fee`) did this with squashing β€” that part is now done statically in dockerfile.go, so this design is narrower: only cache-driven elimination, on top of the existing graph-driven elimination. + +## What "eliminate" means here + +Three terminal states for a stage after precompute: + +1. **Build normally** β€” at least one command misses cache, or the cached result cannot replace the stage's role (e.g. it must be unpacked because a downstream stage `FROM`s an uncached stage that branches off it). +2. **Materialize from cache, do not run commands** β€” every command in the stage cache-hits. The final layer in the registry, combined with the cached intermediate layers, fully reconstructs the stage's FS / image. We still need the *result* (because something downstream consumes it), but we never execute commands or unpack the base into the rootfs. +3. **Drop entirely** β€” the stage exists only to feed downstream consumers, and every consumer's reference to it is itself cache-resolved. No need to fetch the base image, no need to materialize anything. + +(1) is the status quo. (2) and (3) are what this design adds. + +The cleanest distinction: (2) is about *not running commands*, (3) is about *not even constructing a stageBuilder*. + +## Decision algorithm + +After precompute populates `cacheInfo[idx]` for every stage, walk stages and classify them. + +For each stage `s`, define: + +- `s.fullyCached` ≑ every non-`MetadataOnly` command in `s` had `cacheHits[i] == true` during precompute (i.e. precompute's `optimize` actually swapped the command for a `Cached` impl). +- `s.consumedAsBase` ≑ some later stage has `BaseImageStoredLocally && BaseImageIndex == s.Index`. +- `s.consumedAsCopyFrom` ≑ some later stage has a `COPY --from=`. + +Then walk consumers backward to compute `s.fsNeeded`: does anybody downstream actually need `s`'s filesystem materialised? + +- A `FROM s` consumer needs `s`'s FS unless that consumer is itself in state (3). +- A `COPY --from=s` consumer needs `s`'s FS unless the consuming COPY command itself cache-hits (which the precompute pass already knows: it's the redirect hit recorded in `cacheInfo[consumer].redirectHits[copy_index]`, gated by `FF_KANIKO_INFER_CROSS_STAGE_CACHE_KEY`). +- The final stage always needs its FS materialised (we push it). + +Classification rule: + +| `fullyCached` | `fsNeeded` | classification | +|---------------|------------|----------------| +| false | true | (1) build normally | +| false | false | unreachable β€” see below | +| true | true | (2) materialize-from-cache | +| true | false | (3) drop | + +The `fullyCached=false && fsNeeded=false` row is *currently* unreachable: if a stage isn't fully cached but nobody needs its FS, static skip-unused-stages would already have eliminated it. We assert this and panic if it ever fires β€” that catches a logic regression cheaply. + +## Concrete cases (from `test_issue_mz334/Dockerfile`) + +``` +FROM busybox AS first +RUN touch /blubb + +FROM first AS second +RUN touch /bla + +FROM second AS third +RUN touch /bli + +FROM second AS final +COPY --from=first /blubb /blubb +COPY --from=third /bli /bli +RUN ls -lah /blubb +``` + +Suppose everything is in the cache. The classification under this design: + +- `first`: fully cached. Consumed as base by `second`. `fsNeeded` propagates up from `second`. If `second` is dropped, and `first`'s COPY --from in `final` is a redirect-hit, `first` drops entirely (no base fetch, no FS). +- `second`: fully cached. Consumed as base by `third` and `final`. Both downstream consumers need `second` as their base FS unless they themselves drop. +- `third`: fully cached. Only consumed via `COPY --from=third /bli` in `final`. If that COPY is a redirect-hit, `third` drops; otherwise we materialize from cache to extract `/bli`. +- `final`: must build (it's the push target), but every command precomputed as a hit β†’ final is in state (2): materialize from cache. + +The all-cached happy path collapses the build to "pull final image from cache, push it". That's the headline win. + +## Where this slots into build.go + +The precompute loop currently lives at `pkg/executor/build.go:1004-1051`. The real build loop at `pkg/executor/build.go:1113-1255` does the work today. Stage elimination changes both: + +### Precompute pass + +Augment the precompute output. Today it produces `cacheInfo[]` and writes `stageFinalCacheKeys`. Add: + +- A per-stage `disposition` (drop / materialize / build), computed after the precompute walk by the classification above. +- For materialize: capture the cached final image reference (the layer returned by `layerCache.RetrieveLayer(finalCacheKey)`) so the real build doesn't repeat the lookup. + +The precompute pass already calls `layerCache.RetrieveLayer` per command via `optimize`. We don't need new round trips β€” we just need to thread the per-command hit signal up to a per-stage signal, and reuse the layer references. + +One blocker: precompute currently aborts cache key tracking when it hits `COPY --from` because `populateCompositeKey` cannot hash a not-yet-built stage's files (`pkg/executor/build.go:323-330`). But the `crossStageCacheKey` shortcut (using `stageFinalCacheKeys` instead of hashing files) already exists for the build pass β€” it just isn't invoked from precompute. To make `fullyCached` meaningful for stages that contain `COPY --from`, precompute must take the shortcut path too. That is a prerequisite β€” without it, any stage with `COPY --from` always classifies as (1). + +This prerequisite is small but it changes the meaning of the assertion at `build.go:1156`: precompute will now produce non-empty `finalCacheKey` for stages with `COPY --from`, and the real-build key must match. The existing equality check already handles that β€” but we should add a golden test that includes `COPY --from` precisely to exercise it. + +### Real build pass + +Branch by disposition: + +- **drop**: skip the stage entirely. Do not call `RetrieveSourceImage`, do not call `newStageBuilder`, do not record into `stageArgs`. Cross-stage deps (`SAVE FILES`) must already be unneeded β€” that's part of `fsNeeded=false`. +- **materialize**: construct a `stageBuilder` but replace `sb.build(...)` with a fast path that sets `sb.image` to the cached image and computes the cross-stage outputs without running commands. We still need `reviewConfig` and `mutate.Config` to produce the right config for downstream `FROM` consumers and for `final.push`. +- **build**: today's path, unchanged. + +The materialize path's tricky part: if a downstream stage uses this stage's FS via `FROM` and that downstream stage is **not** itself materialized (some downstream command is a miss), we need the FS unpacked into `/kaniko/stages/N`. So materialize still needs an "unpack into stage dir" step when downstream consumers need the FS. That's the same condition that drives `SaveStage` today. + +For final-stage materialize with no downstream cross-stage consumers, the fast path is: pull cached final image, run push logic, exit. No snapshotter, no unpack, no command loop. + +## Stage args propagation + +`stageArgs` (`build.go:1002`) tracks each stage's `BuildArgs` because downstream `FROM stage_N` inherits the parent's args. Today, both the precompute and the real build pass populate it. With drops, we still need `stageArgs[droppedIndex]` populated for any consumer that does `FROM droppedStage` β€” *unless* the consumer is itself dropped. The simplest rule: always populate `stageArgs` in the precompute pass; drops only skip the real build, not the precompute bookkeeping. (Precompute already does this today.) + +## Cross-stage dependency files + +`crossStageDependencies` is calculated up front from the dockerfile and tells us which files each stage must `SAVE FILES` for downstream use. A materialized stage still has to produce those files β€” it must unpack just enough to copy them out, OR a smarter path: read the files directly from the cached image's layers without unpacking. The simple approach (unpack-then-copy) is enough for v1; the layer-direct path is an optimization for later. + +A dropped stage by construction has no downstream FS consumers (that was the condition for dropping), so it has no `SAVE FILES` obligation. + +## Plan rendering + +The dryrun `RenderStages` output is what the golden tests assert on. With elimination, the plan changes meaningfully: + +- A dropped stage produces a single line: `DROP STAGE ` (or omitted entirely β€” TBD by what reads better in golden diffs). +- A materialized stage replaces `UNPACK ... ... CLEAN` with something like `MATERIALIZE FROM CACHE: ` followed by `SAVE FILES` / `SAVE STAGE` as needed. +- A normally-built stage looks exactly like today. + +Concrete: the `test_issue_mz334/plans/cached` golden currently shows every command with a `CACHE HIT:` line. Under elimination it should collapse the all-hits case to a single `MATERIALIZE` line per stage, dramatically shortening the plan. + +We will need a third plan file for the test (or replace `cached` with the elimination output). I lean toward an additional `eliminated` plan, gated by a new feature flag (see below), so we can ship behind a flag and keep the existing assertion path intact for one release. + +## Feature flag + +Introduce `FF_KANIKO_CACHE_STAGE_ELIMINATION`, off by default. It requires `FF_KANIKO_CACHE_LOOKAHEAD=1` to take effect β€” assert this combination at startup. The flag wraps: + +- the disposition classification, +- the drop/materialize branches in the real build loop, +- the new `RenderStages` output. + +Keeping it separate from `FF_KANIKO_CACHE_LOOKAHEAD` lets us turn on elimination without losing the lookahead assertion safety net during rollout. + +## Prerequisites and ordering + +These should land as separate commits, in this order: + +1. **Precompute past `COPY --from`**. Replace the `stopCache=true, keyValid=false` bailout in `optimize` with the `crossStageCacheKey` shortcut when `!hasContext` and `stageFinalCacheKeys[fromIdx]` is set. Validates immediately via the existing equality assertion. Golden test: extend `test_issue_mz334` so the precompute plan's `CACHE HIT/MISS` annotations cover the `COPY --from` commands in `final`. +2. **Classification scaffold**. Compute disposition per stage after the precompute walk, log it, but still take the existing build path. Golden test renders disposition labels but does not change behaviour. +3. **Materialize fast path**. Implement (2) β€” fully-cached stages that still need their FS get materialized from the cached final image instead of running commands. Behind `FF_KANIKO_CACHE_STAGE_ELIMINATION`. +4. **Drop fast path**. Implement (3) β€” fully-cached stages with no FS consumers are dropped completely. +5. **Final-stage materialize push-only fast path**. Special-case the common "everything cached, final image pulled from cache and pushed" path so we skip snapshotter init and `fetchExtraStages` when nothing remains to build. + +Each step is independently testable via golden tests with the `KeySequence` mechanism already used in `test_issue_mz334`. + +## Risks + +- **Image equivalence**. The layer cache stores per-command diffs; "materialize from cache" relies on the assumption that fetching+applying all cached layers yields a bit-identical image to building from scratch. The current per-command cache substitution already relies on this. For state (2) we additionally need to make sure config-file mutations (`reviewConfig`, OS/arch override, labels) still happen β€” `sb.image` is set, but the subsequent `mutate.Config` / `mutate.ConfigFile` block at `build.go:1168-1190` must still run. +- **`fetchExtraStages`** (`build.go:1064`) pulls images referenced via `--from=` or similar non-stage references. Dropping a stage that contained such a reference would skip its fetch. The cleanest rule: precompute walks every stage's commands, so `fetchExtraStages` runs based on the union of all stages including dropped ones. (Today `fetchExtraStages(kanikoStages, opts)` already gets the full list β€” so this is automatically correct as long as we don't filter `kanikoStages` before that call.) +- **`PreserveContext` interactions**. The build loop saves and restores the build context across stages. A drop must still leave the snapshotter in the right state for the next stage. Materialize might be able to skip snapshot ops entirely if nothing in the materialized stage touched the rootfs. +- **Cache poisoning blast radius**. With elimination, a single poisoned cache entry can short-circuit an entire chain of stages. The lookahead assertion (`precomputedKey == finalCacheKey`) doesn't help here because elimination *replaces* the build pass β€” there's nothing to assert against. Mitigation: keep the assertion alive for stages classified as (1), and accept that (2)/(3) inherit the existing per-command-cache trust assumption. +- **`fullyCached` vs. the `stopCache` semantics**. Today `optimize` may set `stopCache=true` on a miss (or under `FF_KANIKO_CACHE_PROBE_AFTER_MISS`, may continue probing). `fullyCached` must agree with "every non-MetadataOnly command was actually replaced with a `Cached` impl", not just "every key resolved" β€” `MetadataOnly` commands are not in the layer cache and must be excluded from the count. + +## Open questions + +- Should `FF_KANIKO_CACHE_STAGE_ELIMINATION` subsume `FF_KANIKO_CACHE_LOOKAHEAD` once stable, or stay as a separate knob long-term? +- Do we want a "soft" mode where we classify and log dispositions but never actually drop/materialize? Could be useful for telemetry-only rollout before flipping behaviour. +- For materialize: pull the entire image vs. apply layer-by-layer? The former needs fewer registry round trips but breaks if any single intermediate layer is missing while the final is present. The latter matches today's per-command path more closely. Start with the latter for safety. diff --git a/docs/design_proposals/mz351-multi-target-bake.md b/docs/design_proposals/mz351-multi-target-bake.md new file mode 100644 index 000000000..d9c68a911 --- /dev/null +++ b/docs/design_proposals/mz351-multi-target-bake.md @@ -0,0 +1,166 @@ +# mz351: multi-target builds, push each target stage to its own destination + +Tracking: discussion mz351. + +## Goal of the first step + +Build several target stages of one Dockerfile in a single executor run and push each one to its own destination. Same context, same Dockerfile, one global set of build args. The only new capability over what kaniko does today is the per-target destination. + +The input is a small JSON bakefile. It is deliberately minimal for this step, but the schema is chosen so we can grow it over time (per-target build args, matrix expansion, per-target context) without a breaking change. + +Explicitly out of scope for this step: + +- Per-target build args. Build args stay global, exactly as today. The combinatoric case from the discussion (same stage built with `PYTHON_VERSION=3.10` and `3.11`) needs per-target args and is a later step. +- Matrix expansion, interpolation, HCL. + +This is one build and one push cycle, not N independent builds. + +## What kaniko already does + +`--target` already takes a list of stages. `dockerfile.targetStages` resolves them and the build loop in `DoBuild` builds the whole required subgraph in topological order, into the shared rootfs, with the existing per-stage `util.DeleteFilesystem()` resetting the filesystem between stages (`build.go:1270`). Cross-stage dependency files are saved to `KanikoInterStageDepsDir` and restored by later stages. Independent target stages are isolated by that same reset. Nothing new is needed here, and there is no cross-target filesystem contamination to design around. + +## The gap + +Only one of those stages is ever pushed. + +- `pushStage = targetStages[0]` and stages are marked `Push: i == pushStage` (`dockerfile.go:305`, `385`). Exactly one stage is a push stage. +- The build loop sets a single `pushImage` when it hits the push stage (`build.go:1238`). +- The loop returns that single image the moment it reaches the `Final` stage (`build.go:1244`), and `DoBuild` returns one `v1.Image`. +- `DoPush` pushes that one image to every entry in the global `opts.Destinations`. + +So every other built target stage is computed and then thrown away unless something downstream consumes it. The first step lifts exactly this: let every selected target stage carry its own destination and get pushed. + +## Input format and invocation + +A new `bake` subcommand, alongside the existing `push` and `login` subcommands, takes a JSON bakefile and the name of the target to build, both positional. The target name may be omitted when the bakefile defines exactly one target. + +``` +/kaniko/executor bake bake.json app --context . --dockerfile Dockerfile --build-arg FOO=bar +``` + +The bakefile may define several targets; selection is positional, in the style of `docker buildx bake `. Building one selected target at a time is the current step; building several in a single pass is the follow-up. The bakefile supplies each target's stage and destination. Everything else stays on the existing flags and applies globally: `--context`, `--dockerfile`, `--build-arg`, cache, registry, platform, secrets. The `bake` command registers the shared build flags (`addSharedBuildFlags`) but not `--target` or `--destination`, which the bakefile owns (see "CLI flags and the bakefile" below). + +The root executor command is left untouched. Single-target builds keep using `executor --destination ...` exactly as today. + +Step-one schema: + +```json +{ + "version": "1", + "targets": { + "app": { + "target": "app", + "destination": ["registry.example.com/app:latest"] + }, + "tools": { + "target": "tools", + "destination": ["registry.example.com/tools:latest"] + } + } +} +``` + +- `version` is a string. Present from day one so the parser can reject or adapt to later schema revisions. +- `targets` is a map keyed by an arbitrary target id. +- `target` is the Dockerfile stage to build. Optional, defaults to the map key when omitted. +- `destination` is a list of image references the built stage is pushed to. A target with no destination is an error unless `--no-push` is set. + +### How the format is meant to grow + +The schema is shaped so future steps are additive, never breaking: + +- Per-target build args: add an optional `buildArgs` object to a target. It layers over the global `--build-arg` set, target keys winning. Same merge model for future `labels` and `annotations`. +- Per-target context or dockerfile: add optional `context` and `dockerfile` to a target, falling back to the global flag. +- Matrix expansion: add an optional `matrix` block to a target that expands into several concrete targets at parse time, each resolving to the flat per-target form above. The flat list stays the compilation target, so matrix is pure front-end sugar. +- Top-level shared defaults: a future `version` bump can promote `context` and `dockerfile` into the file itself as top-level keys that targets inherit, for users who want the build fully described by the file rather than split across flags. + +The invariant behind all of this: a target is resolved by layering its own fields over a shared base (flags today, top-level file keys later), and the build consumes a flat list of fully resolved targets. Step one ships the flat list with only `target` and `destination` populated. Every later feature adds optional fields or a pre-expansion pass, and old bakefiles keep parsing. + +## Design + +### 1. Parse the bakefile into resolved targets + +A new small package parses the JSON into a slice of resolved targets, each carrying a stage name and a destination list, through a standalone function (roughly `bake.Parse(path) ([]Target, error)`). It must not live inside the cobra `RunE`, because both the `bake` subcommand and the golden test harness call it. For step one resolution is trivial: stage name and destinations straight from the file, everything else from the global flags. This is the seam where per-target overrides and matrix expansion slot in later. + +### 1b. The bake subcommand and shared build setup + +The `bake` subcommand follows the existing `push` subcommand pattern: its own `*config.KanikoOptions`, `AddKanikoOptionsFlags(bakeCmd, opts)` for the global flags, a positional bakefile path, and a `RunE` that parses the file, runs one `DoBuild`, and pushes each result. + +The catch is that the root executor's pre-build setup (`ValidateFlags`, `resolveSecrets`, `moveKanikoDir`, `resolveEnvironmentBuildArgs`, `resolveSourceContext`, `resolveDockerfilePath`, ignore-list setup) currently lives in `RootCmd.PersistentPreRunE` behind a `cmd.Use == "executor"` guard, so a subcommand does not get it. This setup must be extracted into a shared helper that both the executor `Run` and the `bake` `RunE` call. That refactor is a prerequisite and worth landing on its own. + +### 2. Mark every target as a push stage + +In `dockerfile.go`, drop the "first target is the push stage" rule and mark every selected target as a push stage. `Final` stays the highest-index stage so the loop still has a definite last stage to stop on. The set of push stages becomes "all targets" rather than "targets[0]". Each push stage is associated with its destination list, either stored on `KanikoStage` or kept in a `map[stageIndex][]destination` threaded through `DoBuild`. A map keeps `KanikoStage` free of push concerns and is easy to hand to the push phase. + +### 3. Collect multiple images in the build loop + +Replace the single `pushImage` with an accumulator. At each push stage, run the existing finalization (`mutate.CreatedAt`, reproducible canonicalization, labels, annotations) and append `{image, destinations}` to the result set. Each push image is captured during its own stage iteration, before that stage's `DeleteFilesystem`, so capturing is safe. + +The early `return` at `Final` becomes a return of the accumulated set. Because `Final` is the highest-index target and is itself now a push stage, every push image has already been captured by the time the loop reaches it. + +### 4. DoBuild and DoPush signatures + +`DoBuild` returns a slice instead of one image, roughly: + +```go +type BuiltImage struct { + Image v1.Image + Destinations []string +} + +func DoBuild(opts *config.KanikoOptions) ([]BuiltImage, error) +``` + +`RootCmd.Run` iterates the result and calls push per image. `DoPush` is refactored so its core pushes one image to one destination set, with the per-target destinations replacing the global `opts.Destinations` lookup. The no-push, tar-path, and oci-layout paths extend to "once per built image". The single-target path is just a slice of length one, so the common case stays unchanged in behaviour. + +## What stays global + +Build args, cache settings, registry options, platform, secrets. These are process-wide flags and do not move into the bakefile in this step. Labels and annotations are applied during push-stage finalization today and would apply to every pushed image for now. Per-target labels, annotations, and build args are the documented next step. + +## CLI flags and the bakefile + +`bake` is its own interface, not a backward-compatible wrapper over `executor`/`build`. That decision removes a whole class of complexity up front: a setting lives in exactly one place, never both, so there is no global-flag-versus-bakefile merge behaviour to define and no precedence to reason about. + +The migration is one-directional. A build setting is a plain `bake` CLI flag only until it is added to the bakefile schema; once it is in the schema, it lives in the bakefile and its flag is removed from `bake`: + +- Not yet in the schema: a normal `bake` flag, applied globally to the build (transitional). +- In the schema: bakefile-only, the flag is gone. + +`target` and `destination` are on the bakefile side from day one. As `buildArgs`, `labels`, `platform`, and the rest land in the schema, their flags (`--build-arg`, `--label`, ...) leave `bake`. Because a setting is never in both places, `destination` and `build-arg` need no different treatment; they simply migrate at different times. + +Overriding a bakefile value from the command line is still supported, through a targeted override in the style of docker bake's `--set`: + +``` +/kaniko/executor bake bake.json --set app.destination=registry/app:dev +``` + +`--set .=` names the target, so it stays unambiguous with many targets. It is the single override channel; plain build flags are never an override path. This need not land in the first step (the only bakefile-owned settings are `target`/`destination`, so there is little to override yet), but it is the intended shape, and we should not build a competing global-flag override. + +Because settings are never duplicated across CLI and bakefile, there is also no reason to gate which CLI flags are accepted on the bakefile `version`. `version` governs how the bakefile is interpreted, never the CLI surface. + +One plain validation remains, independent of all the above: destinations must be unique across the resolved targets, so two targets pushing to the same ref fails loudly rather than silently clobbering. + +## Output files + +`--digest-file`, `--image-name-with-digest-file`, and `--image-name-tag-with-digest-file` are single-valued and cannot name N images. First cut: reject them when a bakefile produces more than one target, with a clear error. A per-target or directory-based output is a follow-up. + +## Testing + +Two layers. + +End-to-end golden plan tests reuse the existing `golden` harness. That harness already drives builds through real CLI flags plus an on-disk `Dockerfile`, runs `DoBuild --dryrun`, and diffs the rendered plan against `plans/`. Bake gets its own test function rather than being folded into `TestRun`, because the input model differs: `TestRun` is "Dockerfile plus flags", bake is "Dockerfile plus bakefile plus optional `--target` subset". A new `TestBake` keeps each convention clean and carries the bake-specific cases (target-subset selection, multiple destinations per target, missing destination). The shared part, run `DoBuild --dryrun`, capture the output, and diff-or-update against the plan path, is extracted into a helper that both `TestRun` and `TestBake` call, so the compare and `-update` logic is not duplicated. + +`TestBake` differs only in setup. The bakefile is one more on-disk input next to the `Dockerfile` in the testdata directory. The harness reads it through the same `bake.Parse` the subcommand uses, populates the per-target destinations on `opts`, then calls the shared helper. The bakefile lives as a file, not a struct, so the real parser is exercised and new schema features are covered by adding a fixture and re-rendering with `-update`, no Go changes. + +For this to be meaningful, the dryrun plan (`RenderStages`) must emit a `PUSH ` line at each push stage. Today it renders the stage graph with no push information (see existing plans, which end at the final stage with no `PUSH` line). Adding per-stage push lines makes the golden file capture the stage-to-destination mapping, which is the behaviour under test. + +The bakefile parser and resolver get their own narrow unit test in the bake package, table-driven, with the JSON as an inline string going in and resolved targets coming out. This is where a struct or inline literal is the right tool, pinning the parse boundary directly: `target` defaulting to the map key, missing-destination errors, unknown `version`, malformed JSON. Keep it separate from the golden plan tests. + +## Why not N build-and-push cycles + +An earlier sketch of this proposal ran one `DoBuild` per target with a filesystem reset between them. That was the wrong shape. The build loop already builds the full multi-target subgraph in one pass with correct inter-stage resets, and shared ancestor stages are built once in-process rather than recomputed or replayed from cache per target. Building once and pushing many reuses all of that. The only real work is letting more than one stage be a push target and letting `DoBuild` hand back more than one image. + +## Open questions + +- Whether push-permission checks (`CheckPushPermissions`, run before the build in `Run`) should validate all target destinations up front so a typo in one target fails before any build work. Leaning yes. +- Parallel pushing of the finished images is a cheap later win and does not affect this design. diff --git a/docs/design_proposals/mz762-scoped-dockerignore.md b/docs/design_proposals/mz762-scoped-dockerignore.md new file mode 100644 index 000000000..9e8259958 --- /dev/null +++ b/docs/design_proposals/mz762-scoped-dockerignore.md @@ -0,0 +1,83 @@ +# mz762: scope `.dockerignore` to the build context when hashing `COPY --from` sources + +## Problem + +An allowlist-style `.dockerignore` (`*` then `!keep`) combined with a multi-stage `COPY --from` produces a stale image when the allowlisted file changes (osscontainertools/kaniko#762). With `--cache`, editing the only allowlisted file does not invalidate the layer that copies it; a subsequent build serves the old content. + +PR #763 fixes this in `FileContext.ExcludesFile` by returning "not excluded" for any absolute path that lies outside the build context. We have gated that behind `FF_KANIKO_SCOPED_DOCKERIGNORE` (default off) and added unit + integration coverage. This document proposes the proper fix and argues the `ExcludesFile` guard is a symptom-level workaround that should be replaced rather than promoted to default. + +## Root cause + +`.dockerignore` patterns are matched against paths **relative to the build context**. `ExcludesFile` is fed two unrelated kinds of path: + +1. **Build-context files** β€” e.g. `/workspace/keep`. Under `c.Root`; relativized to `keep` and matched. Correct, and exclusion here is necessary (an ignored file must not enter the cache key, since it is never copied). +2. **Inter-stage `COPY --from` sources** β€” e.g. `/kaniko/deps/builder/app/keep`. These live in a prior stage's extracted filesystem, not the build context. + +The `COPY` command already resolves its sources with the right context. `copyCmdFilesUsedFromContext` switches to a deps-rooted context with **no** ignore patterns when `cmd.From != ""`: + +```go +if cmd.From != "" { + fileContext = util.FileContext{Root: filepath.Join(kConfig.KanikoInterStageDepsDir, cmd.From)} +} +``` + +But the **cache-key hasher does not get that context.** `stageBuilder.optimize` threads its single `fileContext` parameter β€” the *build* context (`Root = `, `ExcludedFiles = .dockerignore`) β€” into `populateCompositeKey(command, files, …, fileContext, …)`, which calls `compositeKey.AddPath(p, fileContext)`, which calls `context.ExcludesFile(p)` unconditionally. So the resolved `/kaniko/deps/...` paths are matched against the *build context's* patterns. With an allowlist, the catch-all `*` matches the absolute path, `!keep` (anchored at the context root) does not rescue it, and the file is reported excluded β†’ dropped from the cache key β†’ stale image. + +So the defect is not in `ExcludesFile` per se; it is that the per-command source context is not threaded into the cache-key computation for `COPY --from`. Applying the build context's `.dockerignore` to prior-stage paths is a category error β€” it only ever appeared harmless because, without an allowlist, no pattern happened to match those absolute paths. + +## Proposed fix + +Hash `COPY --from` source files with the same deps-rooted context the command already uses for resolution (`Root = KanikoInterStageDepsDir/`, empty `ExcludedFiles`). Then `ExcludesFile("/kaniko/deps//app/keep")` relativizes to `app/keep`, matches against *empty* patterns, and never excludes β€” no `IsAbs` special case and no feature flag needed. + +### Cache-key stability + +`hashFile`/`hashDir` hash the file **content** (`util.CacheHasher`) at the absolute path; `context.Root` is used only for the `ExcludesFile` relativization, and `ExcludedFiles` only for the exclusion decision. Therefore, switching the context from build-rooted to deps-rooted changes the cache key **only** for paths whose exclusion decision flips β€” i.e. exactly the wrongly-excluded allowlist case. For every build that was not hitting the bug (no pattern matched the `/kaniko/...` path), the computed keys are byte-for-byte identical, so existing caches stay valid. This is the key argument for shipping the proper fix as default rather than behind a flag. + +## Implementation options + +### Option A β€” derive the context in `populateCompositeKey` (pragmatic) + +In `populateCompositeKey` (it already receives `command`), detect a cross-stage copy and build the deps-rooted context before `AddPath`: + +```go +fc := fileContext +if copyCmd, ok := commands.CastAbstractCopyCommand(command); ok && copyCmd.From() != "" { + fc = util.FileContext{Root: filepath.Join(config.KanikoInterStageDepsDir, copyCmd.From())} +} +for _, f := range files { + if err := compositeKey.AddPath(f, fc); err != nil { return compositeKey, err } +} +``` + +Smallest change; mirrors `copyCmdFilesUsedFromContext`. Downsides: duplicates the deps-context construction and pushes command-type knowledge (`CastAbstractCopyCommand`/`From()`) into the cache layer. The duplication can be removed by extracting a shared `commands.InterStageContext(from)` helper used by both sites. + +### Option B β€” the command owns its source context (cleaner) + +Let the command expose the context its `FilesUsedFromContext` paths belong to, so the hasher never has to know about `From`. Either return it alongside the files or add a method: + +```go +// on DockerCommand +FilesUsedFromContext(*v1.Config, *dockerfile.BuildArgs) (util.FileContext, []string, error) +``` + +`CopyCommand`/`CachingCopyCommand` return the deps-rooted context for `--from`, the build context otherwise; every other command returns the build context. `optimize`/`populateCompositeKey` simply hash each file with the returned context. This removes the category error at the source and keeps the cache layer command-agnostic, at the cost of an interface change touching all command implementations. + +**Recommendation:** Option B. It localizes "which context do these files belong to?" to the command that already answers it for resolution, and leaves `ExcludesFile`/`AddPath` free of `--from` special-casing. Option A is an acceptable interim if the interface churn is undesirable. + +## Relationship to `FF_KANIKO_SCOPED_DOCKERIGNORE` + +The proper fix makes the `ExcludesFile` `IsAbs` guard redundant: with the correct context, out-of-context absolute paths no longer reach the matcher with foreign patterns. Plan: + +- Keep `FF_KANIKO_SCOPED_DOCKERIGNORE` **off by default** as the opt-in workaround for the current release. +- Land the proper fix (Option B) as **default** behavior β€” safe per the cache-key-stability argument above. +- Remove the `ExcludesFile` guard and the flag once the proper fix ships. + +## Testing + +- The existing `Test_ExcludesFile_AbsoluteOutsideBuildContext` unit test should be repointed: it currently asserts `ExcludesFile` behavior under the flag; with the proper fix the relevant assertion moves to the cache-key layer (a `populateCompositeKey`/`AddPath` test that a `COPY --from` source is hashed regardless of the build context's `.dockerignore`). +- `integration.TestCacheInvalidatesOnAllowlistedFileChange` should pass **without** `FF_KANIKO_SCOPED_DOCKERIGNORE` once the proper fix is default; drop the flag from `KanikoEnv` at that point. +- Add a cache-key-stability regression: for a build whose `.dockerignore` does **not** match the `/kaniko/...` source, assert the composite key is unchanged versus the pre-fix computation, to prove existing caches are not invalidated. + +## Blast radius + +Changes the cache-key computation path for `COPY --from`. The stability argument bounds the behavioral change to the allowlist case, but the change touches code shared by every cached build, so it warrants the integration cache suite plus a key-stability assertion before defaulting. diff --git a/docs/security_findings/gzip-bomb-no-size-cap.md b/docs/security_findings/gzip-bomb-no-size-cap.md new file mode 100644 index 000000000..1f0e26fc6 --- /dev/null +++ b/docs/security_findings/gzip-bomb-no-size-cap.md @@ -0,0 +1,32 @@ +# Gzip-bomb / no size cap on layer extraction + +**Severity:** Medium β€” DoS via disk-exhaustion / build-time blowout. +**Affects:** `pkg/util/tar_util.go:UnpackLocalTarArchive` and `pkg/util/fs_util.go:GetFSFromLayers`. Neither applies a maximum-size cap when extracting a compressed tar. +**Reachable via:** any malicious base image's layer or a user-supplied context tar. + +## Demonstration + +A 51 KB gzipped tar containing one zero-filled file expanded to 52 MB on disk in 60 ms. Compression ratio: **1027Γ—**. Scaling that to a few MB of attacker input fills a host disk in seconds. + +``` +compressed=51049 bytes uncompressed=52430336 bytes ratio=1027x +extracted size = 52428800 bytes +DECOMPRESSION-BOMB DoS: 51049 bytes gzip β†’ 52428800 bytes extracted (ratio 1027x). +kaniko has no size limit; a kilobyte-scale gzip can fill the build host's disk. +``` + +Reproducer: `pkg/util/repro_zipbomb_test.go` β†’ `TestGzipBombNoSizeLimit`. + +## Suggested fix + +Wrap the layer reader in an `io.LimitReader` with a configurable max-extracted-size limit (e.g. `--max-extracted-size` flag, default a few GB). Refuse layers that exceed the limit: + +```go +const defaultMaxExtractedSize = 10 << 30 // 10 GiB; CLI-overridable +r := io.LimitReader(layerReader, defaultMaxExtractedSize+1) +// after extraction, error if bytes read > defaultMaxExtractedSize +``` + +## Disclosure + +Borderline β€” listed as DoS, comparable to crashes #2/#3/#5/#6/#7 already filed as public issues. Per `SECURITY.md` the conservative path is a private security advisory; the practical path matches what other DoS-via-malicious-base-image findings did. diff --git a/docs/security_findings/symlink-confinement-primitives.md b/docs/security_findings/symlink-confinement-primitives.md new file mode 100644 index 000000000..e2f1c4e80 --- /dev/null +++ b/docs/security_findings/symlink-confinement-primitives.md @@ -0,0 +1,107 @@ +# Symlink-confinement primitives in ExtractFile (#9–#12) + +**Severity:** HIGH β€” arbitrary file delete, read, symlink creation, and directory creation as root on the build host. +**Affects:** `pkg/util/fs_util.go` β€” `ExtractFile` (every TypeFlag branch that takes a `path` argument computed via `filepath.Join(dest, …)`) and the whiteout handler in `GetFSFromLayers`. +**Same root cause as `tar-slip-via-symlink.md` (no symlink validation in `ExtractFile`). Same fix (securejoin + `O_NOFOLLOW`).** But each is a distinct attack primitive that demonstrates a different class of damage, so they each deserve a test in the fix PR's regression suite. + +Discovered by systematically probing every operation `ExtractFile` performs that takes a `path` argument. Each inherits the same vulnerability: a previously-planted symlink in the path causes the kernel to follow it during the syscall. + +## #9 β€” Arbitrary file delete via whiteout + planted symlink + +**Where:** `pkg/util/fs_util.go:212` (`os.RemoveAll(path)` in the whiteout handler in `GetFSFromLayers`). + +**Trigger:** +``` +SYMLINK evil β†’ /var/log/ (any existing host dir) +REGFILE evil/.wh.audit.log (whiteout for "audit.log") +``` +Layer extraction computes `path := filepath.Join("dest/evil", "audit.log")` β†’ resolves through symlink β†’ `os.RemoveAll("/var/log/audit.log")`. + +**Confirmed:** `TestWhiteoutArbitraryDeleteViaSymlink` β€” file written into `external/victim.txt` then deleted via whiteout entry. + +``` +ARBITRARY DELETE via whiteout+symlink: ".../victim.txt" was deleted outside dest +``` + +## #10 β€” Arbitrary host file read via hardlink + planted symlink + +**Where:** `pkg/util/fs_util.go:409` (`os.Link(link, path)` in the `TypeLink` branch). + +**Trigger:** +``` +SYMLINK evil β†’ /etc +TypeLink leak β†’ evil/passwd (Linkname resolves through planted symlink) +``` +kaniko computes `link := filepath.Clean(filepath.Join(dest, "evil/passwd"))` β†’ kernel `link()` follows the planted symlink β†’ hardlink to `/etc/passwd` lands at `dest/leak`. The build now has a fully-readable alias of `/etc/passwd`, which can be COPY'd into the resulting image or read by any RUN step. + +**Confirmed:** `TestHardlinkLeakViaSymlink` β€” `dest/leak` content reads back as `"CONFIDENTIAL"` (the external secret's content). + +``` +HARDLINK LEAK via planted symlink: dest/leak now mirrors ".../secret" (content="CONFIDENTIAL"). +A malicious layer can read arbitrary host files into the build's filesystem. +``` + +## #11 β€” Arbitrary symlink creation outside dest via planted symlink + +**Where:** `pkg/util/fs_util.go:426` (`os.Symlink(hdr.Linkname, path)` in the `TypeSymlink` branch β€” and its preceding `os.MkdirAll(dir, …)`). + +**Trigger:** +``` +SYMLINK evil β†’ /tmp/host-target +SYMLINK evil/sneaky β†’ /anything (creates host-target/sneaky on the host) +``` + +**Confirmed:** `TestSymlinkChainCreatesSymlinkOutsideDest` β€” `external/sneaky` symlink created on the host with attacker-controlled `Linkname`. Stepping stone for further attacks during subsequent build steps. + +## #12 β€” Arbitrary directory creation outside dest with attacker mode bits + +**Where:** `pkg/util/fs_util.go:374-381` (`MkdirAllWithPermissions(path, mode, uid, gid)` and `os.Chmod(path, mode)` in the `TypeDir` branch). + +**Trigger:** +``` +SYMLINK evil β†’ /tmp/host-target +TypeDir evil/newdir mode=0o777 +``` +`MkdirAll` follows the planted symlink and creates `/tmp/host-target/newdir` with mode 0o777 (world-writable, attacker-chosen). With setuid/setgid bits in mode, even nastier. + +**Confirmed:** `TestMkdirOutsideDestViaSymlink`: + +``` +DIRECTORY CREATED OUTSIDE DEST: ".../newdir" (mode=drwxrwxrwx) was created on the host via planted symlink +``` + +## Combined impact + +All five primitives (the original tar-slip-write plus #9–#12) are reachable from any malicious base image layer tar with no path traversal in entry names (`../` is correctly blocked). The attacker only needs: + +1. One symlink entry with `Linkname` set to an existing absolute path (or any target that resolves to one after subsequent layer extraction) +2. A subsequent entry whose `Name` contains that symlink's name as a directory component + +| Primitive | Mechanism | Reference | +|---|---|---| +| File WRITE | TypeReg β†’ `os.Create` follows symlink | `tar-slip-via-symlink.md` | +| File DELETE | Whiteout β†’ `os.RemoveAll` follows symlink | #9 | +| File READ into build | TypeLink β†’ `os.Link` follows symlink | #10 | +| Symlink CREATE outside | TypeSymlink β†’ `os.Symlink`'s MkdirAll follows | #11 | +| Directory CREATE outside (attacker mode) | TypeDir β†’ `MkdirAll`+`Chmod` follow | #12 | + +## Single suggested fix + +Wrap every path computation in `ExtractFile` (and the whiteout handler in `GetFSFromLayers`) with `securejoin.SecureJoin(dest, cleanedName)` β€” which resolves symlinks and rejects paths that escape `dest`. Plus open the regular-file write with `O_NOFOLLOW` as defense-in-depth. + +```go +// Pseudocode for the fix: +safePath, err := securejoin.SecureJoin(dest, cleanedName) +if err != nil { return err } +// And for TypeSymlink: also validate the Linkname target wouldn't escape: +safeTarget, err := securejoin.SecureJoin(filepath.Dir(safePath), hdr.Linkname) +if err != nil { return fmt.Errorf("symlink target escapes dest: %w", err) } +// And for TypeReg writes: +f, err := os.OpenFile(safePath, os.O_CREATE|os.O_WRONLY|os.O_TRUNC|syscall.O_NOFOLLOW, mode) +``` + +The existing tests at `fs_util_test.go:875` and `:899` already encode the expected behavior using securejoin terminology in their comments β€” they just don't pass against the current production code. + +## Disclosure + +Not yet reported. Per `SECURITY.md`, file via GitHub security advisory at https://github.com/osscontainertools/kaniko/security/advisories β€” do not file as a public issue. Bundle with `tar-slip-via-symlink.md` (same root cause, same fix). diff --git a/docs/security_findings/tar-slip-via-symlink.md b/docs/security_findings/tar-slip-via-symlink.md new file mode 100644 index 000000000..bb5350afa --- /dev/null +++ b/docs/security_findings/tar-slip-via-symlink.md @@ -0,0 +1,107 @@ +# Tar-slip via symlink: arbitrary file write during image unpacking + +**Severity:** HIGH β€” arbitrary file write as root on the build host. +**Affects:** `pkg/util/fs_util.go` β€” `ExtractFile` (TypeSymlink branch, no Linkname validation) and the surrounding loop in `GetFSFromImage` / `UnTar`. +**Reproducible:** Yes, 100%. +**Pattern:** classic *tar slip via symlink* (CVE-2019-14271-class). + +## Exploit recipe + +A malicious base image's layer tar contains: + +``` +SYMLINK evil β†’ /tmp (Typeflag = SYMTYPE, any *existing* abs path) +REGFILE evil/PWNED "content" (Typeflag = REGTYPE) +``` + +When kaniko extracts the layer: + +1. `ExtractFile` for the symlink: `os.Symlink("/tmp", dest+"/evil")` β€” no validation of Linkname. +2. `ExtractFile` for the regular file: + - computes `path = dest + "/evil/PWNED"` + - `os.Stat(dir)` follows the symlink to `/tmp/` β€” succeeds (target exists) + - `os.Create(path)` follows the symlink during write β€” writes `/tmp/PWNED` on the build host. + +Demonstrated end-to-end: a layer tar containing `(symlink "evil" β†’ "/tmp", file "evil/PWNED_BY_KANIKO_FUZZ")` fed to `util.UnTar` results in `/tmp/PWNED_BY_KANIKO_FUZZ` existing after the call returns β€” outside the destination directory. + +## Reproduce + +```bash +go test ./pkg/util/ -run='TestTarSlipViaSymlinkEscape' -v +go test ./pkg/util/ -run='TestTarSlipVariants' -v +``` + +Output of the variants test: + +``` +=== RUN TestTarSlipVariants/absolute_target + ESCAPE: file written outside dest in external: "/tmp/.../leaked.txt" +--- FAIL: TestTarSlipVariants/absolute_target +--- PASS: TestTarSlipVariants/parent_traversal_symlink (linkname=../escaped β€” target doesn't exist before) +--- PASS: TestTarSlipVariants/deep_traversal_symlink (same) +``` + +Key insight from the variants: the escape fires when the symlink target is an existing absolute path. Targets like `../escaped` (non-existent) trip `MkdirAll` on the symlink and the write fails. But the attacker can pick any pre-existing path β€” `/`, `/etc`, `/tmp`, `/usr/local/bin`, `/kaniko/`, … + +## Production impact + +Kaniko's executor runs as uid 0 inside the build container, with the host filesystem mounted as the build root. A malicious base image can therefore: + +- Overwrite `/etc/passwd`, `/etc/shadow`, `/etc/ssh/...` +- Plant a malicious binary at `/usr/local/bin/` to shadow legitimate system tools +- Overwrite `/kaniko/executor` itself, affecting subsequent stages / cached builds +- Modify `/proc/sys/...` to alter kernel-level behavior +- Anywhere a path stat()s successfully and the process has write perms + +The two vulnerable entry points cover both image extraction and Dockerfile-driven tar handling: + +| Caller | Function | Reach | +|---|---|---| +| Base image extraction | `GetFSFromImage` β†’ `ExtractFile` | Any `FROM ` | +| COPY/ADD local tar | `UnpackLocalTarArchive` β†’ `UnTar` β†’ `ExtractFile` | Any user-provided context tar | + +## Root cause + +```go +// pkg/util/fs_util.go, ExtractFile, TypeSymlink branch: +case tar.TypeSymlink: + ... + if err := os.Symlink(hdr.Linkname, path); err != nil { + return err + } +``` + +`hdr.Linkname` is written verbatim with no path-prefix check. Compare to `TypeLink` (hardlink) where there IS a `cleanedLink == ".." || strings.HasPrefix("../")` check β€” but that check only catches relative-parent escapes; an absolute target like `/etc` still slips through (it just produces an `os.Link` to a non-existent path under dest, so the hardlink fails harmlessly). The symlink path doesn't even have the relative-parent check. + +## Suggested fix + +Resolve symlinks during extraction with a chroot-like guard. The simplest variant: + +```go +case tar.TypeSymlink: + // Refuse absolute targets. + if filepath.IsAbs(hdr.Linkname) { + return fmt.Errorf("symlink %q has absolute target %q; refusing", hdr.Name, hdr.Linkname) + } + // Resolve the link's intended target relative to its containing directory + // and refuse anything that escapes dest. + abs := filepath.Clean(filepath.Join(filepath.Dir(path), hdr.Linkname)) + if !strings.HasPrefix(abs+string(os.PathSeparator), dest+string(os.PathSeparator)) && abs != dest { + return fmt.Errorf("symlink %q target %q escapes %q", hdr.Name, hdr.Linkname, dest) + } + if err := os.Symlink(hdr.Linkname, path); err != nil { + return err + } +``` + +Additionally, the regular-file write path should `os.OpenFile(path, O_NOFOLLOW|O_CREATE|O_WRONLY|O_TRUNC, ...)` (or equivalent on Linux) so that even if a symlink escapes the symlink-creation check, the subsequent file write won't follow it. Defense-in-depth: validate both the link creation and the file open. + +Tools like `securejoin.SecureJoin` (`github.com/cyphar/filepath-securejoin`) exist specifically for this. Recommend adopting it across `ExtractFile`, `UnTar`, `GetFSFromLayers`, and any other code that takes attacker-influenced relative paths. + +## Related + +See `symlink-confinement-primitives.md` β€” four additional attack primitives (file delete, file read, symlink creation, directory creation outside dest) share this same root cause and can be fixed together. + +## Disclosure + +Not yet reported. Per `SECURITY.md`, file via GitHub security advisory at https://github.com/osscontainertools/kaniko/security/advisories β€” do not file as a public issue. diff --git a/golden/golden_test.go b/golden/golden_test.go index 308fc4fd6..28bbd44e1 100644 --- a/golden/golden_test.go +++ b/golden/golden_test.go @@ -29,6 +29,7 @@ import ( "github.com/google/go-cmp/cmp" v1 "github.com/google/go-containerregistry/pkg/v1" "github.com/osscontainertools/kaniko/cmd/executor/cmd" + testbake "github.com/osscontainertools/kaniko/golden/testdata/test_bake" testissuemz195 "github.com/osscontainertools/kaniko/golden/testdata/test_issue_mz195" testissuemz333 "github.com/osscontainertools/kaniko/golden/testdata/test_issue_mz333" testissuemz334 "github.com/osscontainertools/kaniko/golden/testdata/test_issue_mz334" @@ -97,6 +98,42 @@ func TestMain(m *testing.M) { os.Exit(exitCode) } +func renderPlan(t *testing.T, opts *config.KanikoOptions, cachedKeys []string) string { + t.Helper() + origNewLayerCache := executor.NewLayerCache + executor.NewLayerCache = func(_ *config.KanikoOptions) cache.LayerCache { + return &fakeLayerCache{cachedKeys: cachedKeys} + } + t.Cleanup(func() { executor.NewLayerCache = origNewLayerCache }) + + var buf bytes.Buffer + executor.Out = &buf + if _, err := executor.DoBuild(opts); err != nil { + t.Error(err) + } + return buf.String() +} + +// comparePlan diffs output against the golden plan at planPath, or rewrites it +// when -update is set. +func comparePlan(t *testing.T, planPath, output string) { + t.Helper() + if update { + if err := os.WriteFile(planPath, []byte(output), 0o644); err != nil { + t.Fatal(err) + } + return + } + expectedPlan, err := os.ReadFile(planPath) + if err != nil { + t.Fatal(err) + } + expected := strings.Trim(string(expectedPlan), "\n") + if diff := cmp.Diff(expected, strings.Trim(output, "\n")); diff != "" { + t.Errorf("plan mismatch (-expected +got):\n%s", diff) + } +} + func TestRun(t *testing.T) { logrus.SetLevel(logrus.WarnLevel) @@ -113,11 +150,6 @@ func TestRun(t *testing.T) { } opts := config.KanikoOptions{} - origNewLayerCache := executor.NewLayerCache - executor.NewLayerCache = func(_ *config.KanikoOptions) cache.LayerCache { - return &fakeLayerCache{cachedKeys: test.CachedKeys} - } - t.Cleanup(func() { executor.NewLayerCache = origNewLayerCache }) exec := &cobra.Command{ Use: "kaniko", } @@ -133,31 +165,56 @@ func TestRun(t *testing.T) { } cmd.ValidateFlags(&opts) - var buf bytes.Buffer - executor.Out = &buf - _, err = executor.DoBuild(&opts) - if err != nil { - t.Error(err) + output := renderPlan(t, &opts, test.CachedKeys) + comparePlan(t, filepath.Join(testDir, "plans", test.Plan), output) + }) + } + }) + } + }) + } +} + +var bakeTests = map[string][]types.GoldenTests{ + "test_bake": {testbake.Tests}, +} + +func TestBake(t *testing.T) { + logrus.SetLevel(logrus.WarnLevel) + + for testName, testSuites := range bakeTests { + t.Run(testName, func(t *testing.T) { + testDir := filepath.Join("testdata", testName) + for _, testSuite := range testSuites { + t.Run(testSuite.Name, func(t *testing.T) { + for _, test := range testSuite.Tests { + t.Run(renderCommand(test.Env, test.Args), func(t *testing.T) { + for k, v := range test.Env { + t.Setenv(k, v) } - planPath := filepath.Join(testDir, "plans", test.Plan) - if update { - err = os.WriteFile(planPath, buf.Bytes(), 0o644) - if err != nil { - t.Fatal(err) - } - } else { - output := strings.Trim(buf.String(), "\n") - expectedPlan, err := os.ReadFile(planPath) - if err != nil { - t.Fatal(err) - } - expected := strings.Trim(string(expectedPlan), "\n") - - if diff := cmp.Diff(expected, output); diff != "" { - t.Errorf("plan mismatch (-expected +got):\n%s", diff) - } + opts := config.KanikoOptions{} + var set []string + exec := &cobra.Command{Use: "bake"} + cmd.AddBakeFlags(exec, &opts, &set) + args := []string{ + filepath.Join(testDir, "bake.json"), + "--dryrun", + "--dockerfile=" + filepath.Join(testDir, testSuite.Dockerfile), + } + args = append(args, test.Args...) + if err := exec.ParseFlags(args); err != nil { + t.Fatal(err) } + cmd.ValidateFlags(&opts) + + rest := exec.Flags().Args() + if err := cmd.ConfigureFromBakefile(&opts, rest[0], rest[1:], set); err != nil { + t.Fatal(err) + } + + output := renderPlan(t, &opts, test.CachedKeys) + comparePlan(t, filepath.Join(testDir, "plans", test.Plan), output) }) } }) diff --git a/golden/testdata/test_bake/Dockerfile b/golden/testdata/test_bake/Dockerfile new file mode 100644 index 000000000..fbb3ffcf2 --- /dev/null +++ b/golden/testdata/test_bake/Dockerfile @@ -0,0 +1,8 @@ +FROM alpine AS base +RUN install + +FROM base AS app +RUN build app + +FROM base AS tools +RUN build tools diff --git a/golden/testdata/test_bake/bake.json b/golden/testdata/test_bake/bake.json new file mode 100644 index 000000000..83c605a81 --- /dev/null +++ b/golden/testdata/test_bake/bake.json @@ -0,0 +1,13 @@ +{ + "version": "1", + "targets": { + "app": { + "target": "app", + "destination": ["registry.example.com/app:latest", "registry.example.com/app:v1"] + }, + "tools": { + "target": "tools", + "destination": ["registry.example.com/tools:latest"] + } + } +} diff --git a/golden/testdata/test_bake/plans/app b/golden/testdata/test_bake/plans/app new file mode 100644 index 000000000..66eb845f7 --- /dev/null +++ b/golden/testdata/test_bake/plans/app @@ -0,0 +1,5 @@ +FROM alpine AS app +UNPACK alpine +RUN install +RUN build app +PUSH [registry.example.com/app:latest registry.example.com/app:v1] diff --git a/golden/testdata/test_bake/plans/app_override b/golden/testdata/test_bake/plans/app_override new file mode 100644 index 000000000..98599b9d3 --- /dev/null +++ b/golden/testdata/test_bake/plans/app_override @@ -0,0 +1,5 @@ +FROM alpine AS app +UNPACK alpine +RUN install +RUN build app +PUSH [registry.example.com/app:override] diff --git a/golden/testdata/test_bake/plans/tools b/golden/testdata/test_bake/plans/tools new file mode 100644 index 000000000..d700c33d4 --- /dev/null +++ b/golden/testdata/test_bake/plans/tools @@ -0,0 +1,5 @@ +FROM alpine AS tools +UNPACK alpine +RUN install +RUN build tools +PUSH [registry.example.com/tools:latest] diff --git a/golden/testdata/test_bake/test.go b/golden/testdata/test_bake/test.go new file mode 100644 index 000000000..dde1ea741 --- /dev/null +++ b/golden/testdata/test_bake/test.go @@ -0,0 +1,22 @@ +package testbake + +import "github.com/osscontainertools/kaniko/golden/types" + +var Tests = types.GoldenTests{ + Name: "test_bake", + Dockerfile: "Dockerfile", + Tests: []types.GoldenTest{ + { + Args: []string{"app"}, + Plan: "app", + }, + { + Args: []string{"tools"}, + Plan: "tools", + }, + { + Args: []string{"app", "--set", "app.destination=registry.example.com/app:override"}, + Plan: "app_override", + }, + }, +} diff --git a/hub-sync.md b/hub-sync.md new file mode 100644 index 000000000..f9b7e494a --- /dev/null +++ b/hub-sync.md @@ -0,0 +1,186 @@ +This is the release channel of https://github.com/osscontainertools/kaniko +Supported Tags +--- +* [latest](https://hub.docker.com/layers/martizih/kaniko/latest/images/sha256-95779f52d460ca70b65a4f4679d4f5163ad6c33edea3303dc8bbff847de5e05c): The standard kaniko image +* [slim](https://hub.docker.com/layers/martizih/kaniko/slim/images/sha256-94c88af15f7e1d1eb2b581d3578eccbed547e4f31333330e5329796909c0330b): An image variant without the credential-helpers, for the security minded +* [debug](https://hub.docker.com/layers/martizih/kaniko/debug/images/sha256-9047d4477bea3db093de3362f1c948cc86421e8403b7a720e2d490fb5a8aa7b2): An image variant with a barebones shell - use this for gitlab-ci +* [alpine](https://hub.docker.com/layers/martizih/kaniko/alpine/images/sha256-795a358f6c22a9fcd66bb7e14bd97728155e1c171ca951f3c3ba6501054234ce): An alpine-based image variant +* [warmer](https://hub.docker.com/layers/martizih/kaniko/warmer/images/sha256-28a84232bad1f7301973cfb1ca29d7c0f559bc80ca892b2e239afcdcbeea2e6a): The warmer image, used for preheating image cache +* [bootstrap](https://hub.docker.com/layers/martizih/kaniko/bootstrap/images/sha256-bdd3a62ca64487e691a3aafedb3759c8552399a69e645c0407b732fa94aa951e): The bootstrap image, to build kaniko itself + +v1.27.x +--- +* v1.27.6 ([05.06.2026](https://github.com/osscontainertools/kaniko/releases/tag/v1.27.6)): +[v1.27.6](https://hub.docker.com/layers/martizih/kaniko/v1.27.6/images/sha256-95779f52d460ca70b65a4f4679d4f5163ad6c33edea3303dc8bbff847de5e05c) , +[v1.27.6-slim](https://hub.docker.com/layers/martizih/kaniko/v1.27.6-slim/images/sha256-94c88af15f7e1d1eb2b581d3578eccbed547e4f31333330e5329796909c0330b) , +[v1.27.6-debug](https://hub.docker.com/layers/martizih/kaniko/v1.27.6-debug/images/sha256-9047d4477bea3db093de3362f1c948cc86421e8403b7a720e2d490fb5a8aa7b2) , +[v1.27.6-alpine](https://hub.docker.com/layers/martizih/kaniko/v1.27.6-alpine/images/sha256-795a358f6c22a9fcd66bb7e14bd97728155e1c171ca951f3c3ba6501054234ce) , +[v1.27.6-warmer](https://hub.docker.com/layers/martizih/kaniko/v1.27.6-warmer/images/sha256-28a84232bad1f7301973cfb1ca29d7c0f559bc80ca892b2e239afcdcbeea2e6a) , +[v1.27.6-bootstrap](https://hub.docker.com/layers/martizih/kaniko/v1.27.6-bootstrap/images/sha256-bdd3a62ca64487e691a3aafedb3759c8552399a69e645c0407b732fa94aa951e) + +* v1.27.5 ([15.05.2026](https://github.com/osscontainertools/kaniko/releases/tag/v1.27.5)): +[v1.27.5](https://hub.docker.com/layers/martizih/kaniko/v1.27.5/images/sha256-670d2dafb33ee9591e1ce39258ca313fe81aa99d2686da9c37b9f971e4bea164) , +[v1.27.5-slim](https://hub.docker.com/layers/martizih/kaniko/v1.27.5-slim/images/sha256-703116531bcd63eeb27ac9b9cc61c4689107bc90b0669a27ad57bcf59426c736) , +[v1.27.5-debug](https://hub.docker.com/layers/martizih/kaniko/v1.27.5-debug/images/sha256-6f2a60489f03ce7ab78240c82c27358cfa394cfa7b91ee6b0bb72fc66f21c520) , +[v1.27.5-alpine](https://hub.docker.com/layers/martizih/kaniko/v1.27.5-alpine/images/sha256-6e5f9ed4280c272a93c61e90ea5038b09f5e0ac75f264fb5ab1e4d875b2d1130) , +[v1.27.5-warmer](https://hub.docker.com/layers/martizih/kaniko/v1.27.5-warmer/images/sha256-3b9a90a0ad4dcd06b70205ad953d5ba01ef5dc98dbb23f89e1b678e3069b28f0) , +[v1.27.5-bootstrap](https://hub.docker.com/layers/martizih/kaniko/v1.27.5-bootstrap/images/sha256-7ddfaf93d19461592c42557e76020280d7934275eafe56c90d0fd42daa9634f9) + +* v1.27.4 ([30.04.2026](https://github.com/osscontainertools/kaniko/releases/tag/v1.27.4)): +[v1.27.4](https://hub.docker.com/layers/martizih/kaniko/v1.27.4/images/sha256-24eb1a4a320dbb759469eb7caa47c9eeaed8542d4ebce40100fec20ef9eda323) , +[v1.27.4-slim](https://hub.docker.com/layers/martizih/kaniko/v1.27.4-slim/images/sha256-3e77b1b5e131aa409284ed435e666ca863e825fbbf2cf43cc149053983bebece) , +[v1.27.4-debug](https://hub.docker.com/layers/martizih/kaniko/v1.27.4-debug/images/sha256-a394e5bd9c75c93e207b3949f3d7f49fbe59f21ea91d96ba6939ddf6226233f8) , +[v1.27.4-alpine](https://hub.docker.com/layers/martizih/kaniko/v1.27.4-alpine/images/sha256-c9566b51713bdc74aa345e437a71de8c3ee2f37100209590e99cd898d4cae487) , +[v1.27.4-warmer](https://hub.docker.com/layers/martizih/kaniko/v1.27.4-warmer/images/sha256-7c37a551e60384183acf3c4812948ea65ab03576345006929506c6791b93f490) , +[v1.27.4-bootstrap](https://hub.docker.com/layers/martizih/kaniko/v1.27.4-bootstrap/images/sha256-70c728cd79974a476ca489efcd26b10d0f75506f21af603cb0d1760631da8a3e) + +* v1.27.3 ([17.04.2026](https://github.com/osscontainertools/kaniko/releases/tag/v1.27.3)): +[v1.27.3](https://hub.docker.com/layers/martizih/kaniko/v1.27.3/images/sha256-6965f7a5dfdff726b4019ec54411fdba054836c7d510733a48f72b1f0fa58b96) , +[v1.27.3-slim](https://hub.docker.com/layers/martizih/kaniko/v1.27.3-slim/images/sha256-27fe74f83182e34611a0eceae6f03127fdede5fc21daca53b08dcd9181d38c5b) , +[v1.27.3-debug](https://hub.docker.com/layers/martizih/kaniko/v1.27.3-debug/images/sha256-a7e2df20e8ee98874f9856a04622de6a76e93609f7e5cba5949eb5436959720c) , +[v1.27.3-warmer](https://hub.docker.com/layers/martizih/kaniko/v1.27.3-warmer/images/sha256-39ec36f66bf96d739af643c97a9941f63ca6108a32b64d1911a2d12f04cc6c5f) , +[v1.27.3-bootstrap](https://hub.docker.com/layers/martizih/kaniko/v1.27.3-bootstrap/images/sha256-f7b8efac08b36883b7e86083d3931bd1fa6fdc571d3a9a3157f7c7df2aaf3423) + +* v1.27.2 ([02.04.2026](https://github.com/osscontainertools/kaniko/releases/tag/v1.27.2)): +[v1.27.2](https://hub.docker.com/layers/martizih/kaniko/v1.27.2/images/sha256-fabef65012d66590736fde3aabc10614cba1ce39ee9ea805282b1065a2aae840) , +[v1.27.2-slim](https://hub.docker.com/layers/martizih/kaniko/v1.27.2-slim/images/sha256-be27e3e3beffdc08cb290dc7637071b3b019abaeac8e34188f971f337738ac83) , +[v1.27.2-debug](https://hub.docker.com/layers/martizih/kaniko/v1.27.2-debug/images/sha256-e6758dd08d8fa31707ace2cb72d3ba437667fe5c709101e48223f655b58b27bd) , +[v1.27.2-warmer](https://hub.docker.com/layers/martizih/kaniko/v1.27.2-warmer/images/sha256-61dc779302212865a8221f84bc31690c0f17cb8f23951ae287c5e80797dbe7b3) , +[v1.27.2-bootstrap](https://hub.docker.com/layers/martizih/kaniko/v1.27.2-bootstrap/images/sha256-d32a277ff99dd7d13dc1fe35f2ea8b42ec0d8d3dcd70a50b5be31f24f325b930) + +* v1.27.1 ([19.03.2026](https://github.com/osscontainertools/kaniko/releases/tag/v1.27.1)): +[v1.27.1](https://hub.docker.com/layers/martizih/kaniko/v1.27.1/images/sha256-75346bea5ef04eac0dd70d1593daea19db34af00cf27f060c7cfa8a172f589dd) , +[v1.27.1-slim](https://hub.docker.com/layers/martizih/kaniko/v1.27.1-slim/images/sha256-0cd05ca3062f88715c3c8011e79525bdb37510b9bf2d72a6e6853618eb962a47) , +[v1.27.1-debug](https://hub.docker.com/layers/martizih/kaniko/v1.27.1-debug/images/sha256-4ce0a4af1479f27f66ca170c287215d3b51e61153fa8b87f6bce5f2f791f5e7e) , +[v1.27.1-warmer](https://hub.docker.com/layers/martizih/kaniko/v1.27.1-warmer/images/sha256-e0d31e770c1c78f325bae4032c0a940de2aeb894910e2c1d0032565525492066) , +[v1.27.1-bootstrap](https://hub.docker.com/layers/martizih/kaniko/v1.27.1-bootstrap/images/sha256-5590d18338e777dab3e447474745ce9dc954ffabd060d62d6b04c47298489bae) + +* v1.27.0 ([05.03.2026](https://github.com/osscontainertools/kaniko/releases/tag/v1.27.0)): +[v1.27.0](https://hub.docker.com/layers/martizih/kaniko/v1.27.0/images/sha256-33c366e9733cef1ddec732d8b6eb9aaba73141a08537bddaf23d75f101c46631) , +[v1.27.0-slim](https://hub.docker.com/layers/martizih/kaniko/v1.27.0-slim/images/sha256-8ea2be116b2de2c3cb5eb8695b87e46a5258dd4ffc025348538d7aff7e52fd61) , +[v1.27.0-debug](https://hub.docker.com/layers/martizih/kaniko/v1.27.0-debug/images/sha256-c6acf2ef8593502b122344bcf23e60ba0f1a25e8b8fee763dcb524504613c367) , +[v1.27.0-warmer](https://hub.docker.com/layers/martizih/kaniko/v1.27.0-warmer/images/sha256-8e029bb1f9489ecd6058244df9897b48c731c439a88b642c44c8a1e076977543) , +[v1.27.0-bootstrap](https://hub.docker.com/layers/martizih/kaniko/v1.27.0-bootstrap/images/sha256-eff42bc71ae426e05aa057afef080db157270c816cca5a95c519bf36af07c0f6) + +v1.26.x +--- +* v1.26.6 ([19.02.2026](https://github.com/osscontainertools/kaniko/releases/tag/v1.26.6)): +[v1.26.6](https://hub.docker.com/layers/martizih/kaniko/v1.26.6/images/sha256-d6d8caa7067ba4113e079dddc7f5c5efc51b18606e29db7f83d8973e36aefb58) , +[v1.26.6-slim](https://hub.docker.com/layers/martizih/kaniko/v1.26.6-slim/images/sha256-eb1725e664ca06e8fc22840c0ea0a1df115ba73bb92657ba2b140c8cbb083d04) , +[v1.26.6-debug](https://hub.docker.com/layers/martizih/kaniko/v1.26.6-debug/images/sha256-d2170bed1affc9d886853691c5ae7aa36163f2643de42693eb890bad8d1763ad) , +[v1.26.6-warmer](https://hub.docker.com/layers/martizih/kaniko/v1.26.6-warmer/images/sha256-6be19182368b5bcd41050ee91fb708fbe7f86e7f53d9cb6f14e02833234ffefe) , +[v1.26.6-bootstrap](https://hub.docker.com/layers/martizih/kaniko/v1.26.6-bootstrap/images/sha256-131695ac426856b352ef6c9c7aa8f458727237fba7b0bbe2f909fcc6bb550f78) + +* v1.26.5 ([05.02.2026](https://github.com/osscontainertools/kaniko/releases/tag/v1.26.5)): +[v1.26.5](https://hub.docker.com/layers/martizih/kaniko/v1.26.5/images/sha256-22749894e70216ba32fbe6d99ea0d39542cff73be8a723caa6c0dd5378031a44) , +[v1.26.5-slim](https://hub.docker.com/layers/martizih/kaniko/v1.26.5-slim/images/sha256-7674b25efe4fb7ee0d52d6f61300abb7f3575bc10da18ac48c1a12772e5a41f6) , +[v1.26.5-debug](https://hub.docker.com/layers/martizih/kaniko/v1.26.5-debug/images/sha256-e402135385d718516d6f9b45c412d79108e320b3f1147784c5514ef151300d43) , +[v1.26.5-warmer](https://hub.docker.com/layers/martizih/kaniko/v1.26.5-warmer/images/sha256-05ee9df4f464751f33b2189b446bfaf1f36715054e6caab43f8b000a43d6d778) , +[v1.26.5-bootstrap](https://hub.docker.com/layers/martizih/kaniko/v1.26.5-bootstrap/images/sha256-4f4a7848c3755633a292db6afbb5aa880243349e14e2b3a04a1d35b6cb8f3b97) + +* v1.26.4 ([09.01.2026](https://github.com/osscontainertools/kaniko/releases/tag/v1.26.4)): +[v1.26.4](https://hub.docker.com/layers/martizih/kaniko/v1.26.4/images/sha256-c6ce4bad444f3622d003ff6401d459d608003dda8d90c1a70110a29711eda1d9) , +[v1.26.4-slim](https://hub.docker.com/layers/martizih/kaniko/v1.26.4-slim/images/sha256-1448ff34bd948326b18bbff8e551e69e1ac60fe39716f428579b1dc76b040eb8) , +[v1.26.4-debug](https://hub.docker.com/layers/martizih/kaniko/v1.26.4-debug/images/sha256-dddd2b776303620f0dc7962f16461ac764515f6b4648586e2bd8ac7e48d69b50) , +[v1.26.4-warmer](https://hub.docker.com/layers/martizih/kaniko/v1.26.4-warmer/images/sha256-582b73da2feb787cc10507b84e871346d19dea4ccaa81ce3455555a3da71ccc2) , +[v1.26.4-bootstrap](https://hub.docker.com/layers/martizih/kaniko/v1.26.4-bootstrap/images/sha256-ce5a1e3d8f15f647d9709410f770a38a611f498b1a26250a22ef704f69857f71) + +* v1.26.3 ([05.12.2025](https://github.com/osscontainertools/kaniko/releases/tag/v1.26.3)): +[v1.26.3](https://hub.docker.com/layers/martizih/kaniko/v1.26.3/images/sha256-56663681b60dbbe274fe5439b01c3119bb464e6d8e930b191eac3bf3b00f85ac) , +[v1.26.3-slim](https://hub.docker.com/layers/martizih/kaniko/v1.26.3-slim/images/sha256-478ad3290e65315f1411be574df2f79a9b15f039bea01dae85bd4f19bd3d5167) , +[v1.26.3-debug](https://hub.docker.com/layers/martizih/kaniko/v1.26.3-debug/images/sha256-4bfb139b641b04302f85f855e83ee1b551ad98812f41d35817b10ccd7a66ec87) , +[v1.26.3-warmer](https://hub.docker.com/layers/martizih/kaniko/v1.26.3-warmer/images/sha256-027c41245ad423b3c52aeed858e3780798e817681475a998ff55671c0cd6255f) , +[v1.26.3-bootstrap](https://hub.docker.com/layers/martizih/kaniko/v1.26.3-bootstrap/images/sha256-35a71cdfdea29aa86c4df5383c8b8b5f881ad684659057cddf2229f59d6179fa) + +* v1.26.2 ([20.11.2025](https://github.com/osscontainertools/kaniko/releases/tag/v1.26.2)): +[v1.26.2](https://hub.docker.com/layers/martizih/kaniko/v1.26.2/images/sha256-c188aa3cec374a425a0b44014c8ad1b8b188e61d70f2923e3b1d6ad0c6785868) , +[v1.26.2-slim](https://hub.docker.com/layers/martizih/kaniko/v1.26.2-slim/images/sha256-43e8265e792848751bf73c314a7f672559a13c48869224a64cd8edfbdc7ce706) , +[v1.26.2-debug](https://hub.docker.com/layers/martizih/kaniko/v1.26.2-debug/images/sha256-5ba27b429203d13890042fb98b91e15c1a421696beb6cf436134101897d925b3) , +[v1.26.2-warmer](https://hub.docker.com/layers/martizih/kaniko/v1.26.2-warmer/images/sha256-48e013d02d799abfc70dc08fc24fdd050ed55465a4a85b687f687dd3727ed1da) , +[v1.26.2-bootstrap](https://hub.docker.com/layers/martizih/kaniko/v1.26.2-bootstrap/images/sha256-ab608f93888e2062c0a4eb5e7276b57b4d8542969e187de6dc6a3d087f76a9f5) + +* v1.26.1 ([07.11.2025](https://github.com/osscontainertools/kaniko/releases/tag/v1.26.1)): +[v1.26.1](https://hub.docker.com/layers/martizih/kaniko/v1.26.1/images/sha256-d2b4bc7a4ed6922a0b5d3db2498e63afb4d4009ee6d4c6f04742ec8b8ad62f7a) , +[v1.26.1-slim](https://hub.docker.com/layers/martizih/kaniko/v1.26.1-slim/images/sha256-d8575a20bf395ad4c71845d73fb858e4a1f38cb29923e589d44aa439fcadb70b) , +[v1.26.1-debug](https://hub.docker.com/layers/martizih/kaniko/v1.26.1-debug/images/sha256-6a01e35fc43218b104cebaa63e1a41c811db7c69a5dcf37b80537ff41a2d08de) , +[v1.26.1-warmer](https://hub.docker.com/layers/martizih/kaniko/v1.26.1-warmer/images/sha256-15cd1723d2478554e20153f06ece6dd3f94862c4b049e3db5ce51c710b51f706) , +[v1.26.1-bootstrap](https://hub.docker.com/layers/martizih/kaniko/v1.26.1-bootstrap/images/sha256-c7c9e86c5961962c05f2350b3aa6cb77b0597c32d9ede4dfc567ca0266626ffa) + +* v1.26.0 ([16.10.2025](https://github.com/osscontainertools/kaniko/releases/tag/v1.26.0)): +[v1.26.0](https://hub.docker.com/layers/martizih/kaniko/v1.26.0/images/sha256-e7100b2777fe34b659e79a679b41569404ebb4c543968361655e25b4667786bf) , +[v1.26.0-slim](https://hub.docker.com/layers/martizih/kaniko/v1.26.0-slim/images/sha256-2d5e40e317f00e6c83cb84bf0640f0672261db0d34c6de97f2a277705850e3e2) , +[v1.26.0-debug](https://hub.docker.com/layers/martizih/kaniko/v1.26.0-debug/images/sha256-79e919821131a5d7846992de4c2fdb64bb1aa298a6cfd772a8740cda8648e545) , +[v1.26.0-warmer](https://hub.docker.com/layers/martizih/kaniko/v1.26.0-warmer/images/sha256-9dca8aca1570877f7b6e353da8da19b48dad858f459824023f43f3720b66985f) , +[v1.26.0-bootstrap](https://hub.docker.com/layers/martizih/kaniko/v1.26.0-bootstrap/images/sha256-63d95f9d64125efc157ed9fdfc40d063592f48278026ab7fa5ec99a58785d8ec) + +v1.25.x +--- +* v1.25.6 ([08.10.2025](https://github.com/osscontainertools/kaniko/releases/tag/v1.25.6)): +[v1.25.6](https://hub.docker.com/layers/martizih/kaniko/v1.25.6/images/sha256-037360d9d5adbcc3ef3c7dc69a29816c464235bb416e65aa5d828aa2bcb8da99) , +[v1.25.6-slim](https://hub.docker.com/layers/martizih/kaniko/v1.25.6-slim/images/sha256-4621d02d10dc2673bf01b1dbb8a46d325ead8474890c2136d95fbaaf013c65c9) , +[v1.25.6-debug](https://hub.docker.com/layers/martizih/kaniko/v1.25.6-debug/images/sha256-76ca529af8d11976676fc83c6fcb93427c9584bac283fd4d71a43e803bf07635) , +[v1.25.6-warmer](https://hub.docker.com/layers/martizih/kaniko/v1.25.6-warmer/images/sha256-d31778febacccd9d8470c6a25dd26b2011370b5062005b34cb7f0ec3a71178ec) , +[v1.25.6-bootstrap](https://hub.docker.com/layers/martizih/kaniko/v1.25.6-bootstrap/images/sha256-288f7b761a5a3dab847463ba41a2684c86604b41119d57918efa5bd9440be999) + +* v1.25.5 ([18.09.2025](https://github.com/mzihlmann/kaniko/releases/tag/v1.25.5)): +[v1.25.5](https://hub.docker.com/layers/martizih/kaniko/v1.25.5/images/sha256-ba8c21a95ef67f369b1a2cf6987f5b09eb8bd1802b233121ea5c4dbfa8b94428) , +[v1.25.5-slim](https://hub.docker.com/layers/martizih/kaniko/v1.25.5-slim/images/sha256-496d80c776aa29d6e59c7a00be12e7079520f4d6948405142fd2924ab99c8859) , +[v1.25.5-debug](https://hub.docker.com/layers/martizih/kaniko/v1.25.5-debug/images/sha256-b544f98f36c86fc9b7bd43edb3fa0e8766a8e4919795af50c597621949a91519) , +[v1.25.5-warmer](https://hub.docker.com/layers/martizih/kaniko/v1.25.5-warmer/images/sha256-3b5b11f3fd365864aa48bddee4ce7da9901b213f22e2261066699b8a39fd0cb0) , +[v1.25.5-bootstrap](https://hub.docker.com/layers/martizih/kaniko/v1.25.5-bootstrap/images/sha256-577a3e096547c4bea3dc95dec3116082336a866bf922894891f182bf0d1327e3) + +* v1.25.4 ([02.09.2025](https://github.com/mzihlmann/kaniko/releases/tag/v1.25.4)): +[v1.25.4](https://hub.docker.com/layers/martizih/kaniko/v1.25.4/images/sha256-bd9d3dbc86380e7e23cb8d6e07ab255297f6af7d3a97f1b9ef4d59ebad4e8b68) , +[v1.25.4-slim](https://hub.docker.com/layers/martizih/kaniko/v1.25.4-slim/images/sha256-e91b2e9db57a627def0920bd1f28e26b271749bc53afa72e417555e9b639116f) , +[v1.25.4-debug](https://hub.docker.com/layers/martizih/kaniko/v1.25.4-debug/images/sha256-a374015759d380dd2883bac1241510d80e5c785199c9350f3344ff1c892b82f4) , +[v1.25.4-warmer](https://hub.docker.com/layers/martizih/kaniko/v1.25.4-warmer/images/sha256-a19547bc4d595993818b87e867a119804c9afafac5d547f8e40a916b8cf816c4) , +[v1.25.4-bootstrap](https://hub.docker.com/layers/martizih/kaniko/v1.25.4-bootstrap/images/sha256-1e70040e60989db4e1ce29e5082894f4cc97a6bb68211f892c026de2cf955172) + +* v1.25.3 ([20.08.2025](https://github.com/mzihlmann/kaniko/releases/tag/v1.25.3)): +[v1.25.3](https://hub.docker.com/layers/martizih/kaniko/v1.25.3/images/sha256-8db8d10a37a05fcae76070eebe8fff04b9101708daf03eb1fda12b92c363f833) , +[v1.25.3-slim](https://hub.docker.com/layers/martizih/kaniko/v1.25.3-slim/images/sha256-3155963be71addfc86703e3de7689d829fea0ab3213b5c057ab1f2e96da791a4) , +[v1.25.3-debug](https://hub.docker.com/layers/martizih/kaniko/v1.25.3-debug/images/sha256-8b83fd187e889afba27d1dc1201bb2ec377ebba470fbcb615c148a16059722ab) , +[v1.25.3-warmer](https://hub.docker.com/layers/martizih/kaniko/v1.25.3-warmer/images/sha256-44837758d3520c764fe4c7c9deedeec3896f790b96d424294836214471ace599) , +[v1.25.3-bootstrap](https://hub.docker.com/layers/martizih/kaniko/v1.25.3-bootstrap/images/sha256-4c303e5c4bac4bc8318c50c6250e343a824fd57a70c73c47d1ace7eb9090fb7a) + +* v1.25.2 ([08.08.2025](https://github.com/mzihlmann/kaniko/releases/tag/v1.25.2)): +[v1.25.2](https://hub.docker.com/layers/martizih/kaniko/v1.25.2/images/sha256-943f9872e7ae8986efe9231dd9035b53ba3fe69a237cbd6797a2ee9082758b87) , +[v1.25.2-slim](https://hub.docker.com/layers/martizih/kaniko/v1.25.2-slim/images/sha256-ff97a994dd96cb67b9ba05982cac593eba6d37d127a131cb40898a1285f94475) , +[v1.25.2-debug](https://hub.docker.com/layers/martizih/kaniko/v1.25.2-debug/images/sha256-1c6b836a48fab720ca964d8d1af283fb1d6285b33f3514e953a28d2c3eb59938) , +[v1.25.2-warmer](https://hub.docker.com/layers/martizih/kaniko/v1.25.2-warmer/images/sha256-1974c33c5a7fdf660f4003de63d08cdffc7198657d8b2eb9e8c1974a598f0026) , +[v1.25.2-bootstrap](https://hub.docker.com/layers/martizih/kaniko/v1.25.2-bootstrap/images/sha256-ab0f6096de982a491bcc1a0dcff2f41fe0c4a33dd84f2578d5f0a3093b3c5249) + +* v1.25.1 ([17.07.2025](https://github.com/mzihlmann/kaniko/releases/tag/v1.25.1)): +[v1.25.1](https://hub.docker.com/layers/martizih/kaniko/v1.25.1/images/sha256-678be3129940df1d31d39630b8dcffc1506a33ce35966cd9cca2392a8183ec4d) , +[v1.25.1-slim](https://hub.docker.com/layers/martizih/kaniko/v1.25.1-slim/images/sha256-65024bf7092d212e4395223d36371ee4348177e4db587e5372048614a380732b) , +[v1.25.1-debug](https://hub.docker.com/layers/martizih/kaniko/v1.25.1-debug/images/sha256-4ba37b393656310b5ce7589a9699b07b5a4714e8542ee1c6fbdc580734e3ea69) , +[v1.25.1-warmer](https://hub.docker.com/layers/martizih/kaniko/v1.25.1-warmer/images/sha256-c6df4eaf6f8183df98a82447c9b8c5af080db2a3c991ec938daf906ea4412299) , +[v1.25.1-bootstrap](https://hub.docker.com/layers/martizih/kaniko/v1.25.1-bootstrap/images/sha256-d76704e7e6f52cfedf5b09f0f7ac243372c55d135145c7558dbf0f406953182d) + +* v1.25.0 ([08.07.2025](https://github.com/mzihlmann/kaniko/releases/tag/v1.25.0)): +[v1.25.0](https://hub.docker.com/layers/martizih/kaniko/v1.25.0/images/sha256-11c070813f7301c826782933da328cc8a3bd68e41b15e1d69d82b5263944e9fb) , +[v1.25.0-slim](https://hub.docker.com/layers/martizih/kaniko/v1.25.0-slim/images/sha256-44ff2c622d28dc1e5af81bcf7a02fa067e1f75129757fde35d8d76253663027a) , +[v1.25.0-debug](https://hub.docker.com/layers/martizih/kaniko/v1.25.0-debug/images/sha256-6eea9a36fa44465cf9c91069c5632323a48f122a40d26880ec8a1bc8d65ab66e) , +[v1.25.0-warmer](https://hub.docker.com/layers/martizih/kaniko/v1.25.0-warmer/images/sha256-364a1a0d0508b44868c858dd04befe8bb9fab64b8354345a8fb291ec6d57af2e) , +[v1.25.0-bootstrap](https://hub.docker.com/layers/martizih/kaniko/v1.25.0-bootstrap/images/sha256-23c77f645c872cd0122bb1befa4216eb7f28229f0b96f11186f6a6da968a0c64) + +v1.24.x +--- +* v1.24.3 ([29.06.2025](https://github.com/mzihlmann/kaniko/releases/tag/v1.24.3)): +[v1.24.3](https://hub.docker.com/layers/martizih/kaniko/v1.24.3/images/sha256-774ce9c3134b6620ce5675ff034274ff92ba2549f71572e63c80803c83e303e4) , +[v1.24.3-slim](https://hub.docker.com/layers/martizih/kaniko/v1.24.3-slim/images/sha256-638fc2e1c4dc8f73358ae9e08acb22c08efe281425d3f3b15c1a8b799edebd48) , +[v1.24.3-debug](https://hub.docker.com/layers/martizih/kaniko/v1.24.3-debug/images/sha256-5b419c60fa33658c41f6bdb11c0663951edfa8804c56f15ff906479b9e8adabc) , +[v1.24.3-warmer](https://hub.docker.com/layers/martizih/kaniko/v1.24.3-warmer/images/sha256-938921850da305806bc88a3921bd5f9c4e1af5a56e64f50f67f2d365a0380975) , +[v1.24.3-bootstrap](https://hub.docker.com/layers/martizih/kaniko/v1.24.3-bootstrap/images/sha256-188618fbce8cdfe2f9aa850fb4f741ea24457751d95604bc773d4961ff8919a4) + +* v1.24.2 ([19.06.2025](https://github.com/mzihlmann/kaniko/releases/tag/v1.24.2)): +[v1.24.2](https://hub.docker.com/layers/martizih/kaniko/v1.24.2/images/sha256-3565f8eeb8023b8d6d41706653a3d6c6a997a3bc47efcfbe231a787b3c1db900) , +[v1.24.2-slim](https://hub.docker.com/layers/martizih/kaniko/v1.24.2-slim/images/sha256-78efeb3cae7fffafe815858245b0c54015460cdb99079ed1c906dc2b56a58096) , +[v1.24.2-debug](https://hub.docker.com/layers/martizih/kaniko/v1.24.2-debug/images/sha256-54c4010d4d14787bc8667d36279daf7ac9e4b0fbec7fed8c918ed3fbb5d37d49) , +[v1.24.2-warmer](https://hub.docker.com/layers/martizih/kaniko/v1.24.2-warmer/images/sha256-8bfa80eb51d769efcad334b8e5b69ab2844f885d729e695dc521f9656ff7fa61) + +* v1.24.1 ([12.06.2025](https://github.com/mzihlmann/kaniko/releases/tag/v1.24.1)): +[v1.24.1](https://hub.docker.com/layers/martizih/kaniko/v1.24.1/images/sha256-7fb3089564212727c112f76f08d19e957bbb8a90a9a719b6d5f5fb6f5d62f939) , +[v1.24.1-slim](https://hub.docker.com/layers/martizih/kaniko/v1.24.1-slim/images/sha256-62f5a7d1fc803c686623550d863e410fb46c35ffe879fb15cca19f9af648d287) , +[v1.24.1-debug](https://hub.docker.com/layers/martizih/kaniko/v1.24.1-debug/images/sha256-8d47ae1280eceb15788ba04fe3e1f71e78c6c4eb6c2c52f0c8f738ddb34beb38) , +[v1.24.1-warmer](https://hub.docker.com/layers/martizih/kaniko/v1.24.1-warmer/images/sha256-39e49c82061922acb1a42292084a02ea790918a46d1bdd1cb1b40606b9b09080) diff --git a/integration/bakefiles/test_issue_mz351/Dockerfile b/integration/bakefiles/test_issue_mz351/Dockerfile new file mode 100644 index 000000000..3fd08e4c1 --- /dev/null +++ b/integration/bakefiles/test_issue_mz351/Dockerfile @@ -0,0 +1,8 @@ +FROM alpine:3.11 AS base +RUN echo base > /base + +FROM base AS app +RUN echo app > /app + +FROM base AS tools +RUN echo tools > /tools diff --git a/integration/bakefiles/test_issue_mz351/bake.json b/integration/bakefiles/test_issue_mz351/bake.json new file mode 100644 index 000000000..9f379e973 --- /dev/null +++ b/integration/bakefiles/test_issue_mz351/bake.json @@ -0,0 +1,9 @@ +{ + "version": "1", + "targets": { + "app": { + "target": "app", + "destination": ["test_issue_mz351:latest"] + } + } +} diff --git a/integration/bakefiles/test_issue_mz351/docker-bake.hcl b/integration/bakefiles/test_issue_mz351/docker-bake.hcl new file mode 100644 index 000000000..5f6e1e82e --- /dev/null +++ b/integration/bakefiles/test_issue_mz351/docker-bake.hcl @@ -0,0 +1,5 @@ +target "app" { + context = "." + dockerfile = "Dockerfile" + target = "app" +} diff --git a/integration/dockerfiles/Dockerfile_test_preserve_context b/integration/dockerfiles/Dockerfile_test_preserve_context new file mode 100644 index 000000000..5ab50a2a5 --- /dev/null +++ b/integration/dockerfiles/Dockerfile_test_preserve_context @@ -0,0 +1,11 @@ +FROM busybox AS stage1 +RUN echo "stage1 done" > /stage1.txt + +FROM busybox AS final +COPY --from=stage1 /stage1.txt /stage1.txt +RUN if ls -l /proc/1/exe | grep -q '/kaniko/executor'; then \ + [ ! -d /kaniko ]; \ + [ -f /kaniko2/executor ]; \ + else \ + echo "PID1 is not Kaniko, skipping check"; \ + fi diff --git a/integration/integration_bake_test.go b/integration/integration_bake_test.go new file mode 100644 index 000000000..f60a4d9be --- /dev/null +++ b/integration/integration_bake_test.go @@ -0,0 +1,92 @@ +/* +Copyright 2026 OSS Container Tools + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package integration + +import ( + "os" + "os/exec" + "path/filepath" + "testing" + + "github.com/osscontainertools/kaniko/pkg/bake" +) + +// TestBake is a smoke test for the bake subcommand. For each folder under +// bakefiles/ it builds the bakefile's target with kaniko and the equivalent +// docker bake HCL with buildx, then checks the two images match. The push +// destinations are injected with --set, so the fixtures stay registry-agnostic. +func TestBake(t *testing.T) { + cwd, err := os.Getwd() + if err != nil { + t.Fatal(err) + } + dir := filepath.Join(cwd, "bakefiles") + entries, err := os.ReadDir(dir) + if err != nil { + t.Fatal(err) + } + + for _, entry := range entries { + if !entry.IsDir() { + continue + } + name := entry.Name() + ctxDir := filepath.Join(dir, name) + + t.Run(name, func(t *testing.T) { + t.Parallel() + + bakefile, err := bake.Parse(filepath.Join(ctxDir, "bake.json")) + if err != nil { + t.Fatal(err) + } + targets, err := bakefile.Resolve(nil) + if err != nil { + t.Fatal(err) + } + if len(targets) != 1 { + t.Fatalf("want a single target, got %d", len(targets)) + } + target := targets[0] + + kanikoImage := GetKanikoImage(config.imageRepo, name) + dockerImage := GetDockerImage(config.imageRepo, name) + + kanikoFlags := []string{"run", "--rm", "--net=host", "-v", ctxDir + ":/ctx"} + kanikoFlags = addServiceAccountFlags(kanikoFlags, config.serviceAccount) + kanikoFlags = addCoverageFlags(kanikoFlags) + kanikoFlags = append(kanikoFlags, ExecutorImage, + "bake", "/ctx/bake.json", "-c", "/ctx", + "--set", target.ID+".destination="+kanikoImage) + kanikoCmd := exec.Command("docker", kanikoFlags...) + if out, err := RunCommandWithoutTest(kanikoCmd); err != nil { + t.Fatalf("%v: %v\n%s", kanikoCmd.Args, err, string(out)) + } + + dockerCmd := exec.Command("docker", "buildx", "bake", + "-f", "docker-bake.hcl", + "--set", target.ID+".tags="+dockerImage, + "--push") + dockerCmd.Dir = ctxDir + if out, err := RunCommandWithoutTest(dockerCmd); err != nil { + t.Fatalf("%v: %v\n%s", dockerCmd.Args, err, string(out)) + } + + containerDiff(t, dockerImage, kanikoImage, "--ignore-history") + }) + } +} diff --git a/memory/MEMORY.md b/memory/MEMORY.md new file mode 100644 index 000000000..f704356da --- /dev/null +++ b/memory/MEMORY.md @@ -0,0 +1,4 @@ +# Memory Index + +- [Assertion style preference](feedback_assertions.md) β€” only assert properties that encode real logic, not trivial glue-code consequences +- [RUN command formatting in Dockerfiles](feedback_run_format.md) β€” backslash-continuation, one command per line, 4-space indent diff --git a/memory/feedback_assertions.md b/memory/feedback_assertions.md new file mode 100644 index 000000000..0d59ade7d --- /dev/null +++ b/memory/feedback_assertions.md @@ -0,0 +1,11 @@ +--- +name: assertion style preference +description: What makes a good vs bad assertion candidate +type: feedback +--- + +Only add assertions that encode real logic β€” properties that emerge non-trivially from the function's logic across multiple branches or operations. + +**Why:** Asserting "I just set this to non-nil, therefore it is non-nil" is weak glue-code assertion with no value. + +**How to apply:** A good assertion checks a property whose truth requires understanding how the branches/operations combine β€” e.g. a length invariant, a structural relationship between two values, a state-machine postcondition, or a semantic contract at a function boundary (like "result is always absolute" in ToAbsPath, which requires reasoning about all 3 branches). Bad assertions just re-state what the immediately preceding line already made obvious. diff --git a/memory/feedback_run_format.md b/memory/feedback_run_format.md new file mode 100644 index 000000000..3df26a3a4 --- /dev/null +++ b/memory/feedback_run_format.md @@ -0,0 +1,17 @@ +--- +name: RUN command formatting in Dockerfiles +description: Preferred multi-command RUN format in Dockerfiles +type: feedback +--- + +Use backslash-continuation with one command per line, indented 4 spaces: + +```dockerfile +RUN apt-get update \ + && apt-get install -y \ + libcap2-bin \ + && rm -rf /var/lib/apt/lists/* +``` + +**Why:** Consistent with the style used across the project's integration test Dockerfiles. +**How to apply:** Any time writing RUN commands with multiple chained operations in a Dockerfile. diff --git a/pkg/bake/bake.go b/pkg/bake/bake.go new file mode 100644 index 000000000..d2949f2d7 --- /dev/null +++ b/pkg/bake/bake.go @@ -0,0 +1,131 @@ +/* +Copyright 2026 OSS Container Tools + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package bake + +import ( + "encoding/json" + "errors" + "fmt" + "os" + "sort" + "strings" +) + +type Target struct { + Target string `json:"target"` + Destination []string `json:"destination"` +} + +type Bakefile struct { + Version string `json:"version"` + Targets map[string]Target `json:"targets"` +} + +type ResolvedTarget struct { + ID string + Stage string + Destination []string +} + +func Parse(path string) (*Bakefile, error) { + data, err := os.ReadFile(path) + if err != nil { + return nil, fmt.Errorf("reading bakefile: %w", err) + } + return parse(data) +} + +func parse(data []byte) (*Bakefile, error) { + b := &Bakefile{} + if err := json.Unmarshal(data, b); err != nil { + return nil, fmt.Errorf("parsing bakefile: %w", err) + } + if b.Version != "1" { + return nil, fmt.Errorf("unsupported bakefile version %q, expected %q", b.Version, "1") + } + if len(b.Targets) == 0 { + return nil, errors.New("bakefile defines no targets") + } + return b, nil +} + +func (b *Bakefile) Resolve(selected []string) ([]ResolvedTarget, error) { + ids := selected + if len(ids) == 0 { + ids = make([]string, 0, len(b.Targets)) + for id := range b.Targets { + ids = append(ids, id) + } + sort.Strings(ids) + } + + resolved := make([]ResolvedTarget, 0, len(ids)) + for _, id := range ids { + t, ok := b.Targets[id] + if !ok { + return nil, fmt.Errorf("unknown target %q", id) + } + stage := t.Target + if stage == "" { + stage = id + } + resolved = append(resolved, ResolvedTarget{ID: id, Stage: stage, Destination: t.Destination}) + } + return resolved, nil +} + +type Override struct { + Target string + Field string + Value string +} + +func ParseOverride(s string) (Override, error) { + key, value, ok := strings.Cut(s, "=") + if !ok { + return Override{}, fmt.Errorf("invalid --set %q, want .=", s) + } + target, field, ok := strings.Cut(key, ".") + if !ok || target == "" || field == "" { + return Override{}, fmt.Errorf("invalid --set %q, want .=", s) + } + return Override{Target: target, Field: field, Value: value}, nil +} + +func ApplyOverrides(targets []ResolvedTarget, overrides []Override) error { + idx := make(map[string]int, len(targets)) + for i, t := range targets { + idx[t.ID] = i + } + dests := map[int][]string{} + for _, o := range overrides { + i, ok := idx[o.Target] + if !ok { + return fmt.Errorf("--set target %q is not built", o.Target) + } + switch o.Field { + case "destination": + dests[i] = append(dests[i], o.Value) + default: + return fmt.Errorf("--set field %q is not supported", o.Field) + } + } + for i, d := range dests { + targets[i].Destination = d + } + return nil +}