mz822: fold resolved heredoc contents into COPY/ADD cache key

README.md

@@ -149,6 +149,7 @@ expect - see [Known Issues](#known-issues).
       - [Flag `FF_KANIKO_SECUREJOIN_EXTRACTION`](#flag-ff_kaniko_securejoin_extraction)
       - [Flag `FF_KANIKO_RESOLVE_CACHE_KEY`](#flag-ff_kaniko_resolve_cache_key)
       - [Flag `FF_KANIKO_UNTAR_SKIP_ROOT`](#flag-ff_kaniko_untar_skip_root)
+      - [Flag `FF_KANIKO_EXPAND_HEREDOC`](#flag-ff_kaniko_expand_heredoc)
     - [Assertion Overrides](#assertion-overrides)
     - [Debug Image](#debug-image)
   - [Security](#security)
@@ -1305,6 +1306,13 @@ Set this flag to `true` to skip the root `.` entry when untarring.
 Defaults to `false`.
 Becomes default in `v1.29.0`.
 
+#### Flag `FF_KANIKO_EXPAND_HEREDOC`
+
+Docker applies Dockerfile word-expansion to a `COPY` or `ADD` heredoc body when the delimiter is unquoted, so `${VAR}` expands and `\${VAR}` keeps the literal text. A quoted delimiter (`<<'EOF'`) leaves the body verbatim. kaniko writes the body verbatim in every case, so the expanded files diverge from Docker.
+Set this flag to `true` to expand build args and env in unquoted `COPY` and `ADD` heredoc bodies.
+Defaults to `false`.
+Becomes default in `v1.29.0`.
+
 ### Assertion Overrides
 
 Kaniko checks internal invariants at runtime. If one is violated the build stops with a message like:

golden/golden_test.go

@@ -41,6 +41,7 @@ import (
 	testissuemz703 "github.com/osscontainertools/kaniko/golden/testdata/test_issue_mz703"
 	testissuemz791 "github.com/osscontainertools/kaniko/golden/testdata/test_issue_mz791"
 	testissuemz813 "github.com/osscontainertools/kaniko/golden/testdata/test_issue_mz813"
+	testissuemz822 "github.com/osscontainertools/kaniko/golden/testdata/test_issue_mz822"
 	testunittests "github.com/osscontainertools/kaniko/golden/testdata/test_unittests"
 	"github.com/osscontainertools/kaniko/golden/types"
 	"github.com/osscontainertools/kaniko/pkg/cache"
@@ -96,6 +97,7 @@ var allTests = map[string][]types.GoldenTests{
 	"test_issue_mz703": {testissuemz703.Tests},
 	"test_issue_mz791": {testissuemz791.Tests},
 	"test_issue_mz813": {testissuemz813.Tests},
+	"test_issue_mz822": {testissuemz822.Tests},
 	"test_unittests":   testunittests.Tests,
 }
 var update bool

golden/testdata/test_issue_mz822/Dockerfile

@@ -0,0 +1,10 @@
+# mz822: a COPY heredoc whose body expands a build arg is not reflected in the
+# layer cache key. The body is omitted from the instruction text the key is built
+# from, so a build that only changes the arg reuses the layer cached for a
+# different resolved content and serves a stale file. Same defect as mz791 for
+# COPY/ADD paths and mz813 for WORKDIR.
+FROM busybox@sha256:fd8d9aa63ba2f0982b5304e1ee8d3b90a210bc1ffb5314d980eb6962f1a9715d
+ARG A
+COPY <<EOF /out.txt
+value=${A}
+EOF

golden/testdata/test_issue_mz822/plans/resolved_one

@@ -0,0 +1,6 @@
+FROM busybox@sha256:fd8d9aa63ba2f0982b5304e1ee8d3b90a210bc1ffb5314d980eb6962f1a9715d
+UNPACK busybox@sha256:fd8d9aa63ba2f0982b5304e1ee8d3b90a210bc1ffb5314d980eb6962f1a9715d
+CACHE MISS: c45e020b8d05e23cd5c90f472a31d3ec323c23b7119a455e05a91d9c2403205b
+ARG A
+CACHE MISS: 4c4aa816a4721aedf65fe06c91ec06af949a218aecf79402ccf9dd9547aec538
+COPY <<EOF /out.txt

golden/testdata/test_issue_mz822/plans/resolved_two

@@ -0,0 +1,6 @@
+FROM busybox@sha256:fd8d9aa63ba2f0982b5304e1ee8d3b90a210bc1ffb5314d980eb6962f1a9715d
+UNPACK busybox@sha256:fd8d9aa63ba2f0982b5304e1ee8d3b90a210bc1ffb5314d980eb6962f1a9715d
+CACHE MISS: c45e020b8d05e23cd5c90f472a31d3ec323c23b7119a455e05a91d9c2403205b
+ARG A
+CACHE MISS: 60f8b56b477272a9ca6ded042fcca6b3da3c7e1942d05419961310100b64f59f
+COPY <<EOF /out.txt

golden/testdata/test_issue_mz822/plans/unresolved_one

@@ -0,0 +1,6 @@
+FROM busybox@sha256:fd8d9aa63ba2f0982b5304e1ee8d3b90a210bc1ffb5314d980eb6962f1a9715d
+UNPACK busybox@sha256:fd8d9aa63ba2f0982b5304e1ee8d3b90a210bc1ffb5314d980eb6962f1a9715d
+CACHE MISS: c45e020b8d05e23cd5c90f472a31d3ec323c23b7119a455e05a91d9c2403205b
+ARG A
+CACHE MISS: 43fe9047775871de424a17ba5c97bcccbd99e1f16ea798d62db0a931e44f03a0
+COPY <<EOF /out.txt

golden/testdata/test_issue_mz822/plans/unresolved_two

@@ -0,0 +1,6 @@
+FROM busybox@sha256:fd8d9aa63ba2f0982b5304e1ee8d3b90a210bc1ffb5314d980eb6962f1a9715d
+UNPACK busybox@sha256:fd8d9aa63ba2f0982b5304e1ee8d3b90a210bc1ffb5314d980eb6962f1a9715d
+CACHE MISS: c45e020b8d05e23cd5c90f472a31d3ec323c23b7119a455e05a91d9c2403205b
+ARG A
+CACHE MISS: 43fe9047775871de424a17ba5c97bcccbd99e1f16ea798d62db0a931e44f03a0
+COPY <<EOF /out.txt

golden/testdata/test_issue_mz822/test.go

@@ -0,0 +1,49 @@
+package testissuemz822
+
+import "github.com/osscontainertools/kaniko/golden/types"
+
+var Tests = types.GoldenTests{
+	Name:       "test_issue_mz822",
+	Dockerfile: "Dockerfile",
+	Tests: []types.GoldenTest{
+		// Without FF_KANIKO_RESOLVE_CACHE_KEY the heredoc body is omitted from the
+		// key, so A=one and A=two share one COPY cache key and a build that only
+		// changes A serves a stale file. The two plans are identical.
+		{
+			Args: []string{"--no-push", "--cache", "--build-arg", "A=one"},
+			Env: map[string]string{
+				"FF_KANIKO_CACHE_LOOKAHEAD": "1",
+			},
+			Plan: "unresolved_one",
+		},
+		{
+			Args: []string{"--no-push", "--cache", "--build-arg", "A=two"},
+			Env: map[string]string{
+				"FF_KANIKO_CACHE_LOOKAHEAD": "1",
+			},
+			Plan: "unresolved_two",
+		},
+		// With the flag the resolved heredoc body is folded into the key, so the
+		// COPY cache key tracks A. The unquoted delimiter expands the body, so
+		// FF_KANIKO_EXPAND_HEREDOC must be on for the key to match what the
+		// executor writes. The two plans differ in the COPY key.
+		{
+			Args: []string{"--no-push", "--cache", "--build-arg", "A=one"},
+			Env: map[string]string{
+				"FF_KANIKO_CACHE_LOOKAHEAD":   "1",
+				"FF_KANIKO_RESOLVE_CACHE_KEY": "1",
+				"FF_KANIKO_EXPAND_HEREDOC":    "1",
+			},
+			Plan: "resolved_one",
+		},
+		{
+			Args: []string{"--no-push", "--cache", "--build-arg", "A=two"},
+			Env: map[string]string{
+				"FF_KANIKO_CACHE_LOOKAHEAD":   "1",
+				"FF_KANIKO_RESOLVE_CACHE_KEY": "1",
+				"FF_KANIKO_EXPAND_HEREDOC":    "1",
+			},
+			Plan: "resolved_two",
+		},
+	},
+}

integration/dockerfiles/Dockerfile_test_issue_mz817

@@ -0,0 +1,19 @@
+# mz817: on v1.27.6 COPY heredoc does not expand build variables.
+# Docker applies Dockerfile word-expansion to a COPY heredoc body when the
+# delimiter is unquoted: ${VAR} expands and \${VAR} keeps the literal text with
+# the backslash stripped. A quoted delimiter (<<'EOF') leaves the body verbatim.
+# kaniko writes the body verbatim in every case, so the expanded files diverge.
+FROM scratch
+
+ENV DO_EXPAND=expanded
+
+# unquoted delimiter expands ${DO_EXPAND}, keeps \${DO_EXPAND} literal
+COPY <<EOF /tmp/expand.txt
+value=${DO_EXPAND}
+escaped=\${DO_EXPAND}
+EOF
+
+# quoted delimiter leaves the body untouched
+COPY <<'EOF' /tmp/literal.txt
+value=${DO_EXPAND}
+EOF

integration/images.go

@@ -112,6 +112,7 @@ var KanikoEnv = []string{
 	"FF_KANIKO_RESOLVE_CACHE_KEY=1",
 	"FF_KANIKO_UNTAR_SKIP_ROOT=1",
 	"FF_KANIKO_REPRODUCIBLE_PRESERVE_BASE_LAYERS=1",
+	"FF_KANIKO_EXPAND_HEREDOC=1",
 	"KANIKO_PRINT_PLAN=1",
 }
 

pkg/commands/add.go

@@ -129,7 +129,7 @@ func (a *AddCommand) String() string {
 }
 
 func (a *AddCommand) CacheKey(replacementEnvs []string) (string, error) {
-	return util.ResolveVariables(a.cmd.String(), replacementEnvs)
+	return resolvedCacheKey(a.cmd.String(), a.cmd.SourceContents, replacementEnvs)
 }
 
 func (a *AddCommand) FilesUsedFromContext(config *v1.Config, buildArgs *dockerfile.BuildArgs) ([]string, error) {
@@ -220,7 +220,7 @@ func (ca *CachingAddCommand) String() string {
 }
 
 func (ca *CachingAddCommand) CacheKey(replacementEnvs []string) (string, error) {
-	return util.ResolveVariables(ca.String(), replacementEnvs)
+	return resolvedCacheKey(ca.cmd.String(), ca.cmd.SourceContents, replacementEnvs)
 }
 
 func addCmdFilesUsedFromContext(config *v1.Config, buildArgs *dockerfile.BuildArgs, cmd *instructions.AddCommand,

pkg/commands/copy.go

@@ -157,7 +157,16 @@ func (c *CopyCommand) ExecuteCommand(config *v1.Config, buildArgs *dockerfile.Bu
 			return nil
 		}
 
-		srcFile := strings.NewReader(src.Data)
+		data := src.Data
+		if src.Expand && kConfig.EnvBool("FF_KANIKO_EXPAND_HEREDOC") {
+			expanded, err := util.ResolveEnvironmentReplacementRaw(src.Data, replacementEnvs)
+			if err != nil {
+				return fmt.Errorf("expanding heredoc content: %w", err)
+			}
+			data = expanded
+		}
+
+		srcFile := strings.NewReader(data)
 		err = util.CreateFile(destPath, srcFile, chmod.Apply(0o644), uint32(uid), uint32(gid))
 		if err != nil {
 			return fmt.Errorf("creating file: %w", err)
@@ -179,7 +188,32 @@ func (c *CopyCommand) String() string {
 }
 
 func (c *CopyCommand) CacheKey(replacementEnvs []string) (string, error) {
-	return util.ResolveVariables(c.cmd.String(), replacementEnvs)
+	return resolvedCacheKey(c.cmd.String(), c.cmd.SourceContents, replacementEnvs)
+}
+
+// resolvedCacheKey resolves the instruction variables and folds in the resolved
+// heredoc contents, expanded the same way the executor materializes them.
+// cmd.String() omits the heredoc body, so without this a build arg referenced
+// only inside a body would not invalidate the layer.
+func resolvedCacheKey(instruction string, contents []instructions.SourceContent, replacementEnvs []string) (string, error) {
+	resolved, err := util.ResolveVariables(instruction, replacementEnvs)
+	if err != nil {
+		return "", err
+	}
+	var key strings.Builder
+	key.WriteString(resolved)
+	for _, src := range contents {
+		content := src.Data
+		if src.Expand && kConfig.EnvBool("FF_KANIKO_EXPAND_HEREDOC") {
+			content, err = util.ResolveEnvironmentReplacementRaw(src.Data, replacementEnvs)
+			if err != nil {
+				return "", err
+			}
+		}
+		key.WriteString("\n")
+		key.WriteString(content)
+	}
+	return key.String(), nil
 }
 
 func (c *CopyCommand) FilesUsedFromContext(config *v1.Config, buildArgs *dockerfile.BuildArgs) ([]string, error) {
@@ -274,7 +308,7 @@ func (cr *CachingCopyCommand) String() string {
 }
 
 func (cr *CachingCopyCommand) CacheKey(replacementEnvs []string) (string, error) {
-	return util.ResolveVariables(cr.String(), replacementEnvs)
+	return resolvedCacheKey(cr.cmd.String(), cr.cmd.SourceContents, replacementEnvs)
 }
 
 func (cr *CachingCopyCommand) From() string {

pkg/util/command_util.go

@@ -97,6 +97,16 @@ func ResolveEnvironmentReplacement(value string, envs []string, isFilepath bool)
 	return fp, nil
 }
 
+// ResolveEnvironmentReplacementRaw resolves env variables in raw content such as a
+// heredoc body. Unlike ResolveEnvironmentReplacement it leaves quotes intact
+// (SkipProcessQuotes), matching how docker expands COPY/ADD heredoc contents.
+func ResolveEnvironmentReplacementRaw(value string, envs []string) (string, error) {
+	shlex := shell.NewLex(parser.DefaultEscapeToken)
+	shlex.SkipProcessQuotes = true
+	fp, _, err := shlex.ProcessWord(value, shell.EnvsFromSlice(envs))
+	return fp, err
+}
+
 func ResolveEnvAndWildcards(sd instructions.SourcesAndDest, fileContext FileContext, envs []string) ([]string, string, error) {
 	// First, resolve any environment replacement
 	resolvedEnvs, err := ResolveEnvironmentReplacementList(sd.SourcePaths, envs, true)