diff --git a/e2e-tests/git-archive-build.yaml b/e2e-tests/git-archive-build.yaml new file mode 100644 index 000000000..0d85e61c1 --- /dev/null +++ b/e2e-tests/git-archive-build.yaml @@ -0,0 +1,52 @@ +package: + name: test-git-archive + version: 1.0.0 + epoch: 0 + description: This package mainly just tests the git-archive pipeline + copyright: + - license: Apache-2.0 + +environment: + contents: + packages: + - busybox + +# git-archive sources a subtree of the local git repository (the one containing +# this config) host-side, without cloning and without mounting .git into the +# build. With no tag/branch it archives the commit being built, so it packages +# the fixture exactly as committed in this repository. Tag/branch/expected-commit +# variants are covered by the unit tests in pkg/build/gitarchive_test.go; this +# e2e test exercises the real build integration. +pipeline: + - name: "Create the bogus package content" + runs: | + echo "package does not do anything" > "${{targets.contextdir}}/README" + + - name: "Archive a subtree of the local repository at the build commit" + uses: git-archive + with: + path: e2e-tests/git-archive-fixture + + - name: "Verify the subtree was archived into the workspace" + runs: | + set -e + dir=e2e-tests/git-archive-fixture + + # The archived files retain their repository-root-relative path prefix. + [ -f "$dir/marker.txt" ] || { echo "FAIL: $dir/marker.txt missing"; exit 1; } + [ -f "$dir/nested/deep.txt" ] || { echo "FAIL: $dir/nested/deep.txt missing"; exit 1; } + + # Content matches the committed fixture. + grep -q "git-archive e2e fixture" "$dir/marker.txt" || { echo "FAIL: marker.txt content"; exit 1; } + grep -q "nested content" "$dir/nested/deep.txt" || { echo "FAIL: nested/deep.txt content"; exit 1; } + + # Only the requested subtree is archived: repository files outside the + # archived path must NOT be present in the workspace. + for f in go.mod e2e-tests/git-archive-build.yaml pkg/build/build.go; do + if [ -e "$f" ]; then + echo "FAIL: '$f' should not be present; only the requested subtree should be archived" + exit 1 + fi + done + + echo "PASS: git-archive populated the workspace with only the requested subtree" diff --git a/e2e-tests/git-archive-fixture/marker.txt b/e2e-tests/git-archive-fixture/marker.txt new file mode 100644 index 000000000..f50b718a8 --- /dev/null +++ b/e2e-tests/git-archive-fixture/marker.txt @@ -0,0 +1 @@ +git-archive e2e fixture diff --git a/e2e-tests/git-archive-fixture/nested/deep.txt b/e2e-tests/git-archive-fixture/nested/deep.txt new file mode 100644 index 000000000..ca281f5b2 --- /dev/null +++ b/e2e-tests/git-archive-fixture/nested/deep.txt @@ -0,0 +1 @@ +nested content diff --git a/pkg/build/build.go b/pkg/build/build.go index e14f56703..c6a1d2033 100644 --- a/pkg/build/build.go +++ b/pkg/build/build.go @@ -25,6 +25,7 @@ import ( "maps" "math" "os" + "os/exec" "path/filepath" "runtime" "slices" @@ -554,6 +555,187 @@ func (b *Build) populateWorkspace(ctx context.Context, src fs.FS) error { }) } +// gitArchiveStepName is the pipeline `uses:` identifier handled host-side. +const gitArchiveStepName = "git-archive" + +// countGitArchiveSteps returns the number of git-archive steps in the given +// pipeline tree, descending into nested sub-pipelines. +func countGitArchiveSteps(steps []config.Pipeline) int { + n := 0 + for i := range steps { + if steps[i].Uses == gitArchiveStepName { + n++ + } + n += countGitArchiveSteps(steps[i].Pipeline) + } + return n +} + +// maybeGitArchiveSource looks for a `git-archive` step in the main pipeline and, +// if present, performs the archive host-side: it extracts the requested subtree +// of the local git repository into a temporary directory and repoints +// b.SourceDir at it, so the normal workspace population copies only the archived +// subtree into the build. This avoids cloning over the network and avoids +// mounting the repository's .git history into the sandbox. git-archive supersedes +// any --source-dir, as it is itself the source-acquisition mechanism. +// +// By default it archives at the commit melange is building (the config file's +// repository commit), so the packaged source matches the commit under build +// (subject to .gitattributes export rules; see gitArchive). +// +// It always returns a non-nil cleanup function (a no-op when no archive was +// performed), so callers can unconditionally defer it. +func (b *Build) maybeGitArchiveSource(ctx context.Context) (func(), error) { + log := clog.FromContext(ctx) + noop := func() {} + + // git-archive runs host-side before the sandbox, so it is only meaningful as + // a single top-level step in the main pipeline. Reject duplicates or + // placement in nested/subpackage/test pipelines rather than silently + // treating the extra steps as in-sandbox no-op markers (`melange test` + // never performs the host-side archive). + total := countGitArchiveSteps(b.Configuration.Pipeline) + if b.Configuration.Test != nil { + total += countGitArchiveSteps(b.Configuration.Test.Pipeline) + } + for i := range b.Configuration.Subpackages { + total += countGitArchiveSteps(b.Configuration.Subpackages[i].Pipeline) + if b.Configuration.Subpackages[i].Test != nil { + total += countGitArchiveSteps(b.Configuration.Subpackages[i].Test.Pipeline) + } + } + if total == 0 { + return noop, nil + } + if total > 1 { + return noop, fmt.Errorf("git-archive: may be used at most once per build, found %d", total) + } + + var step *config.Pipeline + for i := range b.Configuration.Pipeline { + if b.Configuration.Pipeline[i].Uses == gitArchiveStepName { + step = &b.Configuration.Pipeline[i] + break + } + } + if step == nil { + return noop, fmt.Errorf("git-archive: must be a top-level step in the main pipeline (not nested, in a subpackage, or in a test pipeline)") + } + + // git-archive provides the workspace source, which is incompatible with an + // explicitly empty workspace. + if b.EmptyWorkspace { + return noop, fmt.Errorf("git-archive: cannot be used with an empty workspace") + } + + // The archive shells out to host git and tar (the pipeline's `needs` only + // provisions the sandbox). Fail early with a clear message if either is + // missing. + for _, tool := range []string{"git", "tar"} { + if _, err := exec.LookPath(tool); err != nil { + return noop, fmt.Errorf("git-archive: requires %q on the host: %w", tool, err) + } + } + + // Compile has already validated the step's inputs and substituted ${{...}} + // templates into step.With (compilePipeline rewrites With with the mutated, + // non-default values), so the values can be read directly. + get := func(name string) string { return step.With[name] } + + archivePath := get("path") + if archivePath == "" { + return noop, fmt.Errorf("git-archive: 'path' is required") + } + + tag := get("tag") + branch := get("branch") + if tag != "" && branch != "" { + return noop, fmt.Errorf("git-archive: 'tag' and 'branch' are mutually exclusive") + } + + // Anchor git at the directory containing the config file; git discovers the + // enclosing repository root from there. Make the path absolute first so this + // does not depend on the current working directory. + configPath, err := filepath.Abs(b.ConfigFile) + if err != nil { + return noop, fmt.Errorf("resolving config file path: %w", err) + } + repoDir := filepath.Dir(configPath) + + expectedCommit := get("expected-commit") + + // Determine the ref to archive. A tag or branch is used as given; if neither + // is specified, default to the commit melange is building and pin the + // assurance to it. The chart source and the config live in the same + // repository, so the default packages 'path' as committed at the build + // commit (subject to .gitattributes export rules). + var ref string + switch { + case tag != "": + ref = tag + case branch != "": + ref = branch + default: + buildCommit := b.ConfigFileRepositoryCommit + if buildCommit == "" || buildCommit == config.UnknownCommit { + // Melange could not determine the build commit (for example, go-git + // cannot read HEAD in a linked worktree). Fall back to resolving HEAD + // with the git CLI, which is the commit being built. + head, err := gitRevParse(ctx, repoDir, "HEAD") + if err != nil { + return noop, fmt.Errorf("git-archive: no 'tag' or 'branch' given and could not determine the build commit (the config must live in a git repository, or set 'tag'/'branch' explicitly): %w", err) + } + buildCommit = head + } + ref = buildCommit + if expectedCommit == "" { + expectedCommit = buildCommit + } + } + + dest, err := os.MkdirTemp("", "melange-git-archive-") + if err != nil { + return noop, fmt.Errorf("creating temp dir: %w", err) + } + cleanup := func() { + if err := os.RemoveAll(dest); err != nil { + log.Warnf("failed to remove git-archive temp dir %s: %v", dest, err) + } + } + + resolved, err := gitArchive(ctx, &gitArchiveOptions{ + RepositoryDir: repoDir, + Ref: ref, + RefIsBranch: branch != "", + Path: archivePath, + ExpectedCommit: expectedCommit, + Destination: dest, + }) + if err != nil { + cleanup() + return noop, err + } + + // Record the resolved commit back into the step so the SBOM provenance + // entry (which reads this `with` map) reflects the exact source even when + // the manifest left ref/expected-commit to default. It is safe because each + // architecture build parses and owns its own Configuration, so this is not + // shared across goroutines. + if step.With == nil { + step.With = map[string]string{} + } + step.With["expected-commit"] = resolved + + // The extracted subtree is copied again by populateWorkspace into the build + // workspace, so the data is passed over twice. This deliberately reuses + // melange's standard workspace population (ignore rules, compiler config + // files) instead of extracting in place; for chart-sized subtrees the extra + // copy is negligible. Streaming directly into the workspace is a possible + // future optimization. + b.SourceDir = dest + return cleanup, nil +} + type linterTarget struct { pkgName string disabled []string // checks that are downgraded from required -> warn @@ -622,6 +804,16 @@ func (b *Build) BuildPackage(ctx context.Context) error { runner: b.Runner, } + // If the pipeline declares a git-archive source step, perform the archive + // host-side and repoint SourceDir at the extracted subtree before the + // workspace is populated. This sources from a local git repository at a + // pinned ref without mounting the repository's .git history into the build. + archiveCleanup, err := b.maybeGitArchiveSource(ctx) + if err != nil { + return fmt.Errorf("git-archive source: %w", err) + } + defer archiveCleanup() + if b.EmptyWorkspace { log.Debugf("empty workspace requested") } else { diff --git a/pkg/build/gitarchive.go b/pkg/build/gitarchive.go new file mode 100644 index 000000000..7c93da1f3 --- /dev/null +++ b/pkg/build/gitarchive.go @@ -0,0 +1,183 @@ +// Copyright 2026 Chainguard, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package build + +import ( + "bytes" + "context" + "errors" + "fmt" + "os" + "os/exec" + "strings" + + "github.com/chainguard-dev/clog" +) + +// gitArchiveOptions configures gitArchive. +type gitArchiveOptions struct { + // RepositoryDir is any path inside the local git repository to archive + // from; git discovers the enclosing repository root from it. + RepositoryDir string + // Ref is the git ref (tag, branch, or commit) to archive. + Ref string + // RefIsBranch marks Ref as a branch, enabling git-checkout's branch + // semantics for ExpectedCommit: it may be an older commit on the branch + // (an ancestor of the tip), in which case that commit is archived. When + // false, Ref must resolve to ExpectedCommit exactly. + RefIsBranch bool + // Path is the path within the repository (relative to the repository root) + // to extract. + Path string + // ExpectedCommit, if set, is the commit Ref must resolve to. A mismatch is + // a fatal error. + ExpectedCommit string + // Destination is the directory into which the archived subtree is + // extracted. Extracted files retain their Path prefix beneath it. + Destination string +} + +// gitArchive populates Destination from a subtree of a local git repository at +// a specific ref, host-side, without cloning over the network and without +// touching the repository's working tree. It resolves the ref to a commit, +// optionally verifies it against ExpectedCommit, then streams `git archive` of +// Path into Destination. It returns the resolved commit. +// +// Path prefixes are preserved: archiving "charts/foo" extracts to +// Destination/charts/foo/..., matching the layout of the source repository. +// +// Because this uses `git archive`, the repository's .gitattributes are honored: +// paths marked `export-ignore` are omitted and `export-subst` placeholders are +// expanded. The output is therefore the committed tree as filtered by those +// export rules, not necessarily a byte-for-byte copy. Requires `git` and `tar` +// on the host. +func gitArchive(ctx context.Context, opts *gitArchiveOptions) (resolvedCommit string, err error) { + log := clog.FromContext(ctx) + + if opts.Ref == "" { + return "", fmt.Errorf("ref is required") + } + if opts.Path == "" { + return "", fmt.Errorf("path is required") + } + if opts.Destination == "" { + return "", fmt.Errorf("destination is required") + } + + repoDir := opts.RepositoryDir + if repoDir == "" { + repoDir = "." + } + + // Anchor at the repository toplevel. git archive interprets its pathspec + // relative to the current directory prefix, so running from a subdirectory + // would mis-resolve Path; from the toplevel, Path is repository-root + // relative and unambiguous. + topOut, err := exec.CommandContext(ctx, "git", "-C", repoDir, "rev-parse", "--show-toplevel").Output() // #nosec G204 - git arguments come from trusted melange build configuration + if err != nil { + return "", fmt.Errorf("locating git repository from %s: %w", repoDir, err) + } + topLevel := strings.TrimSpace(string(topOut)) + + // Resolve the ref to a concrete commit so the archive (and the + // expected-commit check) operate on an immutable target. + resolved, err := gitRevParse(ctx, topLevel, opts.Ref+"^{commit}") + if err != nil { + return "", fmt.Errorf("resolving ref %q in %s: %w", opts.Ref, topLevel, err) + } + + if opts.ExpectedCommit != "" && resolved != opts.ExpectedCommit { + if !opts.RefIsBranch { + return "", fmt.Errorf("ref %q resolved to commit %s, expected %s", opts.Ref, resolved, opts.ExpectedCommit) + } + // Branch semantics match git-checkout: expected-commit pins what is + // archived and may be an older commit on the branch, so a moving tip + // does not break the build. Anything not on the branch is an error. + if err := exec.CommandContext(ctx, "git", "-C", topLevel, "merge-base", "--is-ancestor", opts.ExpectedCommit, resolved).Run(); err != nil { // #nosec G204 - git arguments come from trusted melange build configuration + return "", fmt.Errorf("branch %q is at %s and expected-commit %s is not an ancestor of it", opts.Ref, resolved, opts.ExpectedCommit) + } + log.Infof("expected-commit %s is on branch %q (tip %s); archiving it", opts.ExpectedCommit, opts.Ref, resolved) + resolved, err = gitRevParse(ctx, topLevel, opts.ExpectedCommit+"^{commit}") + if err != nil { + return "", fmt.Errorf("resolving expected-commit %q in %s: %w", opts.ExpectedCommit, topLevel, err) + } + } + if opts.ExpectedCommit == "" { + // Only reached when the caller passed a tag/branch with no + // expected-commit (genuinely unpinned). The default-ref path in + // maybeGitArchiveSource backfills ExpectedCommit with the build commit, + // so it does not trigger this warning. + log.Warnf("git archive: no expected-commit; ref %q resolved to %s", opts.Ref, resolved) + } + + // Archiving HEAD with uncommitted changes under Path is a common local + // iteration trap: the archive contains the committed tree, not the edits. + if head, headErr := gitRevParse(ctx, topLevel, "HEAD^{commit}"); headErr == nil && head == resolved { + if out, statusErr := exec.CommandContext(ctx, "git", "-C", topLevel, "status", "--porcelain", "--", opts.Path).Output(); statusErr == nil && len(bytes.TrimSpace(out)) > 0 { // #nosec G204 - git arguments come from trusted melange build configuration + log.Warnf("git archive: uncommitted changes under %s are NOT included; archiving %s as committed", opts.Path, resolved) + } + } + + if err := os.MkdirAll(opts.Destination, 0o755); err != nil { + return "", fmt.Errorf("creating destination %s: %w", opts.Destination, err) + } + + log.Infof("archiving %s at %s from %s into %s", opts.Path, resolved, topLevel, opts.Destination) + + // `git archive -- | tar -x -C dest`. We pipe a tar stream + // so extraction is direct and independent of tar's format autodetection. + // The `--` forces Path to be a pathspec: git otherwise parses options even + // after the tree-ish, so a Path like `--output=/host/file` would write the + // archive to an arbitrary host path. + archive := exec.CommandContext(ctx, "git", "-C", topLevel, "archive", "--format=tar", resolved, "--", opts.Path) // #nosec G204 - option injection blocked by --; remaining arguments come from melange build configuration + extract := exec.CommandContext(ctx, "tar", "-x", "-C", opts.Destination) // #nosec G204 - destination is a melange-created temp dir + + pipe, err := archive.StdoutPipe() + if err != nil { + return "", fmt.Errorf("creating archive pipe: %w", err) + } + archive.Stderr = os.Stderr + extract.Stdin = pipe + extract.Stdout = os.Stdout + extract.Stderr = os.Stderr + + if err := extract.Start(); err != nil { + return "", fmt.Errorf("starting tar: %w", err) + } + + // Always wait on both commands so the tar child is reaped even when git + // archive fails, and join their errors so a failure in either is surfaced. + // Reporting only one would mask the real cause: if tar dies first, git + // archive sees EPIPE; if git archive dies first, tar sees EOF. + archiveErr := archive.Run() + extractErr := extract.Wait() + if archiveErr != nil || extractErr != nil { + return "", fmt.Errorf("git archive %s %s: %w", resolved, opts.Path, + errors.Join(archiveErr, extractErr)) + } + + return resolved, nil +} + +// gitRevParse resolves rev to a commit hash in the repository containing dir. +// --end-of-options forces rev to be parsed as a revision, so a configured ref +// starting with `-` cannot inject rev-parse options. +func gitRevParse(ctx context.Context, dir, rev string) (string, error) { + out, err := exec.CommandContext(ctx, "git", "-C", dir, "rev-parse", "--verify", "--end-of-options", rev).Output() // #nosec G204 - option injection blocked by --end-of-options + if err != nil { + return "", err + } + return strings.TrimSpace(string(out)), nil +} diff --git a/pkg/build/gitarchive_test.go b/pkg/build/gitarchive_test.go new file mode 100644 index 000000000..af978cf81 --- /dev/null +++ b/pkg/build/gitarchive_test.go @@ -0,0 +1,397 @@ +// Copyright 2026 Chainguard, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package build + +import ( + "os" + "os/exec" + "path/filepath" + "strings" + "testing" + + "github.com/chainguard-dev/clog/slogtest" + "github.com/stretchr/testify/require" + + "chainguard.dev/melange/pkg/config" +) + +// newTestRepo creates a git repository under t.TempDir() containing, on the +// main branch: +// +// sub/chart/Chart.yaml +// sub/chart/values.yaml +// top.txt +// +// It also creates a tag "v1.0.0" at that (initial) commit, and a branch +// "feature" carrying an extra file sub/chart/feature-only.txt that is absent on +// main. HEAD is left on main. It returns the repository root and the main HEAD +// commit hash (== the tag's commit). +func newTestRepo(t *testing.T) (repoDir, commit string) { + t.Helper() + repoDir = t.TempDir() + + write := func(rel, content string) { + p := filepath.Join(repoDir, rel) + require.NoError(t, os.MkdirAll(filepath.Dir(p), 0o755)) + require.NoError(t, os.WriteFile(p, []byte(content), 0o644)) + } + write("sub/chart/Chart.yaml", "name: common\nversion: 1.0.0\n") + write("sub/chart/values.yaml", "replicas: 1\n") + write("top.txt", "top-level\n") + + // Run git with a self-contained identity so the test does not depend on the + // host's global git config. + git := func(args ...string) { + t.Helper() + cmd := exec.Command("git", append([]string{"-C", repoDir}, args...)...) + cmd.Env = append(os.Environ(), + "GIT_AUTHOR_NAME=test", "GIT_AUTHOR_EMAIL=test@example.com", + "GIT_COMMITTER_NAME=test", "GIT_COMMITTER_EMAIL=test@example.com", + ) + out, err := cmd.CombinedOutput() + require.NoErrorf(t, err, "git %v: %s", args, out) + } + git("init", "-q", "-b", "main") + git("add", ".") + git("commit", "-q", "-m", "initial") + + // A tag at the initial commit. Use an explicit message and disable signing + // so this works regardless of the host's tag.gpgSign / annotated-tag config. + git("-c", "tag.gpgSign=false", "tag", "-m", "v1.0.0", "v1.0.0") + + // A branch carrying a file not present on main, so branch-ref tests can + // prove they archive the branch tip rather than main. + git("checkout", "-q", "-b", "feature") + write("sub/chart/feature-only.txt", "feature\n") + git("add", ".") + git("commit", "-q", "-m", "feature commit") + git("checkout", "-q", "main") + + out, err := exec.Command("git", "-C", repoDir, "rev-parse", "HEAD").Output() + require.NoError(t, err) + commit = string(out[:len(out)-1]) // strip trailing newline + return repoDir, commit +} + +func TestGitArchive(t *testing.T) { + ctx := slogtest.Context(t) + repoDir, commit := newTestRepo(t) + + t.Run("extracts subtree at HEAD with path prefix", func(t *testing.T) { + dest := t.TempDir() + resolved, err := gitArchive(ctx, &gitArchiveOptions{ + RepositoryDir: repoDir, + Ref: "HEAD", + Path: "sub/chart", + ExpectedCommit: commit, + Destination: dest, + }) + require.NoError(t, err) + require.Equal(t, commit, resolved) + + // The path prefix is preserved beneath the destination. + require.FileExists(t, filepath.Join(dest, "sub/chart/Chart.yaml")) + require.FileExists(t, filepath.Join(dest, "sub/chart/values.yaml")) + // Only the requested subtree is extracted. + require.NoFileExists(t, filepath.Join(dest, "top.txt")) + }) + + t.Run("archives by tag", func(t *testing.T) { + dest := t.TempDir() + resolved, err := gitArchive(ctx, &gitArchiveOptions{ + RepositoryDir: repoDir, + Ref: "v1.0.0", + Path: "sub/chart", + Destination: dest, + }) + require.NoError(t, err) + // The tag points at the initial commit. + require.Equal(t, commit, resolved) + require.FileExists(t, filepath.Join(dest, "sub/chart/Chart.yaml")) + // feature-only.txt only exists on the feature branch, not at the tag. + require.NoFileExists(t, filepath.Join(dest, "sub/chart/feature-only.txt")) + }) + + t.Run("archives by branch", func(t *testing.T) { + dest := t.TempDir() + resolved, err := gitArchive(ctx, &gitArchiveOptions{ + RepositoryDir: repoDir, + Ref: "feature", + Path: "sub/chart", + Destination: dest, + }) + require.NoError(t, err) + // The branch tip is a different commit than main/the tag. + require.NotEqual(t, commit, resolved) + require.FileExists(t, filepath.Join(dest, "sub/chart/Chart.yaml")) + // The branch-only file proves we archived the branch tip, not main. + require.FileExists(t, filepath.Join(dest, "sub/chart/feature-only.txt")) + }) + + t.Run("anchors at repository toplevel from a subdirectory", func(t *testing.T) { + dest := t.TempDir() + // RepositoryDir points inside the repo, not at its root. git archive + // must still resolve Path relative to the repository root. + resolved, err := gitArchive(ctx, &gitArchiveOptions{ + RepositoryDir: filepath.Join(repoDir, "sub", "chart"), + Ref: "HEAD", + Path: "sub/chart", + ExpectedCommit: commit, + Destination: dest, + }) + require.NoError(t, err) + require.Equal(t, commit, resolved) + require.FileExists(t, filepath.Join(dest, "sub/chart/Chart.yaml")) + }) + + t.Run("defaults RepositoryDir to current directory", func(t *testing.T) { + dest := t.TempDir() + // Empty RepositoryDir means ".", so run with the working directory set + // inside the repo. + t.Chdir(repoDir) + resolved, err := gitArchive(ctx, &gitArchiveOptions{ + Ref: "HEAD", + Path: "sub/chart", + Destination: dest, + }) + require.NoError(t, err) + require.Equal(t, commit, resolved) + require.FileExists(t, filepath.Join(dest, "sub/chart/Chart.yaml")) + }) + + t.Run("branch with older expected-commit archives that commit", func(t *testing.T) { + dest := t.TempDir() + resolved, err := gitArchive(ctx, &gitArchiveOptions{ + RepositoryDir: repoDir, + Ref: "feature", + RefIsBranch: true, + Path: "sub/chart", + ExpectedCommit: commit, // main HEAD: an ancestor of the feature tip + Destination: dest, + }) + require.NoError(t, err) + require.Equal(t, commit, resolved) + // The pinned (older) commit is archived, not the branch tip. + require.FileExists(t, filepath.Join(dest, "sub/chart/Chart.yaml")) + require.NoFileExists(t, filepath.Join(dest, "sub/chart/feature-only.txt")) + }) + + t.Run("branch with expected-commit not on the branch fails", func(t *testing.T) { + out, err := exec.Command("git", "-C", repoDir, "rev-parse", "feature").Output() + require.NoError(t, err) + featureTip := strings.TrimSpace(string(out)) + _, err = gitArchive(ctx, &gitArchiveOptions{ + RepositoryDir: repoDir, + Ref: "main", + RefIsBranch: true, + Path: "sub/chart", + ExpectedCommit: featureTip, // not an ancestor of main + Destination: t.TempDir(), + }) + require.Error(t, err) + require.Contains(t, err.Error(), "not an ancestor") + }) + + t.Run("path starting with dash cannot inject git options", func(t *testing.T) { + outFile := filepath.Join(t.TempDir(), "pwned.tar") + _, err := gitArchive(ctx, &gitArchiveOptions{ + RepositoryDir: repoDir, + Ref: "HEAD", + Path: "--output=" + outFile, + Destination: t.TempDir(), + }) + require.Error(t, err) + // The -- separator forces Path to be a pathspec, so git must not have + // written the file. + require.NoFileExists(t, outFile) + }) + + t.Run("ref starting with dash cannot inject rev-parse options", func(t *testing.T) { + _, err := gitArchive(ctx, &gitArchiveOptions{ + RepositoryDir: repoDir, + Ref: "--all", + Path: "sub/chart", + Destination: t.TempDir(), + }) + require.Error(t, err) + }) + + t.Run("expected-commit mismatch fails", func(t *testing.T) { + dest := t.TempDir() + _, err := gitArchive(ctx, &gitArchiveOptions{ + RepositoryDir: repoDir, + Ref: "HEAD", + Path: "sub/chart", + ExpectedCommit: "deadbeefdeadbeefdeadbeefdeadbeefdeadbeef", + Destination: dest, + }) + require.Error(t, err) + require.Contains(t, err.Error(), "expected") + }) + + t.Run("nonexistent path fails", func(t *testing.T) { + dest := t.TempDir() + _, err := gitArchive(ctx, &gitArchiveOptions{ + RepositoryDir: repoDir, + Ref: "HEAD", + Path: "does/not/exist", + Destination: dest, + }) + require.Error(t, err) + }) + + t.Run("missing required inputs fail", func(t *testing.T) { + for _, tc := range []struct { + name string + opts *gitArchiveOptions + }{ + {"no ref", &gitArchiveOptions{RepositoryDir: repoDir, Path: "sub/chart", Destination: t.TempDir()}}, + {"no path", &gitArchiveOptions{RepositoryDir: repoDir, Ref: "HEAD", Destination: t.TempDir()}}, + {"no destination", &gitArchiveOptions{RepositoryDir: repoDir, Ref: "HEAD", Path: "sub/chart"}}, + } { + t.Run(tc.name, func(t *testing.T) { + _, err := gitArchive(ctx, tc.opts) + require.Error(t, err) + }) + } + }) +} + +func TestGitRevParse(t *testing.T) { + ctx := slogtest.Context(t) + repoDir, commit := newTestRepo(t) + + t.Run("resolves HEAD to full commit", func(t *testing.T) { + got, err := gitRevParse(ctx, repoDir, "HEAD") + require.NoError(t, err) + require.Equal(t, commit, got) + }) + + t.Run("resolves from a subdirectory", func(t *testing.T) { + got, err := gitRevParse(ctx, filepath.Join(repoDir, "sub", "chart"), "HEAD") + require.NoError(t, err) + require.Equal(t, commit, got) + }) + + t.Run("unknown ref fails", func(t *testing.T) { + _, err := gitRevParse(ctx, repoDir, "no-such-ref") + require.Error(t, err) + }) +} + +// TestGitArchive_ExportIgnore locks in that git archive honors .gitattributes: +// paths marked export-ignore are omitted from the archived output. +func TestGitArchive_ExportIgnore(t *testing.T) { + ctx := slogtest.Context(t) + repoDir := t.TempDir() + + write := func(rel, content string) { + p := filepath.Join(repoDir, rel) + require.NoError(t, os.MkdirAll(filepath.Dir(p), 0o755)) + require.NoError(t, os.WriteFile(p, []byte(content), 0o644)) + } + git := func(args ...string) { + t.Helper() + cmd := exec.Command("git", append([]string{"-C", repoDir}, args...)...) + cmd.Env = append(os.Environ(), + "GIT_AUTHOR_NAME=test", "GIT_AUTHOR_EMAIL=test@example.com", + "GIT_COMMITTER_NAME=test", "GIT_COMMITTER_EMAIL=test@example.com", + ) + out, err := cmd.CombinedOutput() + require.NoErrorf(t, err, "git %v: %s", args, out) + } + + write("chart/Chart.yaml", "name: c\n") + write("chart/ignored.txt", "should be excluded\n") + write("chart/.gitattributes", "ignored.txt export-ignore\n") + git("init", "-q", "-b", "main") + git("add", ".") + git("commit", "-q", "-m", "initial") + + dest := t.TempDir() + _, err := gitArchive(ctx, &gitArchiveOptions{ + RepositoryDir: repoDir, + Ref: "HEAD", + Path: "chart", + Destination: dest, + }) + require.NoError(t, err) + + require.FileExists(t, filepath.Join(dest, "chart/Chart.yaml")) + // export-ignore means git archive omits this file. + require.NoFileExists(t, filepath.Join(dest, "chart/ignored.txt")) +} + +// TestMaybeGitArchiveSource_Errors covers the placement/uniqueness guards, which +// live in maybeGitArchiveSource (not the standalone gitArchive helper). +func TestMaybeGitArchiveSource_Errors(t *testing.T) { + ctx := slogtest.Context(t) + ga := config.Pipeline{Uses: "git-archive", With: map[string]string{"path": "x"}} + + t.Run("rejects multiple git-archive steps", func(t *testing.T) { + b := &Build{Configuration: &config.Configuration{ + Pipeline: []config.Pipeline{ga, ga}, + }} + _, err := b.maybeGitArchiveSource(ctx) + require.Error(t, err) + require.Contains(t, err.Error(), "at most once") + }) + + t.Run("rejects git-archive in a subpackage", func(t *testing.T) { + b := &Build{Configuration: &config.Configuration{ + Subpackages: []config.Subpackage{{Name: "sub", Pipeline: []config.Pipeline{ga}}}, + }} + _, err := b.maybeGitArchiveSource(ctx) + require.Error(t, err) + require.Contains(t, err.Error(), "top-level step in the main pipeline") + }) + + t.Run("rejects git-archive nested in a sub-pipeline", func(t *testing.T) { + b := &Build{Configuration: &config.Configuration{ + Pipeline: []config.Pipeline{{Pipeline: []config.Pipeline{ga}}}, + }} + _, err := b.maybeGitArchiveSource(ctx) + require.Error(t, err) + require.Contains(t, err.Error(), "top-level step in the main pipeline") + }) + + t.Run("rejects git-archive in the test pipeline", func(t *testing.T) { + b := &Build{Configuration: &config.Configuration{ + Test: &config.Test{Pipeline: []config.Pipeline{ga}}, + }} + _, err := b.maybeGitArchiveSource(ctx) + require.Error(t, err) + require.Contains(t, err.Error(), "test pipeline") + }) + + t.Run("rejects git-archive in a subpackage test pipeline", func(t *testing.T) { + b := &Build{Configuration: &config.Configuration{ + Subpackages: []config.Subpackage{{Name: "sub", Test: &config.Test{Pipeline: []config.Pipeline{ga}}}}, + }} + _, err := b.maybeGitArchiveSource(ctx) + require.Error(t, err) + require.Contains(t, err.Error(), "test pipeline") + }) + + t.Run("no git-archive step returns a non-nil no-op cleanup", func(t *testing.T) { + b := &Build{Configuration: &config.Configuration{ + Pipeline: []config.Pipeline{{Uses: "strip"}}, + }} + cleanup, err := b.maybeGitArchiveSource(ctx) + require.NoError(t, err) + require.NotNil(t, cleanup) + cleanup() // safe to call + }) +} diff --git a/pkg/build/pipelines/README.md b/pkg/build/pipelines/README.md index 5f898d1df..050f0faeb 100644 --- a/pkg/build/pipelines/README.md +++ b/pkg/build/pipelines/README.md @@ -9,6 +9,7 @@ new built-in pipelines, consult [Creating a new built-in pipeline](/docs/PIPELIN - [fetch](#fetch) - [git-am](#git-am) +- [git-archive](#git-archive) - [git-checkout](#git-checkout) - [patch](#patch) - [strip](#strip) @@ -46,6 +47,19 @@ Apply patches with git am | ---- | -------- | ----------- | ------- | | patches | true | A list of patches to apply with git am, as a whitespace delimited string. Patches are resolved relative to the workspace root, which is where melange copies the contents of the source directory (--source-dir, defaulting to the directory containing the melange YAML file). This is the same convention used by the 'patch' pipeline: place patch files in the package's source directory (e.g. ./my-package/) alongside the YAML file. This pipeline assumes that git-checkout used the default destination ('.'), so the workspace root is the git repository. If git-checkout clones into a subdirectory, the patches must include the path relative to the workspace root. | | +## git-archive + +Archive sources from the local git repository + +### Inputs + +| Name | Required | Description | Default | +| ---- | -------- | ----------- | ------- | +| branch | false | The branch to archive. Branch and tag are mutually exclusive. For reproducibility, prefer tag. | | +| expected-commit | false | The commit to verify against. A tag must resolve to it exactly. With branch, it pins what is archived and (matching git-checkout) may be an older commit on the branch; a commit not on the branch fails the build. When neither tag nor branch is set, this defaults to the commit melange is building, so the assurance holds automatically. | | +| path | true | Path within the repository (relative to the repository root) to extract into the workspace. The extracted files retain this path prefix in the workspace, matching the layout of the source repository. | | +| tag | false | The tag to archive. For reproducibility, tag is generally favored over branch. Branch and tag are mutually exclusive. If neither is specified, the commit melange is building (the config file's repository commit) is archived. | | + ## git-checkout Check out sources from git diff --git a/pkg/build/pipelines/git-archive.yaml b/pkg/build/pipelines/git-archive.yaml new file mode 100644 index 000000000..f364cc3a4 --- /dev/null +++ b/pkg/build/pipelines/git-archive.yaml @@ -0,0 +1,66 @@ +name: Archive sources from the local git repository + +# git-archive populates the build workspace from a subtree of the local git +# repository that contains this melange configuration, WITHOUT cloning over the +# network and without mounting the repository's .git history into the build +# sandbox. +# +# It is intended for monorepos where the package definition and the source it +# packages live in the same repository. By default it archives `path` at the +# very commit melange is building (the commit of the config file's repository), +# so the packaged source matches the source as committed at that commit. +# +# Because the archive is produced with `git archive`, the repository's +# .gitattributes are honored: paths marked `export-ignore` are omitted and +# `export-subst` placeholders are expanded. The result is the committed tree as +# filtered by those export rules, not necessarily a byte-for-byte copy. In a +# repository with many contributors, note that a committed .gitattributes can +# influence (via export-subst) the content of the archived files. +# +# The archive is performed host-side by melange before the sandbox starts, so it +# requires `git` and `tar` on the host (the `needs` below only provisions the +# sandbox). Melange resolves the commit, verifies it, and extracts only `path` +# into the workspace. This step's in-sandbox body is therefore a no-op marker; +# it exists so the source acquisition is declared honestly in the manifest and +# so SBOM provenance can be recorded. +# +# git-archive is the source-acquisition mechanism, so it supersedes any +# --source-dir passed to melange. Uncommitted working-tree changes under `path` +# are never included (a warning is logged when they exist). + +needs: + packages: + - busybox + +inputs: + path: + description: | + Path within the repository (relative to the repository root) to extract + into the workspace. The extracted files retain this path prefix in the + workspace, matching the layout of the source repository. + required: true + tag: + description: | + The tag to archive. For reproducibility, tag is generally favored over + branch. Branch and tag are mutually exclusive. If neither is specified, + the commit melange is building (the config file's repository commit) is + archived. + branch: + description: | + The branch to archive. Branch and tag are mutually exclusive. For + reproducibility, prefer tag. + expected-commit: + description: | + The commit to verify against. A tag must resolve to it exactly. With + branch, it pins what is archived and (matching git-checkout) may be an + older commit on the branch; a commit not on the branch fails the build. + When neither tag nor branch is set, this defaults to the commit melange is + building, so the assurance holds automatically. + +pipeline: + - runs: | + #!/bin/sh + # shellcheck shell=busybox + # The archive was performed host-side by melange before the sandbox + # started; the workspace is already populated. Nothing to do here. + echo "[git archive] workspace populated host-side from ${{inputs.path}}" diff --git a/pkg/cli/build.go b/pkg/cli/build.go index 2912db94e..d48c15770 100644 --- a/pkg/cli/build.go +++ b/pkg/cli/build.go @@ -35,6 +35,7 @@ import ( "golang.org/x/sync/errgroup" "chainguard.dev/melange/pkg/build" + "chainguard.dev/melange/pkg/config" "chainguard.dev/melange/pkg/container" "chainguard.dev/melange/pkg/container/docker" "chainguard.dev/melange/pkg/linter" @@ -179,7 +180,7 @@ func (flags *BuildFlags) BuildOptions(ctx context.Context, args ...string) ([]bu commit, err := detectGitHead(ctx, buildConfigFilePath) if err != nil { log.Warnf("unable to detect commit for build config file: %v", err) - flags.ConfigFileGitCommit = "unknown" + flags.ConfigFileGitCommit = config.UnknownCommit } else { flags.ConfigFileGitCommit = commit } diff --git a/pkg/config/config.go b/pkg/config/config.go index 875294249..055b59000 100644 --- a/pkg/config/config.go +++ b/pkg/config/config.go @@ -59,6 +59,12 @@ const ( purlTypeAPK = "apk" ) +// UnknownCommit is the sentinel used for the config file's repository commit +// when it cannot be determined (for example, when git auto-detection fails). +// It is set by the CLI and checked by consumers such as git-archive source +// resolution, so it is shared here to keep the two ends from drifting. +const UnknownCommit = "unknown" + type Trigger struct { // Optional: The script to run Script string `json:"script,omitempty"` @@ -825,6 +831,40 @@ func (p Pipeline) SBOMPackageForUpstreamSource(licenseDeclared, supplier string, } else if gitPackage != nil { return gitPackage, nil } + + case "git-archive": + // git-archive sources a subtree of a local repository at a pinned ref. + // The meaningful provenance is the subpath plus the commit it was taken + // at. Without a pinned commit there is no immutable identifier to + // record, so (like git-checkout) we emit nothing. + archivePath := with["path"] + expectedCommit := with["expected-commit"] + if expectedCommit == "" || archivePath == "" { + break + } + + pu := &purl.PackageURL{ + Type: "generic", + Name: archivePath, + Version: expectedCommit, + } + if err := pu.Normalize(); err != nil { + return nil, err + } + + idComponents := []string{"Source", archivePath, expectedCommit} + if uniqueID != "" { + idComponents = append(idComponents, uniqueID) + } + + return &sbom.Package{ + IDComponents: idComponents, + Name: archivePath, + Version: expectedCommit, + Namespace: supplier, + PURL: pu, + PrimaryPurpose: "SOURCE", + }, nil } // This is not a fetch or git-checkout step. diff --git a/pkg/config/config_test.go b/pkg/config/config_test.go index a4d739cad..0778015c7 100644 --- a/pkg/config/config_test.go +++ b/pkg/config/config_test.go @@ -997,6 +997,82 @@ func TestGetGitSBOMPackage(t *testing.T) { } } +func TestSBOMPackageForUpstreamSource_GitArchive(t *testing.T) { + const ( + archivePath = "charts/iamguarded/charts/common" + commit = "c64abdbdd120a5fb2e1474ec7243c90ffce7f79c" + supplier = "wolfi" + ) + + testCases := []struct { + name string + with map[string]string + uniqueID string + // nil means we expect no SBOM package (nil, nil). + wantPackage bool + wantIDTail []string + }{ + { + name: "path and expected-commit", + with: map[string]string{"path": archivePath, "expected-commit": commit}, + uniqueID: "0", + wantPackage: true, + wantIDTail: []string{"Source", archivePath, commit, "0"}, + }, + { + name: "path and expected-commit without uniqueID", + with: map[string]string{"path": archivePath, "expected-commit": commit}, + uniqueID: "", + wantPackage: true, + wantIDTail: []string{"Source", archivePath, commit}, + }, + { + // The build backfills expected-commit, but if it is somehow absent + // there is no immutable identifier to record, so emit nothing. + name: "path without expected-commit", + with: map[string]string{"path": archivePath}, + uniqueID: "0", + wantPackage: false, + }, + { + name: "expected-commit without path", + with: map[string]string{"expected-commit": commit}, + uniqueID: "0", + wantPackage: false, + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + p := Pipeline{Uses: "git-archive", With: tc.with} + + pkg, err := p.SBOMPackageForUpstreamSource("Apache-2.0", supplier, tc.uniqueID) + require.NoError(t, err) + + if !tc.wantPackage { + require.Nil(t, pkg) + return + } + + require.NotNil(t, pkg) + require.Equal(t, archivePath, pkg.Name) + require.Equal(t, commit, pkg.Version) + require.Equal(t, supplier, pkg.Namespace) + require.Equal(t, "SOURCE", pkg.PrimaryPurpose) + require.Equal(t, tc.wantIDTail, pkg.IDComponents) + + // PURL: generic type keyed on path + commit. Build the expected + // value the same way the implementation does so normalization + // matches exactly. + wantPURL := &purl.PackageURL{Type: "generic", Name: archivePath, Version: commit} + require.NoError(t, wantPURL.Normalize()) + require.Equal(t, wantPURL.Type, pkg.PURL.Type) + require.Equal(t, wantPURL.Name, pkg.PURL.Name) + require.Equal(t, wantPURL.Version, pkg.PURL.Version) + }) + } +} + func TestSetCap(t *testing.T) { tests := []struct { setcap []Capability