diff --git a/kubernetes/Dockerfile.image-committer b/kubernetes/Dockerfile.image-committer index c4269ad64..996a1dc57 100644 --- a/kubernetes/Dockerfile.image-committer +++ b/kubernetes/Dockerfile.image-committer @@ -13,10 +13,7 @@ # limitations under the License. # Build stage -FROM golang:1.25-alpine AS builder - -# Use Aliyun mirror for faster downloads in China -RUN sed -i 's/dl-cdn.alpinelinux.org/mirrors.aliyun.com/g' /etc/apk/repositories +FROM golang:1.26.5-alpine3.23@sha256:622e56dbc11a8cfe87cafa2331e9a201877271cbff918af53d3be315f3da88cc AS builder WORKDIR /workspace @@ -31,18 +28,31 @@ COPY cmd/image-committer/ cmd/image-committer/ RUN CGO_ENABLED=0 GOOS=linux go build -o /usr/local/bin/image-committer ./cmd/image-committer/ # Runtime stage -FROM alpine:3.19 +FROM alpine:3.23@sha256:fd791d74b68913cbb027c6546007b3f0d3bc45125f797758156952bc2d6daf40 -# Use Aliyun mirror for faster downloads in China -RUN sed -i 's/dl-cdn.alpinelinux.org/mirrors.aliyun.com/g' /etc/apk/repositories +ARG TARGETARCH # Install nerdctl for container operations # nerdctl is used to find containers, commit rootfs, and push images. # We use nerdctl directly (not crictl or ctr) to avoid CRI API version issues. RUN apk add --no-cache \ curl \ - jq \ - nerdctl + jq + +# Install the current upstream minimal client, verify the release checksum, and +# preserve multi-platform builds. The distro package lags the nerdctl behavior +# used by the snapshot path. +RUN case "${TARGETARCH}" in \ + amd64) NERDCTL_SHA256="de3206aeb7cbd5f20f5fb1f55c1e3bf2db1be567812a8a3f5e65eba2488347ee" ;; \ + arm64) NERDCTL_SHA256="76ced9bd0d03f6140f9cf7b927958b654cb8d5ecd3c58af585d096c8bdf9d6c2" ;; \ + *) printf 'unsupported TARGETARCH: %s\n' "${TARGETARCH}" >&2; exit 1 ;; \ + esac \ + && curl -fsSLo /tmp/nerdctl.tgz \ + "https://github.com/containerd/nerdctl/releases/download/v2.3.5/nerdctl-2.3.5-linux-${TARGETARCH}.tar.gz" \ + && printf '%s %s\n' "${NERDCTL_SHA256}" /tmp/nerdctl.tgz | sha256sum -c - \ + && tar -xzf /tmp/nerdctl.tgz -C /usr/local/bin nerdctl \ + && rm /tmp/nerdctl.tgz \ + && nerdctl --version # Create directory for containerd socket mount RUN mkdir -p /var/run/containerd diff --git a/kubernetes/cmd/image-committer/main.go b/kubernetes/cmd/image-committer/main.go index 7717fdf16..0aaf43147 100644 --- a/kubernetes/cmd/image-committer/main.go +++ b/kubernetes/cmd/image-committer/main.go @@ -62,8 +62,9 @@ type ContainerSpec struct { } type discoveredContainer struct { - ID string - Running bool + ID string + Running bool + SourceImage string } type snapshotResult struct { @@ -163,11 +164,28 @@ func main() { fmt.Fprintf(os.Stderr, "ERROR: Failed to find container '%s': %v\n", spec.Name, err) os.Exit(1) } + sourceImage, err := getContainerImage(container.ID) + if err != nil { + fmt.Fprintf(os.Stderr, "ERROR: Failed to identify the source image for container '%s': %v Ensure the sandbox container still exists, then retry the pause operation.\n", spec.Name, err) + os.Exit(1) + } + container.SourceImage = sourceImage - fmt.Printf("Container '%s' -> ID: %s (running: %t)\n", spec.Name, container.ID, container.Running) + fmt.Printf("Container '%s' -> ID: %s (running: %t, source image: %s)\n", spec.Name, container.ID, container.Running, container.SourceImage) containers[spec.Name] = container } + // Kubernetes/containerd may retain unpacked snapshots while garbage-collecting + // the compressed source layers. A committed manifest still references those + // layers, so make them locally available before attempting a registry push. + fmt.Println("\n=== Step 1b: Ensure source image content is available ===") + for _, spec := range containerSpecs { + if err := pullImageContent(containers[spec.Name].SourceImage); err != nil { + fmt.Fprintf(os.Stderr, "ERROR: Failed to make source image content available for container '%s': %v Ensure the snapshot registry secret can read the source image registry, then retry the pause operation.\n", spec.Name, err) + os.Exit(1) + } + } + // Step 2: Flush each running container's filesystem from inside its runtime. // This is required for VM-isolated runtimes such as Kata, where host-side // sync does not flush the guest kernel's page cache. @@ -519,6 +537,53 @@ func commitContainer(containerID, targetImage string) error { return nil } +// getContainerImage returns the source image reference recorded on a container. +func getContainerImage(containerID string) (string, error) { + const imagePrefix = "OPENSANDBOX_SOURCE_IMAGE=" + args := append(nerdctlBaseArgs(), "inspect", "--format", imagePrefix+"{{.Image}}", containerID) + output, err := commandCombinedOutput("nerdctl", args...) + if err != nil { + return "", fmt.Errorf("nerdctl inspect failed for container %s: %v, output: %s", containerID, err, strings.TrimSpace(string(output))) + } + + // nerdctl can emit harmless network-namespace warnings to stderr while + // still returning the requested image on stdout. Prefix the formatted value + // so it can be identified regardless of stdout/stderr interleaving. + for _, line := range strings.Split(string(output), "\n") { + if value, found := strings.CutPrefix(strings.TrimSpace(line), imagePrefix); found && value != "" { + return value, nil + } + } + return "", fmt.Errorf("nerdctl inspect returned an empty source image for container %s", containerID) +} + +// pullImageContent restores any compressed source layers that containerd may +// have garbage-collected after unpacking the image. nerdctl commit reuses those +// layers in the snapshot manifest, and nerdctl push requires their local blobs. +func pullImageContent(sourceImage string) error { + fmt.Printf("Ensuring source image content is available: %s...\n", sourceImage) + + imageParts := strings.Split(sourceImage, "/") + if len(imageParts) == 0 || imageParts[0] == "" { + return fmt.Errorf("invalid source image: %s", sourceImage) + } + registryHost := imageParts[0] + isInsecure := shouldUseInsecureRegistry(registryHost) + loginToRegistryIfConfigured(registryHost, isInsecure) + + pullOpts := append(nerdctlBaseArgs(), "pull", "--all-platforms") + if isInsecure { + pullOpts = append(pullOpts, "--insecure-registry") + } + pullOpts = append(pullOpts, sourceImage) + + output, err := commandCombinedOutput("nerdctl", pullOpts...) + if err != nil { + return fmt.Errorf("failed to pull source image %s: %v, output: %s", sourceImage, err, strings.TrimSpace(string(output))) + } + return nil +} + // pushImage uses nerdctl to push the image to the registry. // nerdctl push does not support --username/--password flags, so we use // nerdctl login first, then nerdctl push with --insecure-registry. @@ -534,27 +599,20 @@ func pushImage(targetImage string) error { isInsecure := shouldUseInsecureRegistry(registryHost) - // Try to login using credentials from mounted secret - credDir := "/var/run/opensandbox/registry" - configPath := filepath.Join(credDir, "config.json") - if _, err := os.Stat(configPath); err == nil { - fmt.Printf("Found registry credentials at %s\n", configPath) - if err := nerdctlLogin(configPath, registryHost, isInsecure); err != nil { - fmt.Fprintf(os.Stderr, "WARNING: nerdctl login failed: %v (will attempt push anyway)\n", err) - } - } else { - fmt.Println("No registry credentials found, assuming insecure or pre-authenticated registry") - } + loginToRegistryIfConfigured(registryHost, isInsecure) // Build push options - pushOpts := append(nerdctlBaseArgs(), "push") + // A committed sandbox image is already a complete local single-platform + // manifest. Without --all-platforms nerdctl first builds a reduced-platform + // temporary image, whose content check tries to pull this brand-new tag from + // the remote registry and fails on the expected 404. + pushOpts := append(nerdctlBaseArgs(), "push", "--all-platforms") if isInsecure { pushOpts = append(pushOpts, "--insecure-registry") } pushOpts = append(pushOpts, targetImage) - cmd := exec.Command("nerdctl", pushOpts...) - output, err := cmd.CombinedOutput() + output, err := commandCombinedOutput("nerdctl", pushOpts...) if err != nil { return fmt.Errorf("failed to push image %s: %v, output: %s", targetImage, err, string(output)) } @@ -562,6 +620,18 @@ func pushImage(targetImage string) error { return nil } +func loginToRegistryIfConfigured(registryHost string, isInsecure bool) { + configPath := filepath.Join("/var/run/opensandbox/registry", "config.json") + if _, err := os.Stat(configPath); err == nil { + fmt.Printf("Found registry credentials at %s\n", configPath) + if err := nerdctlLogin(configPath, registryHost, isInsecure); err != nil { + fmt.Fprintf(os.Stderr, "WARNING: Registry login failed: %v Verify the snapshot registry secret grants access to %s; the operation will continue in case the registry is already authenticated.\n", err, registryHost) + } + return + } + fmt.Println("No registry credentials found, assuming insecure or pre-authenticated registry") +} + // nerdctlLogin extracts credentials from a Docker config.json and runs nerdctl login. func nerdctlLogin(configPath, registryHost string, insecure bool) error { data, err := os.ReadFile(configPath) diff --git a/kubernetes/cmd/image-committer/main_test.go b/kubernetes/cmd/image-committer/main_test.go index df9add332..db9977cf1 100644 --- a/kubernetes/cmd/image-committer/main_test.go +++ b/kubernetes/cmd/image-committer/main_test.go @@ -164,6 +164,80 @@ func TestGetContainerIDByNerdctlReturnsHelpfulErrorWhenBothLookupsAreEmpty(t *te } } +func TestGetContainerImageReturnsSourceReferenceAfterWarning(t *testing.T) { + original := commandCombinedOutput + t.Cleanup(func() { commandCombinedOutput = original }) + commandCombinedOutput = func(name string, args ...string) ([]byte, error) { + if name != "nerdctl" { + t.Fatalf("unexpected command %q", name) + } + if !contains(args, "inspect") || !contains(args, "container-1") { + t.Fatalf("unexpected inspect arguments: %v", args) + } + return []byte("time=\"2026-08-13T13:55:17Z\" level=warning msg=\"failed to inspect NetNS\"\nOPENSANDBOX_SOURCE_IMAGE=registry.example.com/quovy/sandbox:release-1\ntime=\"2026-08-13T13:55:18Z\" level=warning msg=\"cleanup warning\"\n"), nil + } + + image, err := getContainerImage("container-1") + if err != nil { + t.Fatalf("expected source image lookup to succeed, got %v", err) + } + if image != "registry.example.com/quovy/sandbox:release-1" { + t.Fatalf("unexpected source image %q", image) + } +} + +func TestPullImageContentFetchesCompressedLayers(t *testing.T) { + original := commandCombinedOutput + t.Cleanup(func() { commandCombinedOutput = original }) + t.Setenv("SNAPSHOT_REGISTRY_INSECURE", "false") + + var gotArgs []string + commandCombinedOutput = func(name string, args ...string) ([]byte, error) { + if name != "nerdctl" { + t.Fatalf("unexpected command %q", name) + } + gotArgs = append([]string(nil), args...) + return []byte("pulled"), nil + } + + image := "registry.example.com/quovy/sandbox:release-1" + if err := pullImageContent(image); err != nil { + t.Fatalf("expected source content pull to succeed, got %v", err) + } + if !contains(gotArgs, "pull") || !contains(gotArgs, image) { + t.Fatalf("unexpected pull arguments: %v", gotArgs) + } + if !contains(gotArgs, "--all-platforms") { + t.Fatalf("source pull must bypass containerd's transfer-service cache: %v", gotArgs) + } +} + +func TestPushImagePreservesCommittedManifest(t *testing.T) { + original := commandCombinedOutput + t.Cleanup(func() { commandCombinedOutput = original }) + t.Setenv("SNAPSHOT_REGISTRY_INSECURE", "false") + + var gotArgs []string + commandCombinedOutput = func(name string, args ...string) ([]byte, error) { + if name != "nerdctl" { + t.Fatalf("unexpected command %q", name) + } + gotArgs = append([]string(nil), args...) + return []byte("pushed"), nil + } + + image := "registry.example.com/quovy/sandbox:snapshot-1" + if err := pushImage(image); err != nil { + t.Fatalf("expected snapshot push to succeed, got %v", err) + } + if !contains(gotArgs, "push") || !contains(gotArgs, image) { + t.Fatalf("unexpected push arguments: %v", gotArgs) + } + if !contains(gotArgs, "--all-platforms") { + t.Fatalf("snapshot push must preserve the committed manifest: %v", gotArgs) + } +} + func contains(values []string, target string) bool { for _, value := range values { if value == target {