many: compression all the time

cmd/image-builder/build.go

@@ -7,6 +7,7 @@ import (
 	"strings"
 
 	"github.com/osbuild/image-builder/pkg/imagefilter"
+	"github.com/osbuild/image-builder/pkg/manifest"
 	"github.com/osbuild/image-builder/pkg/progress"
 )
 
@@ -20,6 +21,8 @@ type buildOptions struct {
 	WriteManifest bool
 	WriteBuildlog bool
 	Metrics       bool
+
+	Compression manifest.Compression
 }
 
 func buildImage(pbar progress.ProgressBar, res *imagefilter.Result, osbuildManifest []byte, opts *buildOptions) (string, error) {
@@ -59,15 +62,15 @@ func buildImage(pbar progress.ProgressBar, res *imagefilter.Result, osbuildManif
 
 		osbuildOpts.BuildLog = f
 	}
-	if err := progress.RunOSBuild(pbar, osbuildManifest, res.ImgType.Exports(), osbuildOpts); err != nil {
+	if err := progress.RunOSBuild(pbar, osbuildManifest, res.ImgType.Exports(opts.Compression), osbuildOpts); err != nil {
 		return "", err
 	}
 	// Rename *sigh*, see https://github.com/osbuild/image-builder/pull/1039
 	// for my preferred way. Every frontend to images has to duplicate
 	// similar code like this.
-	pipelineDir := filepath.Join(opts.OutputDir, res.ImgType.Exports()[0])
-	srcName := filepath.Join(pipelineDir, res.ImgType.Filename())
-	imgExt := strings.SplitN(res.ImgType.Filename(), ".", 2)[1]
+	pipelineDir := filepath.Join(opts.OutputDir, res.ImgType.Exports(opts.Compression)[0])
+	srcName := filepath.Join(pipelineDir, res.ImgType.Filename(opts.Compression))
+	imgExt := strings.SplitN(res.ImgType.Filename(opts.Compression), ".", 2)[1]
 	dstName := filepath.Join(opts.OutputDir, fmt.Sprintf("%s.%v", basename, imgExt))
 	if err := os.Rename(srcName, dstName); err != nil {
 		return "", fmt.Errorf("cannot rename artifact to final name: %w", err)

cmd/image-builder/cmd.go

@@ -210,6 +210,7 @@ func setupManifestCmd() (*cobra.Command, error) {
 	manifestCmd.Flags().Bool("bootc-no-default-kernel-args", false, `don't use the default kernel arguments`)
 	manifestCmd.Flags().Bool("bootc-pull-container", false, `pull bootc container from remote location instead of using it from local container storage`)
 	manifestCmd.Flags().Uint64("image-size", 0, `override the default image size in bytes`)
+	manifestCmd.Flags().String("compression", "", `override the default compression for the image (e.g. xz, zstd, gzip, none)`)
 	manifestCmd.Flags().Bool("use-librepo", true, `use librepo to download packages (disable if you use old versions of osbuild)`)
 	if err := manifestCmd.Flags().MarkHidden("use-librepo"); err != nil {
 		return nil, err

cmd/image-builder/describeimg.go

@@ -147,7 +147,7 @@ func describeImage(img *imagefilter.Result, out io.Writer) error {
 		Type:             img.ImgType.Name(),
 		Bootmode:         img.ImgType.BootMode().String(),
 		PartitionType:    img.ImgType.PartitionType().String(),
-		DefaultFilename:  img.ImgType.Filename(),
+		DefaultFilename:  img.ImgType.Filename(""),
 		BuildPipelines:   m.BuildPipelines(),
 		PayloadPipelines: m.PayloadPipelines(),
 		Packages:         pkgSets,

cmd/image-builder/describeimg_test.go

@@ -39,11 +39,15 @@ build_pipelines:
 payload_pipelines:
   - os
   - archive
+  - gzip
+  - xz
+  - zstd
 packages:
   build:
     include:
       - coreutils
       - glibc
+      - gzip
       - platform-python
       - policycoreutils
       - python3
@@ -52,6 +56,7 @@ packages:
       - systemd
       - tar
       - xz
+      - zstd
     exclude: []
   os:
     include:

cmd/image-builder/main.go

@@ -23,6 +23,7 @@ import (
 	"github.com/osbuild/image-builder/pkg/customizations/subscription"
 	"github.com/osbuild/image-builder/pkg/distro/generic"
 	"github.com/osbuild/image-builder/pkg/imagefilter"
+	"github.com/osbuild/image-builder/pkg/manifest"
 	"github.com/osbuild/image-builder/pkg/manifestgen"
 	"github.com/osbuild/image-builder/pkg/osbuild"
 	"github.com/osbuild/image-builder/pkg/ostree"
@@ -72,7 +73,7 @@ func basenameFor(img *imagefilter.Result, userBasename string) string {
 		//
 		// This code assumes that all our ImgType filesnames have
 		// $name.$ext.$extraExt (e.g. disk.qcow2 or disk.raw.xz)
-		l := strings.SplitN(img.ImgType.Filename(), ".", 2)
+		l := strings.SplitN(img.ImgType.Filename(""), ".", 2)
 		if len(l) > 1 && l[1] != "" {
 			imgExt := fmt.Sprintf(".%s", l[1])
 			userBasename = strings.TrimSuffix(userBasename, imgExt)
@@ -374,7 +375,7 @@ func getImage(cmd *cobra.Command, args []string) (*imagefilter.Result, error) {
 			return nil, err
 		}
 	}
-	if len(img.ImgType.Exports()) > 1 {
+	if len(img.ImgType.Exports("")) > 1 {
 		return nil, fmt.Errorf("image %q has multiple exports: this is current unsupport: please report this as a bug", basenameFor(img, ""))
 	}
 	return img, err
@@ -484,6 +485,10 @@ func cmdManifestWrapper(pbar progress.ProgressBar, cmd *cobra.Command, args []st
 	if err != nil {
 		return err
 	}
+	compression, err := cmd.Flags().GetString("compression")
+	if err != nil {
+		return err
+	}
 
 	// no error check here as this is (deliberately) not defined on
 	// "manifest" (if "images" learn to set the output filename in
@@ -529,6 +534,7 @@ func cmdManifestWrapper(pbar progress.ProgressBar, cmd *cobra.Command, args []st
 		BootcOmitDefaultKernelArgs: bootcOmitDefaultKernelArgs,
 		BootcRemote:                bootcRemote,
 		ImageSize:                  imageSize,
+		Compression:                manifest.Compression(compression),
 		WithSBOM:                   withSBOM,
 		WithRPMList:                withRPMList,
 		IgnoreWarnings:             ignoreWarnings,
@@ -681,6 +687,10 @@ func cmdBuild(cmd *cobra.Command, args []string) error {
 		}
 	}
 
+	compression, err := cmd.Flags().GetString("compression")
+	if err != nil {
+		return err
+	}
 	buildOpts := &buildOptions{
 		OutputDir:      outputDir,
 		OutputBasename: outputBasename,
@@ -689,6 +699,7 @@ func cmdBuild(cmd *cobra.Command, args []string) error {
 		WriteBuildlog:  withBuildlog,
 		Metrics:        withMetrics,
 		JSONOutput:     format == "json",
+		Compression:    manifest.Compression(compression),
 	}
 	if runInVm {
 		buildOpts.InVm = []string{"image"}

cmd/image-builder/main_test.go

@@ -224,6 +224,58 @@ func TestManifestIntegrationSmoke(t *testing.T) {
 	}
 }
 
+func TestManifestIntegrationCompression(t *testing.T) {
+	restore := main.MockManifestgenDepsolver(fakeDepsolve)
+	defer restore()
+
+	restore = main.MockManifestgenContainerResolver(fakeContainerResolver)
+	defer restore()
+
+	restore = main.MockNewRepoRegistry(testrepos.New)
+	defer restore()
+
+	simpleBP := `
+[[customizations.user]]
+name = "alice"
+`
+
+	for _, tc := range []struct {
+		name        string
+		compression string
+		expectPipe  string
+	}{
+		{"default", "", "image"},
+		{"zstd", "zstd", "zstd"},
+	} {
+		t.Run(tc.name, func(t *testing.T) {
+			args := []string{
+				"manifest",
+				"minimal-raw",
+				"--arch=x86_64",
+				"--distro=fedora-43",
+				fmt.Sprintf("--blueprint=%s", makeTestBlueprint(t, simpleBP)),
+			}
+			if tc.compression != "" {
+				args = append(args, fmt.Sprintf("--compression=%s", tc.compression))
+			}
+
+			restore = main.MockOsArgs(args)
+			defer restore()
+
+			var fakeStdout bytes.Buffer
+			restore = main.MockOsStdout(&fakeStdout)
+			defer restore()
+
+			err := main.Run()
+			require.NoError(t, err)
+
+			pipelineNames, err := manifesttest.PipelineNamesFrom(fakeStdout.Bytes())
+			require.NoError(t, err)
+			assert.Contains(t, pipelineNames, tc.expectPipe)
+		})
+	}
+}
+
 func TestManifestIntegrationAutoDetectDistro(t *testing.T) {
 	restore := main.MockManifestgenDepsolver(fakeDepsolve)
 	defer restore()

cmd/image-builder/manifest.go

@@ -12,6 +12,7 @@ import (
 	"github.com/osbuild/image-builder/pkg/customizations/subscription"
 	"github.com/osbuild/image-builder/pkg/distro"
 	"github.com/osbuild/image-builder/pkg/imagefilter"
+	"github.com/osbuild/image-builder/pkg/manifest"
 	"github.com/osbuild/image-builder/pkg/manifestgen"
 	"github.com/osbuild/image-builder/pkg/osbuild"
 	"github.com/osbuild/image-builder/pkg/ostree"
@@ -33,6 +34,7 @@ type manifestOptions struct {
 	BootcOmitDefaultKernelArgs bool
 	BootcRemote                bool
 	ImageSize                  uint64
+	Compression                manifest.Compression
 	Subscription               *subscription.ImageOptions
 	RpmDownloader              osbuild.RpmDownloader
 	WithSBOM                   bool
@@ -109,6 +111,7 @@ func generateManifest(repoDir string, extraRepos []string, img *imagefilter.Resu
 		OSTree:       opts.Ostree,
 		Subscription: opts.Subscription,
 		Size:         opts.ImageSize,
+		Compression:  opts.Compression,
 		Bootc: &distro.BootcImageOptions{
 			InstallerPayloadRef:      opts.BootcInstallerPayloadRef,
 			OmitDefaultKernelArgs:    opts.BootcOmitDefaultKernelArgs,

data/distrodefs/bootc-generic/imagetypes.yaml

@@ -214,7 +214,8 @@ image_types:
     <<: *raw_image_type
     filename: "pxe.tar.xz"
     mime_type: "application/x-tar"
-    compression: "xz"
+    compression:
+      default: "xz"
     image_func: "pxe_tar"
     exports: ["xz"]
     iso_config:

data/distrodefs/eln-11/disk.yaml

@@ -520,7 +520,8 @@ image_types:
   "azure": &azure
     <<: *vhd
     exports: ["xz"]
-    compression: "xz"
+    compression:
+      default: "xz"
     filename: "disk.vhd.xz"
     partition_table:
       x86_64:
@@ -799,7 +800,8 @@ image_types:
     name_aliases: []
     exports: ["xz"]
     filename: "image.raw.xz"
-    compression: "xz"
+    compression:
+      default: "xz"
     blueprint:
       supported_options: *supported_options_disk
 

data/distrodefs/eln-11/network.yaml

@@ -30,7 +30,8 @@
 image_types:
   "pxe-tar-xz":
     filename: "pxe.tar.xz"
-    compression: "xz"
+    compression:
+      default: "xz"
     mime_type: "application/x-tar"
     image_func: "pxe_tar"
     exports: ["xz"]

data/distrodefs/eln-11/wsl.yaml

@@ -4,7 +4,8 @@ image_types:
     mime_type: "application/x-tar"
     image_func: "tar"
     exports: ["xz"]
-    compression: "xz"
+    compression:
+      default: "xz"
     platforms:
       - arch: "x86_64"
       - arch: "aarch64"

data/distrodefs/fedora/imagetypes.yaml

@@ -1338,7 +1338,8 @@
   atomic_raw_common: &atomic_raw_common
     <<: *atomic_disk_common
     filename: "image.raw.xz"
-    compression: "xz"
+    compression:
+      default: "xz"
     mime_type: "application/xz"
     exports: ["xz"]
     platforms:
@@ -1768,7 +1769,8 @@ image_types:
     <<: *ostree_imgtype_common
     name_aliases: ["iot-raw-image", "fedora-iot-raw-image"]
     filename: "image.raw.xz"
-    compression: "xz"
+    compression:
+      default: "xz"
     mime_type: "application/xz"
     default_size: "6 GiB"
     bootable: true
@@ -2049,15 +2051,16 @@ image_types:
     blueprint:
       supported_options: *supported_options_ostree_commit
 
-  "minimal-raw-xz": &minimal_raw_xz
-    name_aliases: ["minimal-raw"]
-    filename: "disk.raw.xz"
-    compression: "xz"
-    mime_type: "application/xz"
+  "minimal-raw": &minimal_raw
+    filename: "disk.raw"
+    compression:
+      default: "none"
+      allowed: ["none", "xz", "zstd", "gzip"]
+    mime_type: "application/octet-stream"
     bootable: true
     default_size: "2 GiB"
     image_func: "disk"
-    exports: ["xz"]
+    exports: ["image"]
     required_partition_sizes: *default_required_dir_sizes
     platforms:
       - <<: *x86_64_uefi_platform
@@ -2150,11 +2153,19 @@ image_types:
     blueprint:
       supported_options: *supported_options_disk
 
+  "minimal-raw-xz":
+    <<: *minimal_raw
+    filename: "disk.raw.xz"
+    compression:
+      default: "xz"
+    mime_type: "application/xz"
+    exports: ["xz"]
+
   "minimal-raw-zst":
-    <<: *minimal_raw_xz
-    name_aliases: []
+    <<: *minimal_raw
     filename: "disk.raw.zst"
-    compression: zstd
+    compression:
+      default: "zstd"
     exports: ["zstd"]
 
   "iot-installer":
@@ -2454,7 +2465,8 @@ image_types:
     # correct suffix, see:
     # https://learn.microsoft.com/en-us/windows/wsl/build-custom-distro#what-are-wsl-root-filesystem-tar-files
     filename: "image.wsl"
-    compression: "xz"
+    compression:
+      default: "xz"
     mime_type: "application/x-tar"
     image_func: "tar"
     exports: ["xz"]
@@ -2740,7 +2752,8 @@ image_types:
 
   "pxe-tar-xz":
     filename: "pxe.tar.xz"
-    compression: "xz"
+    compression:
+      default: "xz"
     mime_type: "application/x-tar"
     image_func: "pxe_tar"
     exports: ["xz"]
@@ -2847,7 +2860,8 @@ image_types:
   "cloud-ec2": &cloud_ec2
     <<: *cloud_base
     filename: "image.raw.xz"
-    compression: "xz"
+    compression:
+      default: "xz"
     mime_type: "application/xz"
     exports: ["xz"]
     image_config: