diff --git a/tests/README.md b/tests/README.md index 936091a71dd..286635dbb89 100644 --- a/tests/README.md +++ b/tests/README.md @@ -397,6 +397,7 @@ of Firecracker processes. There are two layers: - `rootfs` — the rootfs disk path, composed from `guest_kernel` + `rootfs_mode` (Ubuntu 24.04 for 5.10, Amazon Linux 2023 otherwise). - `pci_enabled` — auto-parametrized over `True`/`False`. + - `vm_backend` — auto-parametrized over `"kvm"`. - `cpu_template` — `None` by default. - `huge_pages` — `HugePagesConfig.NONE` by default. See note below. - `vcpu_count`, `mem_size_mib` — `2` and `256` by default. diff --git a/tests/conftest.py b/tests/conftest.py index 200a8ac4f5d..4f35efbe20c 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -34,9 +34,17 @@ from framework import defs, utils from framework.artifacts import ALL_GUEST_KERNELS, disks from framework.defs import ARTIFACT_DIR, DEFAULT_BINARY_DIR -from framework.microvm import HugePagesConfig, MicroVMFactory, SnapshotType +from framework.microvm import MicroVMFactory, SnapshotType from framework.properties import global_props from framework.utils_cpu_templates import get_cpu_template_name +from framework.utils_hugepages import HugePagesConfig +from framework.vm_backend import ( + available_vm_backend_params, + available_vm_backends, + get_vm_backend, + vm_backend_probe_report, +) +from framework.vm_lifecycle import VmLifecycle from host_tools.metrics import get_metrics_logger from host_tools.network import NetNs @@ -104,10 +112,20 @@ def pytest_report_header(): [ f"EC2 AMI: {global_props.ami}", f"EC2 Instance ID: {global_props.instance_id}", + *vm_backend_probe_report(), ] ) +@pytest.hookimpl(trylast=True) +def pytest_collection_modifyitems(items): + """Fail only when the selected tests require an unavailable VM backend.""" + if not available_vm_backends() and any( + "vm_backend" in getattr(item, "fixturenames", ()) for item in items + ): + pytest.exit("No VM backends are available on this host", returncode=5) + + @pytest.hookimpl(wrapper=True, tryfirst=True) def pytest_runtest_makereport(item, call): # pylint:disable=unused-argument """Plugin to get test results in fixtures @@ -389,7 +407,9 @@ def get(self, _netns_id): @pytest.fixture() # pylint: disable=unused-argument -def microvm_factory(request, record_property, results_dir, netns_factory, reap_orphans): +def microvm_factory( + request, record_property, results_dir, netns_factory, reap_orphans, vm_backend +): """Fixture to create microvms simply. `reap_orphans` is requested only for teardown ordering (reaping runs @@ -420,6 +440,7 @@ def microvm_factory(request, record_property, results_dir, netns_factory, reap_o binary_dir, netns_factory=netns_factory, custom_cpu_template=custom_cpu_template, + backend=vm_backend, ) yield uvm_factory @@ -554,6 +575,20 @@ def huge_pages(request): return getattr(request, "param", HugePagesConfig.NONE) +@pytest.fixture(params=available_vm_backend_params()) +def vm_backend(request, record_property): + """Backend used by the `uvm*` fixtures. + + Tests are auto-multiplied over the backends this host can actually run + (`available_vm_backends`). + """ + if request.param not in available_vm_backends(): + pytest.fail(f"Backend {request.param!r} not available on this host") + backend = get_vm_backend(request.param) + record_property("vm_backend", str(backend)) + return backend + + # ============================================================================= # Composable uvm fixture system # ============================================================================= @@ -570,19 +605,37 @@ def huge_pages(request): # defaults baked into their bodies. Override a dim with # `@pytest.mark.parametrize(, [...], indirect=True)` or use the helpers # from `framework.artifacts` (`pin_guest_kernel`, `pin_rootfs_mode`, -# `pin_pci`, `pin_cpu_template`). +# `pin_pci`) and `framework.utils_cpu_templates` (`pin_cpu_template`). # # Module-level `pytestmark` works for tests that don't override that same # dim per-test — pytest's parametrize markers do NOT merge: a pytestmark + # per-test parametrize on the same argname raises "duplicate parametrization". # +# Modules may also customise a lifecycle *stage* by defining a fixture with +# the same name as `uvm_configured` or `uvm_booted` (fixture shadowing). +# Pytest resolves the module-local fixture when the shared later stages +# (`uvm_restored`, `uvm_any`) request that name, so booted and restored +# variants both pick up the customisation. The override may request the +# stage it replaces (same-name chaining) to decorate rather than +# reimplement it, e.g. `def uvm_booted(uvm_booted): ...`. +# +# Shadow a stage only when every lifecycle-aware test in the module wants +# the identical customisation; files with several boot configurations +# should keep explicitly named fixtures. Prefer overriding a dimension +# over shadowing a stage when the customisation is a plain configuration +# value with an existing dimension; knobs too rare to be worth a dimension +# (e.g. `boot_args`) shadow `uvm_configured` instead. See `test_rng.py` +# (stage overrides), `test_sysgenid.py` (stage decoration) and +# `test_fips.py` (custom boot args via a `uvm_configured` override). +# # Dimensions: -# guest_kernel Path to a guest kernel artifact auto-multiplied +# guest_kernel Logical guest kernel variant auto-multiplied # over ALL_GUEST_KERNELS # rootfs_mode "ro" | "rw" default "ro" # rootfs Path to a rootfs disk, composed from (composed) # guest_kernel + rootfs_mode (Ubuntu for 5.10, AL2023 otherwise) # pci_enabled True / False auto-multiplied +# vm_backend "kvm" auto-multiplied # cpu_template None | static name | custom dict default None # huge_pages HugePagesConfig default NONE # vcpu_count int default 2 @@ -591,21 +644,21 @@ def huge_pages(request): @pytest.fixture(params=ALL_GUEST_KERNELS) def guest_kernel(request, record_property): - """Path to the guest kernel artifact. + """Logical guest kernel variant. Default: parametrized over every supported kernel, so every test that requests this fixture (directly or via `uvm` etc.) runs once per kernel. - Override with `@pin_guest_kernel()` (from + Override with `@pin_guest_kernel()` (from `framework.artifacts`) to restrict to one kernel or a smaller subset — e.g. for tests of Firecracker functionality that don't depend on the guest kernel, use `@pin_guest_kernel(GUEST_KERNEL_DEFAULT)`. """ - kernel_path = request.param - if kernel_path is None: + kernel = request.param + if kernel is None: pytest.fail(f"No kernel artifacts found in {ARTIFACT_DIR}") - record_property("guest_kernel", kernel_path.stem[2:]) - return kernel_path + record_property("guest_kernel", kernel.metric_id) + return kernel @pytest.fixture @@ -623,7 +676,7 @@ def rootfs(guest_kernel, rootfs_mode): Ubuntu for 5.10, AL2023 otherwise (AL2023 does not officially support 5.10). """ - distro = "ubuntu" if guest_kernel.stem[2:] == "linux-5.10" else "amazonlinux" + distro = "ubuntu" if guest_kernel.version.startswith("5.10") else "amazonlinux" suffix = {"ro": "squashfs", "rw": "ext4"}[rootfs_mode] disk_list = disks(f"{distro}*.{suffix}") if not disk_list: @@ -632,10 +685,14 @@ def rootfs(guest_kernel, rootfs_mode): @pytest.fixture -def uvm(microvm_factory, guest_kernel, rootfs, pci_enabled): +def uvm(microvm_factory, vm_backend, guest_kernel, rootfs, pci_enabled): """Built microVM (chroot only). Caller drives spawn/basic_config/start.""" - vm = microvm_factory.build(guest_kernel, rootfs, pci=pci_enabled) - return vm + return microvm_factory.build( + guest_kernel, + rootfs, + pci=pci_enabled, + backend=vm_backend, + ) @pytest.fixture @@ -646,9 +703,8 @@ def uvm_configured(uvm, vcpu_count, mem_size_mib, huge_pages, cpu_template): vcpu_count=vcpu_count, mem_size_mib=mem_size_mib, huge_pages=huge_pages, + cpu_template=cpu_template, ) - if cpu_template is not None: - uvm.set_cpu_template(cpu_template) return uvm @@ -670,7 +726,7 @@ def uvm_restored(uvm_booted, microvm_factory): return restored -@pytest.fixture(params=["booted", "restored"]) +@pytest.fixture(params=tuple(VmLifecycle), ids=lambda lifecycle: lifecycle.value) def uvm_lifecycle(request): """Parametrized over the two lifecycle end-states a test may want. @@ -686,6 +742,7 @@ def uvm_lifecycle(request): def uvm_any( uvm_lifecycle, request, + vm_backend, guest_kernel, rootfs, pci_enabled, @@ -706,4 +763,4 @@ def uvm_any( those names would error with "function uses no fixture"). """ # pylint: disable=unused-argument - return request.getfixturevalue(f"uvm_{uvm_lifecycle}") + return request.getfixturevalue(f"uvm_{uvm_lifecycle.value}") diff --git a/tests/framework/artifacts.py b/tests/framework/artifacts.py index 44319022fee..8221688231f 100644 --- a/tests/framework/artifacts.py +++ b/tests/framework/artifacts.py @@ -3,7 +3,9 @@ """Define classes for interacting with CI artifacts""" +import platform import re +from dataclasses import dataclass from pathlib import Path from typing import Iterator @@ -12,32 +14,130 @@ from framework.defs import ARTIFACT_DIR -def select_supported_kernels(): - """Select guest kernels supported by the current combination of kernel and - instance type. +@dataclass(frozen=True) +class GuestKernel: + """Logical guest kernel variant and its concrete boot artifacts.""" + + version: str + acpi: bool + vmlinux: Path + efi_image: Path | None = None + debug: bool = False + + @classmethod + def from_vmlinux(cls, vmlinux: Path): + """Build a logical guest kernel from a Firecracker `vmlinux-*` artifact.""" + vmlinux = Path(vmlinux) + parsed = parse_vmlinux_name(vmlinux.name) + if parsed is None: + raise ValueError(f"Unsupported guest kernel artifact: {vmlinux}") + version, acpi = parsed + + debug = vmlinux.parent.name == "debug" + artifact_dir = vmlinux.parent.parent if debug else vmlinux.parent + + arch = platform.machine() + if arch == "aarch64": + # aarch64 uses Image format, which can boot both directly and via EFI + efi_image = vmlinux + elif arch == "x86_64": + # x86_64 uses a separate EFI-enabled bzImage. The no-ACPI variant is + # a legacy MPTable boot path and is never built as an EFI image, so + # it has no bzImage sibling to pair with. + path = artifact_dir / f"bzImage-{version}" if acpi else None + efi_image = path if path is not None and path.exists() else None + else: + raise ValueError(f"Unsupported host architecture: {arch}") + + return cls( + version=version, + acpi=acpi, + vmlinux=vmlinux, + efi_image=efi_image, + debug=debug, + ) + + @property + def pytest_id(self): + """Stable pytest id matching the canonical Firecracker kernel artifact.""" + return self.vmlinux.name + + @property + def metric_id(self): + """Kernel dimension value independent of the backend boot image. + + Deliberately ``linux-.`` (the patch version is dropped) so + the CloudWatch ``guest_kernel`` dimension stays stable across guest + kernel patch bumps. This also preserves the historical behavior of + reporting ACPI and no-ACPI variants under the same dimension. + + >>> GuestKernel("5.10.233", False, Path("vmlinux-5.10.233-no-acpi")).metric_id + 'linux-5.10' + """ + major_minor = ".".join(self.version.split(".")[:2]) + return f"linux-{major_minor}" + + +# Guest kernel `.` versions the suite supports. +SUPPORTED_KERNEL_VERSIONS = {"5.10", "6.1", "6.18"} +# Versions for which we also test the non-ACPI (MPTable) variant. +# Booting with MPTable is deprecated, so we only build a 5.10 no-ACPI kernel to +# keep covering it. TODO: remove this once we drop support for MPTable. +NO_ACPI_KERNEL_VERSIONS = {"5.10"} + + +# vmlinux-.[.][-no-acpi]. The single place the artifact +# name format is parsed. +_VMLINUX_NAME_RE = re.compile(r"vmlinux-(\d+\.\d+(?:\.\d+)?)(-no-acpi)?") + + +def parse_vmlinux_name(name: str) -> tuple[str, bool] | None: + """Parse a ``vmlinux-*`` artifact filename. + + Returns ``(version, acpi)`` where ``version`` is the dotted kernel version + (e.g. ``"6.1.168"``) and ``acpi`` is ``False`` for a ``-no-acpi`` artifact, + or ``None`` if `name` is not a recognised ``vmlinux-`` artifact. + + >>> parse_vmlinux_name("vmlinux-6.1.168") + ('6.1.168', True) + >>> parse_vmlinux_name("vmlinux-5.10.233-no-acpi") + ('5.10.233', False) + >>> parse_vmlinux_name("bzImage-6.1.168") is None + True """ - supported_kernels = [r"vmlinux-5.10.\d+", r"vmlinux-6.1.\d+", r"vmlinux-6.18.\d+"] - - # Booting with MPTable is deprecated but we still want to test - # for it. Until we drop support for it we will be building a 5.10 guest - # kernel without ACPI support, so that we are able to test this use-case - # as well. - # TODO: remove this once we drop support for MPTable - supported_kernels.append(r"vmlinux-5.10.\d+-no-acpi") - - return supported_kernels + match = _VMLINUX_NAME_RE.fullmatch(name) + if match is None: + return None + return match.group(1), match.group(2) != "-no-acpi" def kernels(glob, artifact_dir: Path = ARTIFACT_DIR) -> Iterator: - """Return supported kernels as kernels supported by the current combination of kernel and - instance type. + """Yield artifact paths for guest kernels the suite supports. + + Supported means: a parseable ``vmlinux-`` artifact, with a patch + version, whose ``.`` is in `SUPPORTED_KERNEL_VERSIONS` (and, + for ``-no-acpi`` artifacts, in `NO_ACPI_KERNEL_VERSIONS`). """ - supported_kernels = select_supported_kernels() for kernel in sorted(artifact_dir.glob(glob)): - for kernel_regex in supported_kernels: - if re.fullmatch(kernel_regex, kernel.name): - yield kernel - break + parsed = parse_vmlinux_name(kernel.name) + if parsed is None: + if kernel.suffix in {".config", ".debug"}: + continue + raise ValueError(f"Unsupported guest kernel artifact: {kernel}") + version, acpi = parsed + # Require a patch version (e.g. 6.1.168, not 6.1). + if len(version.split(".")) != 3: + raise ValueError( + f"Guest kernel artifact must include a patch version: {kernel}" + ) + major_minor = ".".join(version.split(".")[:2]) + if major_minor not in SUPPORTED_KERNEL_VERSIONS: + raise ValueError(f"Unsupported guest kernel version {version}: {kernel}") + if not acpi and major_minor not in NO_ACPI_KERNEL_VERSIONS: + raise ValueError( + f"Unsupported non-ACPI guest kernel version {version}: {kernel}" + ) + yield kernel def disks(glob) -> list: @@ -48,7 +148,10 @@ def disks(glob) -> list: def kernel_params(glob="vmlinux-*", select=kernels, artifact_dir=ARTIFACT_DIR) -> list: """Return supported kernels or a single None if no kernels are found""" return [ - pytest.param(kernel, id=kernel.name) for kernel in select(glob, artifact_dir) + pytest.param(kernel, id=kernel.pytest_id) + for kernel in ( + GuestKernel.from_vmlinux(path) for path in select(glob, artifact_dir) + ) ] or [pytest.param(None, id="no-kernel-found")] @@ -56,14 +159,10 @@ def kernel_params(glob="vmlinux-*", select=kernels, artifact_dir=ARTIFACT_DIR) - # ids carry the kernel filename (e.g. "vmlinux-6.1.123") rather than "kernel0". ALL_GUEST_KERNELS = list(kernel_params("vmlinux-*")) ACPI_GUEST_KERNELS = [p for p in kernel_params("vmlinux-*") if "no-acpi" not in p.id] -GUEST_KERNELS_5_10 = list(kernel_params("vmlinux-5.10.*")) -GUEST_KERNELS_6_1 = list(kernel_params("vmlinux-6.1.*")) +GUEST_KERNELS_5_10 = list(kernel_params("vmlinux-5.10*")) +GUEST_KERNELS_6_1 = list(kernel_params("vmlinux-6.1*")) GUEST_KERNELS_6_1_DEBUG = list( - kernel_params("vmlinux-6.1.*", artifact_dir=ARTIFACT_DIR / "debug") -) -GUEST_KERNELS_6_18 = list(kernel_params("vmlinux-6.18.*")) -GUEST_KERNELS_6_18_DEBUG = list( - kernel_params("vmlinux-6.18.*", artifact_dir=ARTIFACT_DIR / "debug") + kernel_params("vmlinux-6.1*", artifact_dir=ARTIFACT_DIR / "debug") ) # The single canonical kernel used when a test pins to one specific kernel # (e.g. tests of Firecracker functionality that don't depend on guest kernel). @@ -75,7 +174,7 @@ def kernel_params(glob="vmlinux-*", select=kernels, artifact_dir=ARTIFACT_DIR) - ) -def pin_guest_kernel(kernels_or_path): +def pin_guest_kernel(kernels_or_param): """Convenience marker for pinning the `guest_kernel` dim. The default `guest_kernel` fixture parametrizes over ALL_GUEST_KERNELS; @@ -88,15 +187,15 @@ def pin_guest_kernel(kernels_or_path): @pin_guest_kernel(GUEST_KERNEL_DEFAULT) def test_foo(uvm): ... - Accepts a kernel catalogue (e.g. ACPI_GUEST_KERNELS), a single - `pytest.param`, or a single Path. + Accepts a kernel catalogue (e.g. ACPI_GUEST_KERNELS) or a single + `pytest.param`. """ - # Wrap a single Path or pytest.param into a list. A bare ParameterSet - # passed to `parametrize` would be treated as a sequence of args and - # produce broken parameterizations. - if not isinstance(kernels_or_path, list): - kernels_or_path = [kernels_or_path] - return pytest.mark.parametrize("guest_kernel", kernels_or_path, indirect=True) + # Wrap a single pytest.param into a list. A bare ParameterSet passed to + # `parametrize` would be treated as a sequence of args and produce broken + # parameterizations. + if not isinstance(kernels_or_param, list): + kernels_or_param = [kernels_or_param] + return pytest.mark.parametrize("guest_kernel", kernels_or_param, indirect=True) def pin_rootfs_mode(mode): diff --git a/tests/framework/kvm.py b/tests/framework/kvm.py new file mode 100644 index 00000000000..222e53f447a --- /dev/null +++ b/tests/framework/kvm.py @@ -0,0 +1,120 @@ +# Copyright 2026 Amazon.com, Inc. or its affiliates. All Rights Reserved. +# SPDX-License-Identifier: Apache-2.0 +"""KVM support for the microVM backend dimension. + +``Microvm.basic_config`` delegates to this module via ``VmBackend.KVM`` +(:mod:`framework.vm_backend`). Starting and spawning the Firecracker process +are backend-agnostic and live on ``Microvm`` itself. +""" + +import os +import stat +from pathlib import Path + +from framework.utils_hugepages import HugePagesConfig + +KVM_PATH = Path("/dev/kvm") + + +def has_kvm(): + """Whether this host can run KVM microVMs (i.e. /dev/kvm exists).""" + return KVM_PATH.exists() + + +def kvm_probe_details(): + """Return host KVM probe details for pytest diagnostics.""" + details = { + "device": str(KVM_PATH), + "exists": KVM_PATH.exists(), + "readable": os.access(KVM_PATH, os.R_OK), + "writable": os.access(KVM_PATH, os.W_OK), + } + try: + details["mode"] = oct(stat.S_IMODE(KVM_PATH.stat().st_mode)) + except OSError as exc: + details["stat_error"] = f"{type(exc).__name__}: {exc}" + return details + + +def kvm_basic_config( + vm, + vcpu_count: int = 2, + smt: bool = None, + mem_size_mib: int = 256, + add_root_device: bool = True, + boot_args: str = None, + use_initrd: bool = False, + track_dirty_pages: bool = False, + huge_pages: HugePagesConfig = HugePagesConfig.NONE, + rootfs_io_engine=None, + cpu_template=None, + enable_entropy_device=False, +): + """Shortcut for quickly configuring a microVM. + + It handles: + - CPU and memory. + - Kernel image (will load the one in the microVM allocated path). + - Root File System (will use the one in the microVM allocated path). + - Does not start the microvm. + + The function checks the response status code and asserts that + the response is within the interval [200, 300). + + If boot_args is None, the default boot_args used in tests is + reboot=k panic=1 nomodule swiotlb=noforce console=ttyS0 [pci=off] + which differs from Firecracker's default only in the enabling of the serial console. + Reference: file:../../src/vmm/src/vmm_config/boot_source.rs::DEFAULT_KERNEL_CMDLINE + """ + vm.api.machine_config.put( + vcpu_count=vcpu_count, + smt=smt, + mem_size_mib=mem_size_mib, + track_dirty_pages=track_dirty_pages, + huge_pages=huge_pages, + ) + vm.huge_pages = huge_pages + vm.vcpus_count = vcpu_count + vm.mem_size_bytes = mem_size_mib * 2**20 + + if vm.custom_cpu_template is not None: + vm.set_cpu_template(vm.custom_cpu_template) + + if cpu_template is not None: + vm.set_cpu_template(cpu_template) + + if vm.memory_monitor: + vm.memory_monitor.start() + + if boot_args is not None: + vm.boot_args = boot_args + else: + vm.boot_args = ( + "reboot=k panic=1 nomodule swiotlb=noforce console=ttyS0 cryptomgr.notests" + ) + if not vm.pci_enabled: + vm.boot_args += " pci=off" + boot_source_args = { + "kernel_image_path": vm.create_jailed_resource(vm.kernel_file), + "boot_args": vm.boot_args, + } + + if use_initrd and vm.initrd_file is not None: + boot_source_args.update(initrd_path=vm.create_jailed_resource(vm.initrd_file)) + + vm.api.boot.put(**boot_source_args) + + if add_root_device and vm.rootfs_file is not None: + read_only = vm.rootfs_file.suffix == ".squashfs" + + # Add the root file system + vm.add_drive( + drive_id="rootfs", + path_on_host=vm.rootfs_file, + is_root_device=True, + is_read_only=read_only, + io_engine=rootfs_io_engine, + ) + + if enable_entropy_device: + vm.enable_entropy_device() diff --git a/tests/framework/microvm.py b/tests/framework/microvm.py index 5ed4739a378..79596c778dd 100644 --- a/tests/framework/microvm.py +++ b/tests/framework/microvm.py @@ -32,6 +32,7 @@ import host_tools.cargo_build as build_tools import host_tools.network as net_tools from framework import utils +from framework.artifacts import GuestKernel from framework.defs import DEFAULT_BINARY_DIR, MAX_API_CALL_DURATION_MS from framework.guest import GuestDistro from framework.http_api import Api @@ -40,7 +41,9 @@ from framework.properties import global_props from framework.utils_cpu_templates import get_cpu_template_name from framework.utils_drive import VhostUserBlkBackend, VhostUserBlkBackendType +from framework.utils_hugepages import HugePagesConfig from framework.utils_uffd import spawn_pf_handler, uffd_handler +from framework.vm_backend import VmBackend from host_tools.fcmetrics import FCMetricsMonitor from host_tools.memory import MemoryMonitor @@ -183,14 +186,6 @@ def delete(self): self.vmstate.unlink() -class HugePagesConfig(str, Enum): - """Enum describing the huge pages configurations supported Firecracker""" - - NONE = "None" - TRANSPARENT = "Transparent" - HUGETLBFS_2MB = "2M" - - # pylint: disable=R0904 class Microvm: """Class to represent a Firecracker microvm. @@ -208,6 +203,7 @@ def __init__( fc_binary_path: Path, jailer_binary_path: Path, netns: net_tools.NetNs, + backend: VmBackend, monitor_memory: bool = True, jailer_kwargs: Optional[dict] = None, numa_node=None, @@ -218,9 +214,10 @@ def __init__( # pylint: disable=too-many-statements # Unique identifier for this machine. assert microvm_id is not None + assert backend is not None self._microvm_id = microvm_id - self.kernel_file = None + self.guest_kernel = None self.rootfs_file = None self.distro = None self.ssh_key = None @@ -271,6 +268,9 @@ def __init__( self._spawned = False self._killed = False + # The spawn/basic_config/start verbs delegate to the configured backend. + self.backend = backend + # device dictionaries self.iface = {} self.disks = {} @@ -520,13 +520,23 @@ def dimensions(self): "instance": global_props.instance, "cpu_model": global_props.cpu_model, "host_kernel": f"linux-{global_props.host_linux_version}", - "guest_kernel": self.kernel_file.stem[2:], + "guest_kernel": self.guest_kernel.metric_id, "rootfs": self.rootfs_file.name, "vcpus": str(self.vcpus_count), "guest_memory": f"{self.mem_size_bytes / (1024 * 1024)}MB", "pci": f"{self.pci_enabled}", } + @property + def kernel_file(self): + """Concrete boot image this VM boots. + + Derived from the logical `guest_kernel` by the configured backend. + """ + if self.guest_kernel is None: + return None + return Path(self.backend.kernel_image_for(self.guest_kernel)) + @property def guest_kernel_version(self): """Get the guest kernel version from the filename @@ -809,88 +819,9 @@ def serial_input(self, input_string): input_cmd = f'screen -S {self.screen_session} -p 0 -X stuff "{input_string}"' return utils.check_output(input_cmd) - def basic_config( - self, - vcpu_count: int = 2, - smt: bool = None, - mem_size_mib: int = 256, - add_root_device: bool = True, - boot_args: str = None, - use_initrd: bool = False, - track_dirty_pages: bool = False, - huge_pages: HugePagesConfig = HugePagesConfig.NONE, - rootfs_io_engine=None, - cpu_template: Optional[str] = None, - enable_entropy_device=False, - ): - """Shortcut for quickly configuring a microVM. - - It handles: - - CPU and memory. - - Kernel image (will load the one in the microVM allocated path). - - Root File System (will use the one in the microVM allocated path). - - Does not start the microvm. - - The function checks the response status code and asserts that - the response is within the interval [200, 300). - - If boot_args is None, the default boot_args used in tests is - reboot=k panic=1 nomodule swiotlb=noforce console=ttyS0 [pci=off] - which differs from Firecracker's default only in the enabling of the serial console. - Reference: file:../../src/vmm/src/vmm_config/boot_source.rs::DEFAULT_KERNEL_CMDLINE - """ - self.api.machine_config.put( - vcpu_count=vcpu_count, - smt=smt, - mem_size_mib=mem_size_mib, - track_dirty_pages=track_dirty_pages, - huge_pages=huge_pages, - ) - self.huge_pages = huge_pages - self.vcpus_count = vcpu_count - self.mem_size_bytes = mem_size_mib * 2**20 - - if self.custom_cpu_template is not None: - self.set_cpu_template(self.custom_cpu_template) - - if cpu_template is not None: - self.set_cpu_template(cpu_template) - - if self.memory_monitor: - self.memory_monitor.start() - - if boot_args is not None: - self.boot_args = boot_args - else: - self.boot_args = "reboot=k panic=1 nomodule swiotlb=noforce console=ttyS0 cryptomgr.notests" - if not self.pci_enabled: - self.boot_args += " pci=off" - boot_source_args = { - "kernel_image_path": self.create_jailed_resource(self.kernel_file), - "boot_args": self.boot_args, - } - - if use_initrd and self.initrd_file is not None: - boot_source_args.update( - initrd_path=self.create_jailed_resource(self.initrd_file) - ) - - self.api.boot.put(**boot_source_args) - - if add_root_device and self.rootfs_file is not None: - read_only = self.rootfs_file.suffix == ".squashfs" - - # Add the root file system - self.add_drive( - drive_id="rootfs", - path_on_host=self.rootfs_file, - is_root_device=True, - is_read_only=read_only, - io_engine=rootfs_io_engine, - ) - - if enable_entropy_device: - self.enable_entropy_device() + def basic_config(self, *args, **kwargs): + """Configure the microVM, delegating to the configured backend.""" + return self.backend.basic_config(self, *args, **kwargs) def set_cpu_template(self, cpu_template): """Set guest CPU template.""" @@ -1016,14 +947,11 @@ def add_pmem( ) self.disks[pmem_id] = path_on_host - def start(self): - """Start the microvm. - - This function validates that the microvm boot succeeds. - """ - # Check that the VM has not started yet + def start(self, *args, **kwargs): + """Prepare and start the microVM.""" assert self.state == "Not started" + self.backend.prepare_start(self, *args, **kwargs) self.api.actions.put(action_type="InstanceStart") # Check that the VM has started @@ -1135,9 +1063,18 @@ def restore_from_snapshot( } for key, value in jailed_snapshot.meta.items(): + if key == "kernel_file": + # Meta records the concrete boot image; recover the logical + # kernel it came from, which is what the VM tracks. + try: + self.guest_kernel = GuestKernel.from_vmlinux(Path(value)) + except ValueError as exc: + raise ValueError( + f"snapshot {snapshot.vmstate.parent} records guest kernel " + f"{value!r}, which is not a recognised artifact" + ) from exc + continue setattr(self, key, value) - # Adjust things just in case - self.kernel_file = Path(self.kernel_file) if self.rootfs_file: self.rootfs_file = Path(self.rootfs_file) self.distro = GuestDistro.from_rootfs(self.rootfs_file) @@ -1284,9 +1221,11 @@ def hotplug_memory( class MicroVMFactory: """MicroVM factory""" - def __init__(self, binary_path: Path, **kwargs): + def __init__(self, binary_path: Path, backend: VmBackend, **kwargs): + assert backend is not None self.vms = [] self.binary_path = binary_path + self.backend = backend self.netns_factory = kwargs.pop("netns_factory", net_tools.NetNs) self.kwargs = kwargs @@ -1303,10 +1242,11 @@ def jailer_binary_path(self): """The path to the jailer binary using which this factory will build VMs""" return self.binary_path / "jailer" - def build(self, kernel=None, rootfs=None, **kwargs): + def build(self, kernel: GuestKernel = None, rootfs=None, **kwargs): """Build a microvm""" kwargs = self.kwargs | kwargs microvm_id = kwargs.pop("microvm_id", str(uuid.uuid4())) + backend = kwargs.pop("backend", self.backend) vm = Microvm( microvm_id=microvm_id, fc_binary_path=kwargs.pop("fc_binary_path", self.fc_binary_path), @@ -1314,12 +1254,16 @@ def build(self, kernel=None, rootfs=None, **kwargs): "jailer_binary_path", self.jailer_binary_path ), netns=kwargs.pop("netns", self.netns_factory(microvm_id)), + backend=backend, **kwargs, ) vm.netns.setup() self.vms.append(vm) if kernel is not None: - vm.kernel_file = kernel + vm.guest_kernel = kernel + # Resolve eagerly so an impossible kernel/backend combination + # surfaces at build time (normally deselected at collection). + backend.kernel_image_for(kernel) if rootfs is not None: ssh_key = rootfs.with_suffix(".id_rsa") # copy only iff not a read-only rootfs @@ -1346,12 +1290,16 @@ def build_from_snapshot( ) return vm - def build_booted(self, kernel, rootfs, *, pci=False, **basic_config_kwargs): + def build_booted( + self, kernel, rootfs, *, pci=False, backend=None, **basic_config_kwargs + ): """Build, spawn, basic_config, add a default net iface, start. + `backend` defaults to the factory backend. Extra keyword arguments are forwarded to `Microvm.basic_config`. """ - vm = self.build(kernel, rootfs, pci=pci) + build_kwargs = {"backend": backend} if backend is not None else {} + vm = self.build(kernel, rootfs, pci=pci, **build_kwargs) vm.spawn() vm.basic_config(**basic_config_kwargs) vm.add_net_iface() diff --git a/tests/framework/utils_hugepages.py b/tests/framework/utils_hugepages.py new file mode 100644 index 00000000000..44f7d56c3f0 --- /dev/null +++ b/tests/framework/utils_hugepages.py @@ -0,0 +1,14 @@ +# Copyright 2026 Amazon.com, Inc. or its affiliates. All Rights Reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Huge pages configuration for microVMs.""" + +from enum import Enum + + +class HugePagesConfig(str, Enum): + """Enum describing the huge pages configurations supported Firecracker""" + + NONE = "None" + TRANSPARENT = "Transparent" + HUGETLBFS_2MB = "2M" diff --git a/tests/framework/utils_iperf.py b/tests/framework/utils_iperf.py index 9d0d064159c..a0b28911d54 100644 --- a/tests/framework/utils_iperf.py +++ b/tests/framework/utils_iperf.py @@ -39,7 +39,7 @@ def __init__( def run_test(self, first_free_cpu): """Runs the performance test, using pinning the iperf3 servers to CPUs starting from `first_free_cpu`""" - assert self._num_clients < CpuMap.len() - self._microvm.vcpus_count - 2 + assert first_free_cpu + self._num_clients <= CpuMap.len() for server_idx in range(self._num_clients): assigned_cpu = CpuMap(first_free_cpu) diff --git a/tests/framework/vm_backend.py b/tests/framework/vm_backend.py new file mode 100644 index 00000000000..42b6d8559cf --- /dev/null +++ b/tests/framework/vm_backend.py @@ -0,0 +1,123 @@ +# Copyright 2026 Amazon.com, Inc. or its affiliates. All Rights Reserved. +# SPDX-License-Identifier: Apache-2.0 +"""Backend dimension for pytest microVM fixtures. + +A backend supplies the backend-specific parts of configuring and starting a +microVM. ``Microvm`` owns the lifecycle shared by every backend. +""" + +from enum import Enum +from functools import lru_cache + +import pytest + +from framework.artifacts import GuestKernel +from framework.kvm import has_kvm, kvm_basic_config, kvm_probe_details + + +class VmBackend(str, Enum): + """Backend values and lifecycle behavior for the ``vm_backend`` dimension.""" + + KVM = "kvm" + + def __str__(self): + """Return the stable value used in pytest IDs and reports.""" + return self.value + + def available(self): + """Whether this host can run the backend.""" + match self: + case VmBackend.KVM: + return has_kvm() + case _: + raise AssertionError(f"Unhandled VM backend: {self!r}") + + def probe_details(self): + """Host probe details for diagnostics.""" + match self: + case VmBackend.KVM: + return kvm_probe_details() + case _: + raise AssertionError(f"Unhandled VM backend: {self!r}") + + def kernel_image_for(self, guest_kernel: GuestKernel): + """Return the concrete boot image required by the backend.""" + match self: + case VmBackend.KVM: + return guest_kernel.vmlinux + case _: + raise AssertionError(f"Unhandled VM backend: {self!r}") + + def basic_config(self, vm, *args, **kwargs): + """Configure a microVM using this backend.""" + match self: + case VmBackend.KVM: + return kvm_basic_config(vm, *args, **kwargs) + case _: + raise AssertionError(f"Unhandled VM backend: {self!r}") + + def prepare_start(self, _vm, *args, **kwargs): + """Perform backend-specific preparation before starting a microVM.""" + match self: + case VmBackend.KVM: + if args or kwargs: + raise TypeError("KVM backend does not accept start options") + return + case _: + raise AssertionError(f"Unhandled VM backend: {self!r}") + + +VM_BACKEND_KVM = VmBackend.KVM +VM_BACKENDS_ALL = tuple(VmBackend) + + +@lru_cache(maxsize=1) +def available_vm_backends(): + """Return the backends this host can run, checked once per session.""" + return tuple(backend for backend in VmBackend if backend.available()) + + +def available_vm_backend_params(): + """`pytest.param` list for the backends this host can run. + + Used as the default `params` of the `vm_backend` fixture so unpinned tests + are auto-multiplied only over backends that actually exist on the host. + """ + return [pytest.param(backend) for backend in available_vm_backends()] + + +def _format_probe_value(value): + """Format backend probe values for pytest header output.""" + if isinstance(value, bool): + return "yes" if value else "no" + return str(value) + + +def _format_probe_details(details): + """Format a backend probe-details dict as a single line.""" + return ", ".join( + f"{key}={_format_probe_value(value)}" for key, value in sorted(details.items()) + ) + + +def vm_backend_probe_report(): + """Return human-readable backend probe lines for pytest logs.""" + available = available_vm_backends() + lines = [ + "VM Backends Available: " + (", ".join(available) if available else "none") + ] + for backend in VmBackend: + details = { + "available": backend in available, + **backend.probe_details(), + } + lines.append(f"VM Backend {backend}: {_format_probe_details(details)}") + return lines + + +def get_vm_backend(backend): + """Return the enum member for a backend dimension value.""" + try: + return VmBackend(backend) + except (TypeError, ValueError) as err: + raise ValueError(f"Unknown VM backend: {backend}") from err diff --git a/tests/framework/vm_lifecycle.py b/tests/framework/vm_lifecycle.py new file mode 100644 index 00000000000..04b61ff81cf --- /dev/null +++ b/tests/framework/vm_lifecycle.py @@ -0,0 +1,16 @@ +# Copyright 2026 Amazon.com, Inc. or its affiliates. All Rights Reserved. +# SPDX-License-Identifier: Apache-2.0 +"""Lifecycle dimension for pytest microVM fixtures.""" + +from enum import Enum + + +class VmLifecycle(str, Enum): + """End states supplied by the ``uvm_lifecycle`` dimension.""" + + BOOTED = "booted" + RESTORED = "restored" + + def __str__(self): + """Return the stable value used in pytest IDs and reports.""" + return self.value diff --git a/tests/host_tools/fcmetrics.py b/tests/host_tools/fcmetrics.py index aefdb85e2eb..46e8d3bbc6f 100644 --- a/tests/host_tools/fcmetrics.py +++ b/tests/host_tools/fcmetrics.py @@ -564,11 +564,13 @@ def __init__(self, vm, timer=60): self.running = False self.metrics_logger = get_metrics_logger() + # Read the dimension from the logical GuestKernel so it matches + # `Microvm.dimensions` and is independent of the boot image path. self.metrics_logger.set_dimensions( { "instance": global_props.instance, "host_kernel": "linux-" + global_props.host_linux_version, - "guest_kernel": vm.kernel_file.stem[2:], + "guest_kernel": vm.guest_kernel.metric_id, } ) self.start() diff --git a/tests/integration_tests/functional/test_cpu_features_x86_64.py b/tests/integration_tests/functional/test_cpu_features_x86_64.py index 8d35adce589..30c100bfa23 100644 --- a/tests/integration_tests/functional/test_cpu_features_x86_64.py +++ b/tests/integration_tests/functional/test_cpu_features_x86_64.py @@ -329,7 +329,7 @@ def test_cpu_rdmsr( # Load baseline host_cpu = global_props.cpu_codename host_kv = global_props.host_linux_version - guest_kv = re.search(r"vmlinux-(\d+\.\d+)", guest_kernel.name).group(1) + guest_kv = ".".join(guest_kernel.version.split(".")[:2]) baseline_file_name = ( f"msr_list_{cpu_template_name}_{host_cpu}_{host_kv}host_{guest_kv}guest.csv" ) @@ -597,7 +597,7 @@ def test_cpu_wrmsr_snapshot( # Dump MSR state to a file that will be published to S3 for the 2nd part of the test snapshot_artifacts_dir = ( Path(shared_names["snapshot_artifacts_root_dir_wrmsr"]) - / guest_kernel.name + / guest_kernel.pytest_id / get_cpu_template_name(cpu_template, with_type=True) ) clean_and_mkdir(snapshot_artifacts_dir) @@ -667,7 +667,7 @@ def test_cpu_wrmsr_restore(msr_reader_bin, microvm_factory, cpu_template, guest_ shared_names = SNAPSHOT_RESTORE_SHARED_NAMES snapshot_artifacts_dir = ( Path(shared_names["snapshot_artifacts_root_dir_wrmsr"]) - / guest_kernel.name + / guest_kernel.pytest_id / get_cpu_template_name(cpu_template, with_type=True) ) @@ -739,7 +739,7 @@ def test_cpu_cpuid_snapshot(microvm_factory, guest_kernel, rootfs, cpu_template) # Dump CPUID to a file that will be published to S3 for the 2nd part of the test snapshot_artifacts_dir = ( Path(shared_names["snapshot_artifacts_root_dir_cpuid"]) - / guest_kernel.name + / guest_kernel.pytest_id / get_cpu_template_name(cpu_template, with_type=True) ) clean_and_mkdir(snapshot_artifacts_dir) @@ -794,7 +794,7 @@ def test_cpu_cpuid_restore(microvm_factory, guest_kernel, cpu_template): shared_names = SNAPSHOT_RESTORE_SHARED_NAMES snapshot_artifacts_dir = ( Path(shared_names["snapshot_artifacts_root_dir_cpuid"]) - / guest_kernel.name + / guest_kernel.pytest_id / get_cpu_template_name(cpu_template, with_type=True) ) diff --git a/tests/integration_tests/functional/test_fuzzing.py b/tests/integration_tests/functional/test_fuzzing.py index f2da278d3cc..d102dd5f3d2 100644 --- a/tests/integration_tests/functional/test_fuzzing.py +++ b/tests/integration_tests/functional/test_fuzzing.py @@ -7,12 +7,12 @@ from framework.microvm import MicroVMFactory -def test_fuzzing_warning(guest_kernel, rootfs): +def test_fuzzing_warning(guest_kernel, rootfs, vm_backend): """Checks that a Firecracker binary built with fuzzing logs a warning at startup""" bin_dir = host_tools.cargo_build.build_fuzzing() - vmfcty = MicroVMFactory(bin_dir) + vmfcty = MicroVMFactory(bin_dir, backend=vm_backend) uvm = vmfcty.build(guest_kernel, rootfs) uvm.spawn() uvm.basic_config() diff --git a/tests/integration_tests/functional/test_gdb.py b/tests/integration_tests/functional/test_gdb.py index 66daa01b97f..7aa6a7aee2d 100644 --- a/tests/integration_tests/functional/test_gdb.py +++ b/tests/integration_tests/functional/test_gdb.py @@ -21,12 +21,12 @@ reason="GDB requires a vmlinux but we ship a uImage for ARM in our CI", ) @pin_guest_kernel(GUEST_KERNEL_DEFAULT_DEBUG) -def test_gdb_connects(guest_kernel, rootfs): +def test_gdb_connects(guest_kernel, rootfs, vm_backend): """Checks that GDB works in a FC VM""" bin_dir = host_tools.cargo_build.build_gdb() - vmfcty = MicroVMFactory(bin_dir) + vmfcty = MicroVMFactory(bin_dir, backend=vm_backend) uvm = vmfcty.build(guest_kernel, rootfs) uvm.spawn(validate_api=False) uvm.add_net_iface() @@ -56,7 +56,7 @@ def test_gdb_connects(guest_kernel, rootfs): echo 'waiting for {chroot_gdb_socket}'; sleep 1; done; - gdb {guest_kernel} -batch -x {gdb_script} + gdb {guest_kernel.vmlinux} -batch -x {gdb_script} """, shell=True, stdout=subprocess.PIPE, diff --git a/tests/integration_tests/functional/test_huge_pages.py b/tests/integration_tests/functional/test_huge_pages.py index 9ebe388dafe..bd93ea76adf 100644 --- a/tests/integration_tests/functional/test_huge_pages.py +++ b/tests/integration_tests/functional/test_huge_pages.py @@ -9,7 +9,7 @@ import pytest from framework import utils -from framework.microvm import HugePagesConfig +from framework.utils_hugepages import HugePagesConfig THP_ENABLED_PATH = Path("/sys/kernel/mm/transparent_hugepage/enabled") diff --git a/tests/integration_tests/functional/test_pmem.py b/tests/integration_tests/functional/test_pmem.py index 1458fc93c55..6bbeec6a40b 100644 --- a/tests/integration_tests/functional/test_pmem.py +++ b/tests/integration_tests/functional/test_pmem.py @@ -12,7 +12,7 @@ import host_tools.drive as drive_tools from framework import utils from framework.artifacts import ACPI_GUEST_KERNELS, pin_guest_kernel, pin_rootfs_mode -from framework.microvm import HugePagesConfig +from framework.utils_hugepages import HugePagesConfig pytestmark = pytest.mark.parametrize( "huge_pages", [HugePagesConfig.NONE, HugePagesConfig.TRANSPARENT] diff --git a/tests/integration_tests/functional/test_rng.py b/tests/integration_tests/functional/test_rng.py index 648f545f18b..1a2a74d0d7d 100644 --- a/tests/integration_tests/functional/test_rng.py +++ b/tests/integration_tests/functional/test_rng.py @@ -9,45 +9,33 @@ from host_tools.network import SSHConnection -def uvm_with_rng_booted(uvm, microvm_factory, rate_limiter): - """Return a booted microvm with virtio-rng configured""" - # pylint: disable=unused-argument +# The two fixtures below shadow the conftest lifecycle stages for the whole +# module. `uvm_configured` mirrors the conftest stage but spawns at INFO: the +# virtio-rng device logs so much at DEBUG that the overhead can distort the +# rate-limiter throughput measurements (see b755a67fb). `uvm_booted` attaches +# the entropy device before boot; no test requests it directly — the shared +# `uvm_any`/`uvm_restored` fixtures resolve it by name, so booted and restored +# variants both carry the device. +@pytest.fixture +def uvm_configured(uvm, vcpu_count, mem_size_mib, huge_pages, cpu_template): + """Spawned + configured microVM logging at INFO.""" uvm.spawn(log_level="INFO") - uvm.basic_config(vcpu_count=2, mem_size_mib=256) - uvm.add_net_iface() - uvm.api.entropy.put(rate_limiter=rate_limiter) - uvm.start() - # Just stuff it in the microvm so we can look at it later - uvm.rng_rate_limiter = rate_limiter + uvm.basic_config( + vcpu_count=vcpu_count, + mem_size_mib=mem_size_mib, + huge_pages=huge_pages, + cpu_template=cpu_template, + ) return uvm -def uvm_with_rng_restored(uvm, microvm_factory, rate_limiter): - """Return a restored uvm with virtio-rng configured""" - uvm = uvm_with_rng_booted(uvm, microvm_factory, rate_limiter) - snapshot = uvm.snapshot_full() - uvm.kill() - uvm2 = microvm_factory.build_from_snapshot(snapshot) - uvm2.rng_rate_limiter = uvm.rng_rate_limiter - return uvm2 - - -@pytest.fixture(params=[uvm_with_rng_booted, uvm_with_rng_restored]) -def uvm_ctor(request): - """Fixture to return uvms with different constructors""" - return request.param - - -@pytest.fixture(params=[None]) -def rate_limiter(request): - """Fixture to return different rate limiters""" - return request.param - - @pytest.fixture -def uvm_any(microvm_factory, uvm_ctor, uvm, rate_limiter): - """Return booted and restored uvms""" - return uvm_ctor(uvm, microvm_factory, rate_limiter) +def uvm_booted(uvm_configured): + """Booted microVM with a virtio-rng device.""" + uvm_configured.api.entropy.put() + uvm_configured.add_net_iface() + uvm_configured.start() + return uvm_configured def list_rng_available(ssh_connection: SSHConnection) -> list[str]: @@ -215,17 +203,19 @@ def _rate_limiter_id(rate_limiter): {"bandwidth": {"size": 10000, "refill_time": 100}}, {"bandwidth": {"size": 100000, "refill_time": 100}}, ], - indirect=True, ids=_rate_limiter_id, ) -@pytest.mark.parametrize("uvm_ctor", [uvm_with_rng_booted], indirect=True) -def test_rng_bw_rate_limiter(uvm_any): +def test_rng_bw_rate_limiter(uvm_configured, rate_limiter): """ Test that rate limiter without initial burst budget works """ - vm = uvm_any - size = vm.rng_rate_limiter["bandwidth"]["size"] - refill_time = vm.rng_rate_limiter["bandwidth"]["refill_time"] + vm = uvm_configured + vm.api.entropy.put(rate_limiter=rate_limiter) + vm.add_net_iface() + vm.start() + + size = rate_limiter["bandwidth"]["size"] + refill_time = rate_limiter["bandwidth"]["refill_time"] expected_kbps = size / refill_time assert_virtio_rng_is_current_hwrng_device(vm.ssh) diff --git a/tests/integration_tests/functional/test_shut_down.py b/tests/integration_tests/functional/test_shut_down.py index a5620c0c123..6b130303a49 100644 --- a/tests/integration_tests/functional/test_shut_down.py +++ b/tests/integration_tests/functional/test_shut_down.py @@ -8,7 +8,7 @@ from packaging import version from framework import utils -from framework.microvm import HugePagesConfig +from framework.utils_hugepages import HugePagesConfig @pytest.mark.parametrize( diff --git a/tests/integration_tests/functional/test_snapshot_basic.py b/tests/integration_tests/functional/test_snapshot_basic.py index 6236f0b0709..ee9aec1550a 100644 --- a/tests/integration_tests/functional/test_snapshot_basic.py +++ b/tests/integration_tests/functional/test_snapshot_basic.py @@ -20,10 +20,10 @@ import host_tools.network as net_tools from framework import utils from framework.artifacts import GUEST_KERNEL_DEFAULT, pin_guest_kernel, pin_rootfs_mode -from framework.microvm import HugePagesConfig from framework.properties import global_props from framework.utils import check_filesystem, check_output from framework.utils_cpu_templates import ALL_CPU_TEMPLATES, pin_cpu_template +from framework.utils_hugepages import HugePagesConfig from framework.utils_vsock import ( ECHO_SERVER_PORT, VSOCK_UDS_PATH, diff --git a/tests/integration_tests/functional/test_sysgenid.py b/tests/integration_tests/functional/test_sysgenid.py index aa3495f81e2..296c1e2cbb4 100644 --- a/tests/integration_tests/functional/test_sysgenid.py +++ b/tests/integration_tests/functional/test_sysgenid.py @@ -8,23 +8,18 @@ SYSGENID_OUT_PATH = "/tmp/sysgenid.out" -@pytest.fixture(scope="function") -def vm_with_sysgenid(uvm, bin_sysgenid_path): - """Create a VM with SysGenID support and the `sysgenid` test binary under `/tmp/sysgenid`""" - basevm = uvm - basevm.spawn() +# Decorates the shared `uvm_booted` stage via same-name chaining: every booted +# VM in this module carries the sysgenid test binary at SYSGENID_BIN_PATH. +@pytest.fixture +def uvm_booted(uvm_booted, bin_sysgenid_path): + """Booted microVM with the sysgenid test binary installed.""" + uvm_booted.ssh.scp_put(bin_sysgenid_path, SYSGENID_BIN_PATH) + return uvm_booted - basevm.basic_config() - basevm.add_net_iface() - basevm.start() - basevm.ssh.scp_put(bin_sysgenid_path, SYSGENID_BIN_PATH) - yield basevm - - -def test_sysgenid_via_blocking_read(vm_with_sysgenid): +def test_sysgenid_via_blocking_read(uvm_booted): """Read the SysGenID value via blocking read()""" - vm = vm_with_sysgenid + vm = uvm_booted # Start blocking read()/write() loop. vm.ssh.check_output(f"{SYSGENID_BIN_PATH} -r >{SYSGENID_OUT_PATH} 2>&1 &") @@ -35,9 +30,9 @@ def test_sysgenid_via_blocking_read(vm_with_sysgenid): assert stdout.strip() == f"SysGenID: {i + 1}" -def test_sysgenid_via_poll_and_nonblocking_read(vm_with_sysgenid): +def test_sysgenid_via_poll_and_nonblocking_read(uvm_booted): """Read the SysGenID value via poll() and non-blocking read()""" - vm = vm_with_sysgenid + vm = uvm_booted # Start poll() / non-blocking read() loop. vm.ssh.check_output(f"{SYSGENID_BIN_PATH} -p >{SYSGENID_OUT_PATH} 2>&1 &") @@ -48,9 +43,9 @@ def test_sysgenid_via_poll_and_nonblocking_read(vm_with_sysgenid): assert stdout.strip() == f"SysGenID: {i + 1}" -def test_sysgenid_via_mmap(vm_with_sysgenid): +def test_sysgenid_via_mmap(uvm_booted): """Read the SysGenID value via mmap()""" - vm = vm_with_sysgenid + vm = uvm_booted vm.ssh.check_output(f"{SYSGENID_BIN_PATH} -m >{SYSGENID_OUT_PATH} 2>&1 &") diff --git a/tests/integration_tests/functional/test_vmclock.py b/tests/integration_tests/functional/test_vmclock.py index e9d9b76df14..1a15fc98388 100644 --- a/tests/integration_tests/functional/test_vmclock.py +++ b/tests/integration_tests/functional/test_vmclock.py @@ -9,18 +9,13 @@ pytestmark = pin_guest_kernel(ACPI_GUEST_KERNELS) -@pytest.fixture(scope="function") -def vm_with_vmclock(uvm, bin_vmclock_path): - """Create a VM with VMclock support and the `vmclock` test binary under `/tmp/vmclock`""" - basevm = uvm - basevm.spawn() - - basevm.basic_config() - basevm.add_net_iface() - basevm.start() - basevm.ssh.scp_put(bin_vmclock_path, "/tmp/vmclock") - - yield basevm +# Decorates the shared `uvm_booted` stage via same-name chaining: every booted +# VM in this module carries the vmclock test binary under /tmp/vmclock. +@pytest.fixture +def uvm_booted(uvm_booted, bin_vmclock_path): + """Booted microVM with the vmclock test binary installed.""" + uvm_booted.ssh.scp_put(bin_vmclock_path, "/tmp/vmclock") + return uvm_booted def parse_vmclock(vm, use_mmap=False): @@ -53,9 +48,9 @@ def parse_vmclock_from_poll(vm, expected_notifications): @pytest.mark.parametrize("use_mmap", [False, True], ids=["read()", "mmap()"]) -def test_vmclock_read_fields(vm_with_vmclock, use_mmap): +def test_vmclock_read_fields(uvm_booted, use_mmap): """Make sure that we expose the expected values in the VMclock struct""" - vm = vm_with_vmclock + vm = uvm_booted vmclock = parse_vmclock(vm, use_mmap) assert vmclock["VMCLOCK_FLAG_VM_GEN_COUNTER_PRESENT"] == "true" @@ -70,10 +65,10 @@ def test_vmclock_read_fields(vm_with_vmclock, use_mmap): @pytest.mark.parametrize("use_mmap", [False, True], ids=["read()", "mmap()"]) -def test_snapshot_update(vm_with_vmclock, microvm_factory, snapshot_type, use_mmap): +def test_snapshot_update(uvm_booted, microvm_factory, snapshot_type, use_mmap): """Test that `disruption_marker` and `vm_generation_counter` are updated upon snapshot resume""" - basevm = vm_with_vmclock + basevm = uvm_booted vmclock = parse_vmclock(basevm, use_mmap) assert vmclock["VMCLOCK_FLAG_VM_GEN_COUNTER_PRESENT"] == "true" @@ -92,9 +87,9 @@ def test_snapshot_update(vm_with_vmclock, microvm_factory, snapshot_type, use_mm assert vmclock["VMCLOCK_VM_GENERATION_COUNTER"] == f"{i+1}" -def test_vmclock_notifications(vm_with_vmclock, microvm_factory, snapshot_type): +def test_vmclock_notifications(uvm_booted, microvm_factory, snapshot_type): """Test that Firecracker will send a notification on snapshot load""" - basevm = vm_with_vmclock + basevm = uvm_booted # Launch vmclock utility in polling mode basevm.ssh.check_output("/tmp/vmclock -p > /tmp/vmclock.out 2>&1 &") diff --git a/tests/integration_tests/performance/test_balloon.py b/tests/integration_tests/performance/test_balloon.py index 2aa22da4229..fcd70bfb01d 100644 --- a/tests/integration_tests/performance/test_balloon.py +++ b/tests/integration_tests/performance/test_balloon.py @@ -9,12 +9,12 @@ import pytest from framework.artifacts import GUEST_KERNEL_DEFAULT, pin_guest_kernel -from framework.microvm import HugePagesConfig from framework.utils import ( get_stable_rss_mem, supports_hugetlbfs_discard, track_cpu_utilization, ) +from framework.utils_hugepages import HugePagesConfig # Every test in this module exercises all huge_pages variants. pytestmark = pytest.mark.parametrize( diff --git a/tests/integration_tests/performance/test_boottime.py b/tests/integration_tests/performance/test_boottime.py index c8c855f1e7a..1a93dcd27a7 100644 --- a/tests/integration_tests/performance/test_boottime.py +++ b/tests/integration_tests/performance/test_boottime.py @@ -9,7 +9,7 @@ import pytest from framework.artifacts import ACPI_GUEST_KERNELS, pin_guest_kernel, pin_rootfs_mode -from framework.microvm import HugePagesConfig +from framework.utils_hugepages import HugePagesConfig # Regex for obtaining boot time from some string. diff --git a/tests/integration_tests/performance/test_hotplug_memory.py b/tests/integration_tests/performance/test_hotplug_memory.py index 948e7623aa0..566a6619cce 100644 --- a/tests/integration_tests/performance/test_hotplug_memory.py +++ b/tests/integration_tests/performance/test_hotplug_memory.py @@ -13,9 +13,10 @@ from framework.artifacts import GUEST_KERNEL_DEFAULT, pin_guest_kernel from framework.guest_stats import MeminfoGuest -from framework.microvm import HugePagesConfig, SnapshotType +from framework.microvm import SnapshotType from framework.properties import global_props from framework.utils import get_resident_memory, supports_hugetlbfs_discard +from framework.utils_hugepages import HugePagesConfig MEMHP_BOOTARGS = "console=ttyS0 reboot=k panic=1 memhp_default_state=online_movable" DEFAULT_CONFIG = {"total_size_mib": 1024, "slot_size_mib": 128, "block_size_mib": 2} diff --git a/tests/integration_tests/performance/test_huge_pages.py b/tests/integration_tests/performance/test_huge_pages.py index 83d99531599..d18fa898509 100644 --- a/tests/integration_tests/performance/test_huge_pages.py +++ b/tests/integration_tests/performance/test_huge_pages.py @@ -9,9 +9,9 @@ from framework import utils from framework.artifacts import GUEST_KERNEL_DEFAULT, pin_guest_kernel -from framework.microvm import HugePagesConfig from framework.properties import global_props from framework.utils_ftrace import ftrace_events +from framework.utils_hugepages import HugePagesConfig pytestmark = pin_guest_kernel(GUEST_KERNEL_DEFAULT) diff --git a/tests/integration_tests/performance/test_initrd.py b/tests/integration_tests/performance/test_initrd.py index bcd934c9f1b..9888d48833d 100644 --- a/tests/integration_tests/performance/test_initrd.py +++ b/tests/integration_tests/performance/test_initrd.py @@ -4,7 +4,8 @@ import pytest -from framework.microvm import HugePagesConfig, Serial +from framework.microvm import Serial +from framework.utils_hugepages import HugePagesConfig INITRD_FILESYSTEM = "rootfs" diff --git a/tests/integration_tests/performance/test_network.py b/tests/integration_tests/performance/test_network.py index 3620e399b4b..f09efd07244 100644 --- a/tests/integration_tests/performance/test_network.py +++ b/tests/integration_tests/performance/test_network.py @@ -53,7 +53,6 @@ def network_microvm(request, uvm): vm.basic_config(vcpu_count=guest_vcpus, mem_size_mib=guest_mem_mib) vm.add_net_iface() vm.start() - vm.pin_threads(0) return vm @@ -64,6 +63,7 @@ def test_network_latency(network_microvm, metrics): """ Test network latency by sending pings from the guest to the host. """ + network_microvm.pin_threads(0) target_datapoints = 500 delay = 0.0 @@ -104,6 +104,7 @@ def test_network_tcp_throughput( """ Iperf between guest and host in both directions for TCP workload. """ + first_free_cpu = network_microvm.pin_threads(0) base_port = 5000 # Time (in seconds) for which iperf "warms up" @@ -130,7 +131,7 @@ def test_network_tcp_throughput( connect_to=network_microvm.iface["eth0"]["iface"].host_ip, payload_length=payload_length, ) - data = test.run_test(network_microvm.vcpus_count + 2) + data = test.run_test(first_free_cpu) for i, g2h in enumerate(data["g2h"]): Path(results_dir / f"g2h_{i}.json").write_text( diff --git a/tests/integration_tests/performance/test_snapshot.py b/tests/integration_tests/performance/test_snapshot.py index 1e7b74859ed..430d9ad12f4 100644 --- a/tests/integration_tests/performance/test_snapshot.py +++ b/tests/integration_tests/performance/test_snapshot.py @@ -13,7 +13,8 @@ import host_tools.drive as drive_tools from framework.artifacts import GUEST_KERNEL_DEFAULT, pin_guest_kernel -from framework.microvm import HugePagesConfig, Microvm, SnapshotType +from framework.microvm import Microvm, SnapshotType +from framework.utils_hugepages import HugePagesConfig USEC_IN_MSEC = 1000 NS_IN_MSEC = 1_000_000 diff --git a/tests/integration_tests/security/test_fips.py b/tests/integration_tests/security/test_fips.py index 63b90eb23c1..c44ebe5b96e 100644 --- a/tests/integration_tests/security/test_fips.py +++ b/tests/integration_tests/security/test_fips.py @@ -30,37 +30,38 @@ ] +# No test requests this fixture directly: the conftest `uvm_booted` stage +# resolves this module-local override, so every test below boots with the +# FIPS kernel command line. @pytest.fixture -def uvm_with_fips(uvm): - """Boot a microVM with FIPS mode enabled.""" +def uvm_configured(uvm): + """Spawned microVM configured to boot with FIPS mode enabled.""" uvm.spawn() uvm.basic_config(boot_args="console=ttyS0 reboot=k panic=1 pci=off fips=1") - uvm.add_net_iface() - uvm.start() return uvm @pytest.fixture -def fips_snapshot_pair(uvm_with_fips, microvm_factory): +def fips_snapshot_pair(uvm_booted, microvm_factory): """Boot a FIPS VM, snapshot it, restore two VMs from the same snapshot.""" - snapshot = uvm_with_fips.snapshot_full() - uvm_with_fips.kill() + snapshot = uvm_booted.snapshot_full() + uvm_booted.kill() uvm_a = microvm_factory.build_from_snapshot(snapshot) uvm_b = microvm_factory.build_from_snapshot(snapshot) yield uvm_a, uvm_b -def test_fips_enabled(uvm_with_fips): +def test_fips_enabled(uvm_booted): """Test that FIPS mode is enabled in the guest kernel.""" - _, dmesg, _ = uvm_with_fips.ssh.run("dmesg | grep -i fips") + _, dmesg, _ = uvm_booted.ssh.run("dmesg | grep -i fips") assert "fips mode: enabled" in dmesg.lower() -def test_fips_rng_reseed_on_snapshot_restore(uvm_with_fips, microvm_factory): +def test_fips_rng_reseed_on_snapshot_restore(uvm_booted, microvm_factory): """Test that FIPS RNG reseeding is logged on snapshot restore.""" - snapshot = uvm_with_fips.snapshot_full() - uvm_with_fips.kill() + snapshot = uvm_booted.snapshot_full() + uvm_booted.kill() restored = microvm_factory.build_from_snapshot(snapshot) _, dmesg, _ = restored.ssh.run("dmesg | grep -i fips") diff --git a/tests/integration_tests/security/test_vulnerabilities.py b/tests/integration_tests/security/test_vulnerabilities.py index aaf66cd44eb..362f5af63db 100644 --- a/tests/integration_tests/security/test_vulnerabilities.py +++ b/tests/integration_tests/security/test_vulnerabilities.py @@ -19,6 +19,7 @@ from framework.microvm import MicroVMFactory from framework.properties import global_props from framework.utils_cpu_templates import ALL_CPU_TEMPLATES, pin_cpu_template +from framework.vm_lifecycle import VmLifecycle CHECKER_URL = "https://raw.githubusercontent.com/speed47/spectre-meltdown-checker/master/spectre-meltdown-checker.sh" CHECKER_FILENAME = "spectre-meltdown-checker.sh" @@ -291,12 +292,12 @@ def check_vulnerabilities_files_on_guest(microvm): @pytest.fixture -def microvm_factory_a(record_property): +def microvm_factory_a(record_property, vm_backend): """MicroVMFactory using revision A binaries""" revision_a = global_props.buildkite_revision_a bin_dir = git_clone(Path("../build") / revision_a, revision_a).resolve() record_property("firecracker_bin", str(bin_dir / "firecracker")) - uvm_factory = MicroVMFactory(bin_dir) + uvm_factory = MicroVMFactory(bin_dir, backend=vm_backend) yield uvm_factory uvm_factory.kill() @@ -317,7 +318,7 @@ def uvm_any_a( """ builder = ( microvm_factory_a.build_booted - if uvm_lifecycle == "booted" + if uvm_lifecycle is VmLifecycle.BOOTED else microvm_factory_a.build_restored ) return builder(guest_kernel, rootfs, pci=pci_enabled, cpu_template=cpu_template) diff --git a/tools/sandbox.py b/tools/sandbox.py index 5b5ec9c5ac7..7ea68be2499 100755 --- a/tools/sandbox.py +++ b/tools/sandbox.py @@ -15,9 +15,10 @@ from pathlib import Path import host_tools.cargo_build as build_tools -from framework.artifacts import disks, kernels +from framework.artifacts import GuestKernel, disks, kernels from framework.defs import DEFAULT_BINARY_DIR, FC_WORKSPACE_DIR from framework.microvm import MicroVMFactory +from framework.vm_backend import VM_BACKEND_KVM kernels = list(kernels("vmlinux-*")) rootfs = list(disks("*.ext4")) @@ -126,14 +127,14 @@ def pick_default_rootfs(candidates): cpu_template = None if args.cpu_template_path is not None: cpu_template = json.loads(args.cpu_template_path.read_text("utf-8")) -vmfcty = MicroVMFactory(binary_dir) +vmfcty = MicroVMFactory(binary_dir, backend=VM_BACKEND_KVM) if args.debug or args.gdb: - kernel = args.kernel.parent / "debug" / args.kernel.name + kernel = GuestKernel.from_vmlinux(args.kernel.parent / "debug" / args.kernel.name) else: - kernel = args.kernel + kernel = GuestKernel.from_vmlinux(args.kernel) -print(f"uvm with kernel {kernel} ...") +print(f"uvm with kernel {kernel.vmlinux} ...") uvm = vmfcty.build(kernel, args.rootfs) uvm.help.enable_console() uvm.help.resize_disk(uvm.rootfs_file, args.rootfs_size) diff --git a/tools/test-popular-containers/test-docker-rootfs.py b/tools/test-popular-containers/test-docker-rootfs.py index bdaf1c306cf..e66f59f2539 100755 --- a/tools/test-popular-containers/test-docker-rootfs.py +++ b/tools/test-popular-containers/test-docker-rootfs.py @@ -16,17 +16,18 @@ sys.path.append(os.path.join(os.getcwd(), "tests")) # pylint: disable=wrong-import-position -from framework.artifacts import kernels +from framework.artifacts import GuestKernel, kernels from framework.defs import DEFAULT_BINARY_DIR from framework.microvm import MicroVMFactory +from framework.vm_backend import VM_BACKEND_KVM # pylint: enable=wrong-import-position kernels = list(kernels("vmlinux-*")) # Use the latest guest kernel -kernel = kernels[-1] +kernel = GuestKernel.from_vmlinux(kernels[-1]) -vmfcty = MicroVMFactory(DEFAULT_BINARY_DIR) +vmfcty = MicroVMFactory(DEFAULT_BINARY_DIR, backend=VM_BACKEND_KVM) # (may take a while to compile Firecracker...) for rootfs in Path(".").glob("*.squashfs"):