From c10d9937d983950d91027a47b0e0688fdf3c7668 Mon Sep 17 00:00:00 2001 From: Henry Schreiner Date: Tue, 9 Jun 2026 17:10:41 -0400 Subject: [PATCH 1/9] refactor: add shared build runner and Builder interface Adds cibuildwheel/platforms/runner.py: a Builder ABC (generic over the wheel path type, since Linux wheels live in the container as PurePosixPath) whose methods are the platform-specific build steps, and run_builds(), a single flat loop that owns everything identical across platforms: logging, compatible-wheel reuse, built/repaired wheel validation, the test gate, output bookkeeping, and cleanup. HostBuilder provides the step bodies shared by the five non-Linux platforms. run_before_all() and fatal_on_called_process_error() replace the per-platform copies of the before_all hook and the CalledProcessError-to-FatalError wrapper. No platform uses this yet; subsequent commits port them one at a time. Assisted-by: ClaudeCode:claude-fable-5 --- cibuildwheel/platforms/runner.py | 283 +++++++++++++++++++++++++++++ unit_test/runner_test.py | 293 +++++++++++++++++++++++++++++++ 2 files changed, 576 insertions(+) create mode 100644 cibuildwheel/platforms/runner.py create mode 100644 unit_test/runner_test.py diff --git a/cibuildwheel/platforms/runner.py b/cibuildwheel/platforms/runner.py new file mode 100644 index 000000000..8bbb29012 --- /dev/null +++ b/cibuildwheel/platforms/runner.py @@ -0,0 +1,283 @@ +""" +The shared build loop, used by every platform module. + +Each platform defines a Builder subclass whose methods implement the +platform-specific steps; run_builds() drives those steps for each build +identifier, in a fixed order: + + setup -> before_build -> build_wheel -> repair_wheel -> audit_wheel + -> test_wheel -> move_to_output -> cleanup + +run_builds() also owns everything that's identical across platforms: logging, +reuse of compatible wheels, validation of the built/repaired wheels, the test +gate, and moving wheels to the output directory. +""" + +from __future__ import annotations + +__lazy_modules__ = { + "cibuildwheel.audit", + "cibuildwheel.logger", + "cibuildwheel.util", + "cibuildwheel.util.cmd", + "cibuildwheel.util.file", + "cibuildwheel.util.helpers", + "cibuildwheel.util.packaging", + "shutil", + "subprocess", +} + +import contextlib +import os +import shutil +import subprocess +from abc import ABC, abstractmethod +from pathlib import Path, PurePath +from typing import Generic, TypeVar + +from cibuildwheel import errors +from cibuildwheel.audit import run_audit +from cibuildwheel.logger import log +from cibuildwheel.util import resources +from cibuildwheel.util.cmd import shell +from cibuildwheel.util.file import copy_test_sources, move_file +from cibuildwheel.util.helpers import prepare_command +from cibuildwheel.util.packaging import find_compatible_wheel + +TYPE_CHECKING = False +if TYPE_CHECKING: + from collections.abc import Callable, Generator, Mapping, Sequence + + from cibuildwheel.options import BuildOptions, Options + from cibuildwheel.typing import GenericPythonConfiguration + +PathT = TypeVar("PathT", bound=PurePath) + + +class Builder(ABC, Generic[PathT]): + """ + The platform-specific steps to build one wheel (one build identifier). + + Constructing a Builder must not do any work; all work happens in the step + methods, which run_builds() calls in a fixed order. PathT is the type of + the paths the wheels live at while building: Path on the host platforms, + PurePosixPath inside the Linux container. + """ + + identifier: str + build_options: BuildOptions + + @abstractmethod + def setup(self) -> None: + """Install Python and prepare the build environment, storing any state + the later steps need (e.g. self.env) on the instance. Logs its own + log.step()s and may leave the last one open.""" + + @abstractmethod + def before_build(self) -> None: + """Run the user's before-build command. Only called if one is set.""" + + @abstractmethod + def build_wheel(self) -> PathT: + """Build the wheel with the configured frontend and return its path.""" + + @abstractmethod + def repair_wheel(self, built_wheel: PathT) -> list[PathT]: + """Run the user's repair command (or, if none is set, move the wheel + as-is) and return the contents of the repaired-wheel directory. The + runner validates that exactly one wheel was produced.""" + + @abstractmethod + def audit_wheel(self, repaired_wheel: PathT) -> None: + """Run the audit command, if one is configured.""" + + @abstractmethod + def test_wheel(self, repaired_wheel: PathT) -> None: + """Test the wheel: set up a test environment, run before-test, install + the wheel and its test requirements, and run the test command. Only + called if a test command is set and the test selector matches. Skip + conditions that can only be determined at build time (e.g. an arch + that can't be tested on this machine) live here too.""" + + @abstractmethod + def move_to_output(self, repaired_wheel: PathT) -> PathT: + """Move the wheel to the output directory, returning the path that + later builds can reuse it from (and install it from, in tests).""" + + @abstractmethod + def cleanup(self) -> None: + """Remove this identifier's temporary files.""" + + +class HostBuilder(Builder[Path]): + """A Builder whose wheels live on the host filesystem; implements the + steps that are common to every platform except Linux (which builds inside + a container).""" + + env: dict[str, str] # set by setup() in each subclass + + def __init__( + self, + *, + identifier: str, + build_options: BuildOptions, + tmp_dir: Path, + session_tmp_dir: Path, + ) -> None: + self.identifier = identifier + self.build_options = build_options + # per-identifier scratch space, removed by cleanup() + self.tmp_dir = tmp_dir + # shared across identifiers; run_audit() reuses its venv between calls + self.session_tmp_dir = session_tmp_dir + self.built_wheel_dir = tmp_dir / "built_wheel" + self.repaired_wheel_dir = tmp_dir / "repaired_wheel" + + def before_build(self) -> None: + assert self.build_options.before_build is not None + before_build_prepared = prepare_command( + self.build_options.before_build, + project=".", + package=self.build_options.package_dir, + ) + shell(before_build_prepared, env=self.env) + + def repair_wheel(self, built_wheel: Path) -> list[Path]: + self.repaired_wheel_dir.mkdir(exist_ok=True) + if self.build_options.repair_command: + repair_command_prepared = prepare_command( + self.build_options.repair_command, + wheel=built_wheel, + dest_dir=self.repaired_wheel_dir, + package=self.build_options.package_dir, + project=".", + ) + shell(repair_command_prepared, env=self.env) + else: + shutil.move(str(built_wheel), self.repaired_wheel_dir) + return list(self.repaired_wheel_dir.glob("*.whl")) + + def audit_wheel(self, repaired_wheel: Path) -> None: + run_audit( + tmp_dir=self.session_tmp_dir, build_options=self.build_options, wheel=repaired_wheel + ) + + def move_to_output(self, repaired_wheel: Path) -> Path: + output_wheel = self.build_options.output_dir / repaired_wheel.name + moved_wheel = move_file(repaired_wheel, output_wheel) + if moved_wheel != output_wheel.resolve(): + log.warning(f"{repaired_wheel} was moved to {moved_wheel} instead of {output_wheel}") + return output_wheel + + def cleanup(self) -> None: + # ignore_errors: occasionally Windows fails to unlink a file, and we + # don't want to abort a build because of a leftover temp dir + shutil.rmtree(self.tmp_dir, ignore_errors=True) + + +def run_builds(builders: Sequence[Builder[PathT]]) -> None: + """Build a wheel for each builder, in order.""" + built_wheels: list[PathT] = [] + + for builder in builders: + build_options = builder.build_options + log.build_start(builder.identifier) + + builder.setup() + + compatible_wheel = find_compatible_wheel(built_wheels, builder.identifier) + if compatible_wheel is not None: + log.step_end() + print( + f"\nFound previously built wheel {compatible_wheel.name}, that's compatible with " + f"{builder.identifier}. Skipping build step..." + ) + repaired_wheel = compatible_wheel + else: + if build_options.before_build: + log.step("Running before_build...") + builder.before_build() + + log.step("Building wheel...") + built_wheel = builder.build_wheel() + if built_wheel.name.endswith("none-any.whl"): + raise errors.NonPlatformWheelError() + + if build_options.repair_command: + log.step("Repairing wheel...") + repaired_wheels = builder.repair_wheel(built_wheel) + match repaired_wheels: + case [only_wheel]: + repaired_wheel = only_wheel + case []: + raise errors.RepairStepProducedNoWheelError() + case many_wheels: + raise errors.RepairStepProducedMultipleWheelsError( + [wheel.name for wheel in many_wheels] + ) + + if repaired_wheel.name in {wheel.name for wheel in built_wheels}: + raise errors.AlreadyBuiltWheelError(repaired_wheel.name) + log.step_end() + + builder.audit_wheel(repaired_wheel) + + if build_options.test_command and build_options.test_selector(builder.identifier): + builder.test_wheel(repaired_wheel) + + output_wheel: Path | None = None + if compatible_wheel is None: + tracked_wheel = builder.move_to_output(repaired_wheel) + built_wheels.append(tracked_wheel) + # on Linux, the wheel only arrives at this host path when the + # container exits; the summary reads the file lazily, so that's fine + output_wheel = build_options.output_dir / tracked_wheel.name + + builder.cleanup() + log.build_end(output_wheel) + + +def run_before_all( + options: Options, + python_configurations: Sequence[GenericPythonConfiguration], + *, + env_defaults: Mapping[str, str] | None = None, +) -> None: + """Run the user's before-all command on the host, if one is set.""" + before_all_options = options.build_options(python_configurations[0].identifier) + if not before_all_options.before_all: + return + + log.step("Running before_all...") + env = before_all_options.environment.as_dictionary(prev_environment=os.environ) + for name, value in (env_defaults or {}).items(): + env.setdefault(name, value) + before_all_prepared = prepare_command( + before_all_options.before_all, project=".", package=before_all_options.package_dir + ) + shell(before_all_prepared, env=env) + + +@contextlib.contextmanager +def fatal_on_called_process_error( + troubleshoot: Callable[[subprocess.CalledProcessError], None] | None = None, +) -> Generator[None, None, None]: + """Turn a failed command anywhere in the build into a FatalError.""" + try: + yield + except subprocess.CalledProcessError as error: + if troubleshoot is not None: + troubleshoot(error) + msg = f"Command {error.cmd} failed with code {error.returncode}. {error.stdout or ''}" + raise errors.FatalError(msg) from error + + +def prepare_test_cwd(test_cwd: Path, test_sources: list[str]) -> None: + """Populate the directory the test command runs in: the user's test + sources if configured, otherwise a sentinel test that fails with a + helpful message if the user's test command assumes the project dir.""" + test_cwd.mkdir(exist_ok=True) + if test_sources: + copy_test_sources(test_sources, Path.cwd(), test_cwd) + else: + (test_cwd / "test_fail.py").write_text(resources.TEST_FAIL_CWD_FILE.read_text()) diff --git a/unit_test/runner_test.py b/unit_test/runner_test.py new file mode 100644 index 000000000..b29da8268 --- /dev/null +++ b/unit_test/runner_test.py @@ -0,0 +1,293 @@ +from __future__ import annotations + +import dataclasses +import subprocess +from pathlib import Path + +import pytest + +import cibuildwheel.options +import cibuildwheel.selector +from cibuildwheel import errors +from cibuildwheel.environment import parse_environment +from cibuildwheel.frontend import BuildFrontendConfig +from cibuildwheel.oci_container import OCIContainerEngineConfig +from cibuildwheel.options import BuildOptions, GlobalOptions +from cibuildwheel.platforms import runner +from cibuildwheel.selector import BuildSelector +from cibuildwheel.util.packaging import DependencyConstraints + + +@pytest.fixture +def build_options(tmp_path: Path) -> BuildOptions: + return BuildOptions( + globals=GlobalOptions( + package_dir=Path(), + output_dir=tmp_path / "output", + build_selector=BuildSelector(build_config="*", skip_config=""), + test_selector=cibuildwheel.selector.TestSelector(skip_config=""), + architectures=set(), + allow_empty=False, + ), + environment=parse_environment(""), + before_all="", + before_build=None, + xbuild_tools=None, + xbuild_files={}, + repair_command="", + manylinux_images=None, + musllinux_images=None, + dependency_constraints=DependencyConstraints.latest(), + test_command=None, + before_test=None, + test_sources=[], + test_requires=[], + test_extras="", + test_groups=[], + test_environment=parse_environment(""), + test_runtime=cibuildwheel.options.TestRuntimeConfig(), + audit_requires=[], + audit_command=[], + build_verbosity=0, + build_frontend=BuildFrontendConfig(name="build"), + config_settings="", + container_engine=OCIContainerEngineConfig(name="docker"), + pyodide_version=None, + ) + + +class FakeBuilder(runner.Builder[Path]): + """Records the steps the runner invokes; touches no real files.""" + + def __init__( + self, + *, + identifier: str, + build_options: BuildOptions, + calls: list[tuple[str, str]], + built_wheel_name: str = "spam-0.1.0-cp311-cp311-macosx_11_0_arm64.whl", + repaired_wheel_names: tuple[str, ...] | None = None, + ) -> None: + self.identifier = identifier + self.build_options = build_options + self.calls = calls + self.built_wheel_name = built_wheel_name + self.repaired_wheel_names = repaired_wheel_names + + def record(self, step: str) -> None: + self.calls.append((self.identifier, step)) + + def setup(self) -> None: + self.record("setup") + + def before_build(self) -> None: + self.record("before_build") + + def build_wheel(self) -> Path: + self.record("build_wheel") + return Path("/fake/built") / self.built_wheel_name + + def repair_wheel(self, built_wheel: Path) -> list[Path]: + self.record("repair_wheel") + names = ( + self.repaired_wheel_names + if self.repaired_wheel_names is not None + else (built_wheel.name,) + ) + return [Path("/fake/repaired") / name for name in names] + + def audit_wheel(self, repaired_wheel: Path) -> None: # noqa: ARG002 + self.record("audit_wheel") + + def test_wheel(self, repaired_wheel: Path) -> None: + self.record(f"test_wheel:{repaired_wheel.name}") + + def move_to_output(self, repaired_wheel: Path) -> Path: + self.record("move_to_output") + return self.build_options.output_dir / repaired_wheel.name + + def cleanup(self) -> None: + self.record("cleanup") + + +def test_step_order(build_options: BuildOptions) -> None: + build_options = dataclasses.replace( + build_options, before_build="make deps", test_command="pytest" + ) + calls: list[tuple[str, str]] = [] + wheel_name = "spam-0.1.0-cp311-cp311-macosx_11_0_arm64.whl" + builder = FakeBuilder( + identifier="cp311-macosx_arm64", + build_options=build_options, + calls=calls, + built_wheel_name=wheel_name, + ) + + runner.run_builds([builder]) + + assert calls == [ + ("cp311-macosx_arm64", "setup"), + ("cp311-macosx_arm64", "before_build"), + ("cp311-macosx_arm64", "build_wheel"), + ("cp311-macosx_arm64", "repair_wheel"), + ("cp311-macosx_arm64", "audit_wheel"), + ("cp311-macosx_arm64", f"test_wheel:{wheel_name}"), + ("cp311-macosx_arm64", "move_to_output"), + ("cp311-macosx_arm64", "cleanup"), + ] + + +def test_no_before_build_no_test(build_options: BuildOptions) -> None: + calls: list[tuple[str, str]] = [] + builder = FakeBuilder(identifier="cp311-macosx_arm64", build_options=build_options, calls=calls) + + runner.run_builds([builder]) + + steps = [step for _, step in calls] + assert "before_build" not in steps + assert not any(step.startswith("test_wheel") for step in steps) + assert steps == [ + "setup", + "build_wheel", + "repair_wheel", + "audit_wheel", + "move_to_output", + "cleanup", + ] + + +def test_compatible_wheel_reuse(build_options: BuildOptions) -> None: + build_options = dataclasses.replace(build_options, test_command="pytest") + calls: list[tuple[str, str]] = [] + abi3_wheel_name = "spam-0.1.0-cp311-abi3-macosx_11_0_arm64.whl" + builders = [ + FakeBuilder( + identifier="cp311-macosx_arm64", + build_options=build_options, + calls=calls, + built_wheel_name=abi3_wheel_name, + ), + FakeBuilder( + identifier="cp312-macosx_arm64", + build_options=build_options, + calls=calls, + ), + ] + + runner.run_builds(builders) + + second = [step for identifier, step in calls if identifier == "cp312-macosx_arm64"] + # build/repair/audit/move are skipped, but the reused wheel is still tested + assert second == ["setup", f"test_wheel:{abi3_wheel_name}", "cleanup"] + + +def test_repair_produced_no_wheel(build_options: BuildOptions) -> None: + builder = FakeBuilder( + identifier="cp311-macosx_arm64", + build_options=build_options, + calls=[], + repaired_wheel_names=(), + ) + + with pytest.raises(errors.RepairStepProducedNoWheelError): + runner.run_builds([builder]) + + +def test_repair_produced_multiple_wheels(build_options: BuildOptions) -> None: + builder = FakeBuilder( + identifier="cp311-macosx_arm64", + build_options=build_options, + calls=[], + repaired_wheel_names=( + "spam-0.1.0-cp311-cp311-macosx_11_0_arm64.whl", + "ham-0.1.0-cp311-cp311-macosx_11_0_arm64.whl", + ), + ) + + with pytest.raises(errors.RepairStepProducedMultipleWheelsError): + runner.run_builds([builder]) + + +def test_already_built_wheel(build_options: BuildOptions) -> None: + # two identifiers producing the same (non-reusable) wheel name + calls: list[tuple[str, str]] = [] + wheel_name = "spam-0.1.0-cp311-cp311-macosx_11_0_arm64.whl" + builders = [ + FakeBuilder( + identifier="cp311-macosx_arm64", + build_options=build_options, + calls=calls, + built_wheel_name=wheel_name, + ), + FakeBuilder( + identifier="cp311-macosx_x86_64", + build_options=build_options, + calls=calls, + built_wheel_name=wheel_name, + ), + ] + + with pytest.raises(errors.AlreadyBuiltWheelError): + runner.run_builds(builders) + + +def test_none_any_wheel_rejected(build_options: BuildOptions) -> None: + builder = FakeBuilder( + identifier="cp311-macosx_arm64", + build_options=build_options, + calls=[], + built_wheel_name="spam-0.1.0-py3-none-any.whl", + ) + + with pytest.raises(errors.NonPlatformWheelError): + runner.run_builds([builder]) + + +def test_test_skipped_by_selector(build_options: BuildOptions) -> None: + build_options = dataclasses.replace( + build_options, + globals=dataclasses.replace( + build_options.globals, test_selector=cibuildwheel.selector.TestSelector(skip_config="*") + ), + test_command="pytest", + ) + calls: list[tuple[str, str]] = [] + builder = FakeBuilder(identifier="cp311-macosx_arm64", build_options=build_options, calls=calls) + + runner.run_builds([builder]) + + assert not any(step.startswith("test_wheel") for _, step in calls) + + +def test_fatal_on_called_process_error() -> None: + with ( + pytest.raises(errors.FatalError, match="failed with code 42"), + runner.fatal_on_called_process_error(), + ): + raise subprocess.CalledProcessError(42, ["false"]) + + +def test_fatal_on_called_process_error_troubleshoot() -> None: + seen: list[subprocess.CalledProcessError] = [] + with pytest.raises(errors.FatalError), runner.fatal_on_called_process_error(seen.append): + raise subprocess.CalledProcessError(1, ["false"]) + assert len(seen) == 1 + + +def test_prepare_test_cwd_sentinel(tmp_path: Path) -> None: + test_cwd = tmp_path / "test_cwd" + runner.prepare_test_cwd(test_cwd, []) + assert "Please specify a path to your tests" in (test_cwd / "test_fail.py").read_text() + + +def test_prepare_test_cwd_sources(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + project = tmp_path / "project" + (project / "tests").mkdir(parents=True) + (project / "tests" / "test_spam.py").write_text("def test_spam(): pass\n") + monkeypatch.chdir(project) + + test_cwd = tmp_path / "test_cwd" + runner.prepare_test_cwd(test_cwd, ["tests"]) + + assert (test_cwd / "tests" / "test_spam.py").exists() + assert not (test_cwd / "test_fail.py").exists() From e46ba8133d430ef70c90446f2b80b56f40fa2a66 Mon Sep 17 00:00:00 2001 From: Henry Schreiner Date: Tue, 9 Jun 2026 17:14:06 -0400 Subject: [PATCH 2/9] refactor: port pyodide to the shared build runner The per-wheel loop in build() is replaced by a PyodideBuilder driven by runner.run_builds(). Behavior changes: - a compatible-wheel (abi3) reuse no longer crashes with a NameError (the old loop assigned built_wheel but read repaired_wheel; it was unreachable in practice since pyodide wheels are never reused) - a repair command producing zero wheels now raises RepairStepProducedNoWheelError instead of a raw StopIteration, and producing several is now an error, like on Linux - before_test no longer receives the undocumented {wheel} placeholder; test_command now does, matching the other platforms - the per-identifier temp dir is now removed after each build Assisted-by: ClaudeCode:claude-fable-5 --- cibuildwheel/platforms/pyodide.py | 392 +++++++++++++----------------- 1 file changed, 166 insertions(+), 226 deletions(-) diff --git a/cibuildwheel/platforms/pyodide.py b/cibuildwheel/platforms/pyodide.py index d4657c769..184650467 100644 --- a/cibuildwheel/platforms/pyodide.py +++ b/cibuildwheel/platforms/pyodide.py @@ -2,7 +2,6 @@ __lazy_modules__ = { "cibuildwheel.architecture", - "cibuildwheel.audit", "cibuildwheel.frontend", "cibuildwheel.logger", "cibuildwheel.util", @@ -15,8 +14,6 @@ "filelock", "json", "pathlib", - "shutil", - "subprocess", "tempfile", "tomllib", } @@ -25,8 +22,6 @@ import functools import json import os -import shutil -import subprocess import sys import tomllib import typing @@ -38,22 +33,20 @@ from cibuildwheel import errors from cibuildwheel.architecture import Architecture -from cibuildwheel.audit import run_audit from cibuildwheel.frontend import get_build_frontend_extra_flags, prepare_config_settings from cibuildwheel.logger import log +from cibuildwheel.platforms import runner from cibuildwheel.util import resources from cibuildwheel.util.cmd import call, shell from cibuildwheel.util.file import ( CIBW_CACHE_PATH, - copy_test_sources, download, extract_tar, extract_zip, - move_file, remove_on_error, ) from cibuildwheel.util.helpers import prepare_command, unwrap, unwrap_preserving_paragraphs -from cibuildwheel.util.packaging import find_compatible_wheel, get_pip_version +from cibuildwheel.util.packaging import get_pip_version from cibuildwheel.util.python_build_standalone import ( PythonBuildStandaloneError, create_python_build_standalone_environment, @@ -65,7 +58,7 @@ from collections.abc import Set from cibuildwheel.environment import ParsedEnvironment - from cibuildwheel.options import Options + from cibuildwheel.options import BuildOptions, Options from cibuildwheel.selector import BuildSelector IS_WIN: Final[bool] = sys.platform.startswith("win") @@ -400,237 +393,184 @@ def get_python_configurations( return [c for c in all_python_configurations() if build_selector(c.identifier)] -def build(options: Options, tmp_path: Path) -> None: - python_configurations = get_python_configurations( - options.globals.build_selector, options.globals.architectures - ) +class PyodideBuilder(runner.HostBuilder): + def __init__( + self, + *, + config: PythonConfiguration, + build_options: BuildOptions, + tmp_dir: Path, + session_tmp_dir: Path, + ) -> None: + super().__init__( + identifier=config.identifier, + build_options=build_options, + tmp_dir=tmp_dir, + session_tmp_dir=session_tmp_dir, + ) + self.config = config - if not python_configurations: - return + def setup(self) -> None: + build_options = self.build_options - try: - before_all_options_identifier = python_configurations[0].identifier - before_all_options = options.build_options(before_all_options_identifier) - - if before_all_options.before_all: - log.step("Running before_all...") - env = before_all_options.environment.as_dictionary(prev_environment=os.environ) - before_all_prepared = prepare_command( - before_all_options.before_all, project=".", package=before_all_options.package_dir - ) - shell(before_all_prepared, env=env) + self.tmp_dir.mkdir() + self.built_wheel_dir.mkdir() - built_wheels: list[Path] = [] + constraints_path = build_options.dependency_constraints.get_for_python_version( + version=self.config.version, variant="pyodide", tmp_dir=self.tmp_dir + ) - for config in python_configurations: - build_options = options.build_options(config.identifier) - build_frontend = build_options.build_frontend + env = setup_python( + tmp=self.tmp_dir / "build", + python_configuration=self.config, + constraints_path=constraints_path, + environment=build_options.environment, + user_pyodide_version=build_options.pyodide_version, + ) + env["CIBUILDWHEEL_BUILD_IDENTIFIER"] = self.identifier + self.pip_version = get_pip_version(env) + + # The Pyodide command line runner mounts all directories in the host + # filesystem into the Pyodide file system, except for the custom + # file systems /dev, /lib, /proc, and /tmp. Mounting the mount + # points for alternate file systems causes some mysterious failure + # of the process (it just quits without any clear error). + # + # Because of this, by default Pyodide can't see anything under /tmp. + # This environment variable tells it also to mount our temp + # directory. + oldmounts = "" + extra_mounts = [str(self.tmp_dir)] + if Path.cwd().is_relative_to("/tmp"): + extra_mounts.append(str(Path.cwd())) + + if "_PYODIDE_EXTRA_MOUNTS" in env: + oldmounts = env["_PYODIDE_EXTRA_MOUNTS"] + ":" + env["_PYODIDE_EXTRA_MOUNTS"] = oldmounts + ":".join(extra_mounts) + + self.env = env + + def build_wheel(self) -> Path: + build_options = self.build_options + + extra_flags = get_build_frontend_extra_flags( + build_options.build_frontend, + build_options.build_verbosity, + prepare_config_settings( + build_options.config_settings, + project=Path.cwd(), + package=build_options.package_dir, + ), + ) - log.build_start(config.identifier) + call( + "pyodide", + "build", + build_options.package_dir, + f"--outdir={self.built_wheel_dir}", + *extra_flags, + env=self.env, + ) + try: + return next(self.built_wheel_dir.glob("*.whl")) + except StopIteration: + raise errors.BuildProducedNoWheelError() from None + + def test_wheel(self, repaired_wheel: Path) -> None: + build_options = self.build_options + assert build_options.test_command is not None + + log.step("Testing wheel...") + + # set up a virtual environment to install and test from, to make sure + # there are no dependencies that were pulled in at build time. + venv_dir = self.tmp_dir / "venv-test" + + virtualenv_env = self.env.copy() + virtualenv_env["PATH"] = os.pathsep.join( + [ + str(ensure_node(self.config.node_version)), + virtualenv_env["PATH"], + ] + ) - identifier_tmp_dir = tmp_path / config.identifier + # pyodide venv uses virtualenv under the hood + # use the pip embedded with virtualenv & disable network updates + virtualenv_create_env = virtualenv_env.copy() + virtualenv_create_env["VIRTUALENV_PIP"] = self.pip_version + virtualenv_create_env["VIRTUALENV_NO_PERIODIC_UPDATE"] = "1" - built_wheel_dir = identifier_tmp_dir / "built_wheel" - repaired_wheel_dir = identifier_tmp_dir / "repaired_wheel" - identifier_tmp_dir.mkdir() - built_wheel_dir.mkdir() - repaired_wheel_dir.mkdir() + call("pyodide", "venv", venv_dir, env=virtualenv_create_env) - constraints_path = build_options.dependency_constraints.get_for_python_version( - version=config.version, variant="pyodide", tmp_dir=identifier_tmp_dir - ) + virtualenv_env["PATH"] = os.pathsep.join( + [ + str(venv_dir / "bin"), + virtualenv_env["PATH"], + ] + ) + virtualenv_env["VIRTUAL_ENV"] = str(venv_dir) + + virtualenv_env = build_options.test_environment.as_dictionary( + prev_environment=virtualenv_env + ) - env = setup_python( - tmp=identifier_tmp_dir / "build", - python_configuration=config, - constraints_path=constraints_path, - environment=build_options.environment, - user_pyodide_version=build_options.pyodide_version, + # check that we are using the Python from the virtual environment + call("which", "python", env=virtualenv_env) + + if build_options.before_test: + before_test_prepared = prepare_command( + build_options.before_test, + project=".", + package=build_options.package_dir, ) - env["CIBUILDWHEEL_BUILD_IDENTIFIER"] = config.identifier - pip_version = get_pip_version(env) - # The Pyodide command line runner mounts all directories in the host - # filesystem into the Pyodide file system, except for the custom - # file systems /dev, /lib, /proc, and /tmp. Mounting the mount - # points for alternate file systems causes some mysterious failure - # of the process (it just quits without any clear error). - # - # Because of this, by default Pyodide can't see anything under /tmp. - # This environment variable tells it also to mount our temp - # directory. - oldmounts = "" - extra_mounts = [str(identifier_tmp_dir)] - if Path.cwd().is_relative_to("/tmp"): - extra_mounts.append(str(Path.cwd())) - - if "_PYODIDE_EXTRA_MOUNTS" in env: - oldmounts = env["_PYODIDE_EXTRA_MOUNTS"] + ":" - env["_PYODIDE_EXTRA_MOUNTS"] = oldmounts + ":".join(extra_mounts) - - compatible_wheel = find_compatible_wheel(built_wheels, config.identifier) - if compatible_wheel: - log.step_end() - print( - f"\nFound previously built wheel {compatible_wheel.name}, that's compatible with {config.identifier}. Skipping build step..." - ) - built_wheel = compatible_wheel - else: - if build_options.before_build: - log.step("Running before_build...") - before_build_prepared = prepare_command( - build_options.before_build, project=".", package=build_options.package_dir - ) - shell(before_build_prepared, env=env) - - log.step("Building wheel...") - - extra_flags = get_build_frontend_extra_flags( - build_frontend, - build_options.build_verbosity, - prepare_config_settings( - build_options.config_settings, - project=Path.cwd(), - package=build_options.package_dir, - ), - ) + shell(before_test_prepared, env=virtualenv_env) + + # install the wheel + call( + "pip", + "install", + f"{repaired_wheel}{build_options.test_extras}", + env=virtualenv_env, + ) - call( - "pyodide", - "build", - build_options.package_dir, - f"--outdir={built_wheel_dir}", - *extra_flags, - env=env, - ) - try: - built_wheel = next(built_wheel_dir.glob("*.whl")) - except StopIteration: - raise errors.BuildProducedNoWheelError() from None - - if built_wheel.name.endswith("none-any.whl"): - raise errors.NonPlatformWheelError() - - if build_options.repair_command: - log.step("Repairing wheel...") - - repair_command_prepared = prepare_command( - build_options.repair_command, - wheel=built_wheel, - dest_dir=repaired_wheel_dir, - package=build_options.package_dir, - project=".", - ) - shell(repair_command_prepared, env=env) - log.step_end() - else: - shutil.move(str(built_wheel), repaired_wheel_dir) - - try: - repaired_wheel = next(repaired_wheel_dir.glob("*.whl")) - except StopIteration: - raise errors.RepairStepProducedNoWheelError() from None - - if repaired_wheel.name in {wheel.name for wheel in built_wheels}: - raise errors.AlreadyBuiltWheelError(repaired_wheel.name) - - run_audit(tmp_dir=tmp_path, build_options=build_options, wheel=repaired_wheel) - - if build_options.test_command and build_options.test_selector(config.identifier): - log.step("Testing wheel...") - - venv_dir = identifier_tmp_dir / "venv-test" - # set up a virtual environment to install and test from, to make sure - # there are no dependencies that were pulled in at build time. - - virtualenv_env = env.copy() - virtualenv_env["PATH"] = os.pathsep.join( - [ - str(ensure_node(config.node_version)), - virtualenv_env["PATH"], - ] - ) + # test the wheel + if build_options.test_requires: + call("pip", "install", *build_options.test_requires, env=virtualenv_env) + + # run the tests from a temp dir, with an absolute path in the command + # (this ensures that Python runs the tests against the installed wheel + # and not the repo code) + test_command_prepared = prepare_command( + build_options.test_command, + project=Path.cwd(), + package=build_options.package_dir.resolve(), + wheel=repaired_wheel, + ) - # pyodide venv uses virtualenv under the hood - # use the pip embedded with virtualenv & disable network updates - virtualenv_create_env = virtualenv_env.copy() - virtualenv_create_env["VIRTUALENV_PIP"] = pip_version - virtualenv_create_env["VIRTUALENV_NO_PERIODIC_UPDATE"] = "1" + test_cwd = self.tmp_dir / "test_cwd" + runner.prepare_test_cwd(test_cwd, build_options.test_sources) - call("pyodide", "venv", venv_dir, env=virtualenv_create_env) + shell(test_command_prepared, cwd=test_cwd, env=virtualenv_env) - virtualenv_env["PATH"] = os.pathsep.join( - [ - str(venv_dir / "bin"), - virtualenv_env["PATH"], - ] - ) - virtualenv_env["VIRTUAL_ENV"] = str(venv_dir) - virtualenv_env = build_options.test_environment.as_dictionary( - prev_environment=virtualenv_env - ) +def build(options: Options, tmp_path: Path) -> None: + python_configurations = get_python_configurations( + options.globals.build_selector, options.globals.architectures + ) - # check that we are using the Python from the virtual environment - call("which", "python", env=virtualenv_env) - - if build_options.before_test: - before_test_prepared = prepare_command( - build_options.before_test, - project=".", - package=build_options.package_dir, - wheel=repaired_wheel, - ) - shell(before_test_prepared, env=virtualenv_env) - - # install the wheel - call( - "pip", - "install", - f"{repaired_wheel}{build_options.test_extras}", - env=virtualenv_env, - ) + if not python_configurations: + return - # test the wheel - if build_options.test_requires: - call("pip", "install", *build_options.test_requires, env=virtualenv_env) - - # run the tests from a temp dir, with an absolute path in the command - # (this ensures that Python runs the tests against the installed wheel - # and not the repo code) - test_command_prepared = prepare_command( - build_options.test_command, - project=Path.cwd(), - package=build_options.package_dir.resolve(), + with runner.fatal_on_called_process_error(): + runner.run_before_all(options, python_configurations) + runner.run_builds( + [ + PyodideBuilder( + config=config, + build_options=options.build_options(config.identifier), + tmp_dir=tmp_path / config.identifier, + session_tmp_dir=tmp_path, ) - - test_cwd = identifier_tmp_dir / "test_cwd" - test_cwd.mkdir(exist_ok=True) - - if build_options.test_sources: - copy_test_sources( - build_options.test_sources, - Path.cwd(), - test_cwd, - ) - else: - # Use the test_fail.py file to raise a nice error if the user - # tries to run tests in the cwd - (test_cwd / "test_fail.py").write_text(resources.TEST_FAIL_CWD_FILE.read_text()) - - shell(test_command_prepared, cwd=test_cwd, env=virtualenv_env) - - # we're all done here; move it to output (overwrite existing) - output_wheel: Path | None = None - if compatible_wheel is None: - output_wheel = build_options.output_dir.joinpath(repaired_wheel.name) - moved_wheel = move_file(repaired_wheel, output_wheel) - if moved_wheel != output_wheel.resolve(): - log.warning( - f"{repaired_wheel} was moved to {moved_wheel} instead of {output_wheel}" - ) - built_wheels.append(output_wheel) - log.build_end(output_wheel) - - except subprocess.CalledProcessError as error: - msg = f"Command {error.cmd} failed with code {error.returncode}. {error.stdout or ''}" - raise errors.FatalError(msg) from error + for config in python_configurations + ] + ) From 45132587aef5113eeabb9a4e9ad83c546f5b1741 Mon Sep 17 00:00:00 2001 From: Henry Schreiner Date: Tue, 9 Jun 2026 17:16:44 -0400 Subject: [PATCH 3/9] refactor: port macos to the shared build runner The per-wheel loop becomes a MacOSBuilder; the per-architecture test loop (universal2 testing both halves, Rosetta emulation, arch-specific test-skip identifiers) stays inside test_wheel(), unchanged. Behavior changes: - a repair command producing several wheels is now an error, as on Linux - a missing uv now raises FatalError instead of AssertionError - the per-identifier temp dir is removed with ignore_errors, like Windows Assisted-by: ClaudeCode:claude-fable-5 --- cibuildwheel/platforms/macos.py | 614 +++++++++++++++----------------- 1 file changed, 287 insertions(+), 327 deletions(-) diff --git a/cibuildwheel/platforms/macos.py b/cibuildwheel/platforms/macos.py index 4c3c74440..2ef92eb0c 100644 --- a/cibuildwheel/platforms/macos.py +++ b/cibuildwheel/platforms/macos.py @@ -1,7 +1,6 @@ from __future__ import annotations __lazy_modules__ = { - "cibuildwheel.audit", "cibuildwheel.ci", "cibuildwheel.frontend", "cibuildwheel.logger", @@ -19,7 +18,6 @@ "platform", "re", "shutil", - "subprocess", } import dataclasses @@ -29,7 +27,6 @@ import platform import re import shutil -import subprocess import sys import typing from pathlib import Path @@ -39,7 +36,6 @@ from packaging.version import Version from cibuildwheel import errors -from cibuildwheel.audit import run_audit from cibuildwheel.ci import detect_ci_provider from cibuildwheel.frontend import ( BuildFrontendName, @@ -47,17 +43,16 @@ prepare_config_settings, ) from cibuildwheel.logger import log +from cibuildwheel.platforms import runner from cibuildwheel.util import resources from cibuildwheel.util.cmd import call, shell from cibuildwheel.util.file import ( CIBW_CACHE_PATH, - copy_test_sources, download, - move_file, remove_on_error, ) from cibuildwheel.util.helpers import prepare_command, unwrap -from cibuildwheel.util.packaging import find_compatible_wheel, get_pip_version +from cibuildwheel.util.packaging import get_pip_version from cibuildwheel.venv import constraint_flags, find_uv, target_marker_env, virtualenv TYPE_CHECKING = False @@ -67,7 +62,7 @@ from cibuildwheel.architecture import Architecture from cibuildwheel.environment import ParsedEnvironment - from cibuildwheel.options import Options + from cibuildwheel.options import BuildOptions, Options from cibuildwheel.selector import BuildSelector @@ -458,350 +453,315 @@ def setup_python( return base_python, env -def build(options: Options, tmp_path: Path) -> None: - python_configurations = get_python_configurations( - options.globals.build_selector, options.globals.architectures - ) +class MacOSBuilder(runner.HostBuilder): + def __init__( + self, + *, + config: PythonConfiguration, + build_options: BuildOptions, + tmp_dir: Path, + session_tmp_dir: Path, + ) -> None: + super().__init__( + identifier=config.identifier, + build_options=build_options, + tmp_dir=tmp_dir, + session_tmp_dir=session_tmp_dir, + ) + self.config = config - if not python_configurations: - return + @property + def config_is_arm64(self) -> bool: + return self.identifier.endswith("arm64") - try: - before_all_options_identifier = python_configurations[0].identifier - before_all_options = options.build_options(before_all_options_identifier) + @property + def config_is_universal2(self) -> bool: + return self.identifier.endswith("universal2") - if before_all_options.before_all: - log.step("Running before_all...") - env = before_all_options.environment.as_dictionary(prev_environment=os.environ) - env.setdefault("MACOSX_DEPLOYMENT_TARGET", "10.9") - before_all_prepared = prepare_command( - before_all_options.before_all, project=".", package=before_all_options.package_dir - ) - shell(before_all_prepared, env=env) - - built_wheels: list[Path] = [] - - for config in python_configurations: - build_options = options.build_options(config.identifier) - build_frontend = build_options.build_frontend - use_uv = build_frontend.name in {"build[uv]", "uv"} - uv_path = find_uv() - if use_uv and uv_path is None: - msg = "uv not found" - raise AssertionError(msg) - pip = ["pip"] if not use_uv else [str(uv_path), "pip"] - log.build_start(config.identifier) - - identifier_tmp_dir = tmp_path / config.identifier - identifier_tmp_dir.mkdir() - built_wheel_dir = identifier_tmp_dir / "built_wheel" - repaired_wheel_dir = identifier_tmp_dir / "repaired_wheel" - - config_is_arm64 = config.identifier.endswith("arm64") - config_is_universal2 = config.identifier.endswith("universal2") - - constraints_path = build_options.dependency_constraints.get_for_python_version( - version=config.version, tmp_dir=identifier_tmp_dir - ) + def setup(self) -> None: + build_options = self.build_options + build_frontend = build_options.build_frontend - base_python, env = setup_python( - identifier_tmp_dir / "build", - config, - constraints_path, - build_options.environment, - build_frontend.name, - ) - env["CIBUILDWHEEL_BUILD_IDENTIFIER"] = config.identifier - pip_version = None if use_uv else get_pip_version(env) - - compatible_wheel = find_compatible_wheel(built_wheels, config.identifier) - if compatible_wheel: - log.step_end() - print( - f"\nFound previously built wheel {compatible_wheel.name}, that's compatible with {config.identifier}. Skipping build step..." + self.use_uv = build_frontend.name in {"build[uv]", "uv"} + self.uv_path = find_uv() + if self.use_uv and self.uv_path is None: + msg = "uv not found" + raise errors.FatalError(msg) + self.pip: list[str] = ["pip"] if not self.use_uv else [str(self.uv_path), "pip"] + + self.tmp_dir.mkdir() + + constraints_path = build_options.dependency_constraints.get_for_python_version( + version=self.config.version, tmp_dir=self.tmp_dir + ) + + self.base_python, env = setup_python( + self.tmp_dir / "build", + self.config, + constraints_path, + build_options.environment, + build_frontend.name, + ) + env["CIBUILDWHEEL_BUILD_IDENTIFIER"] = self.identifier + self.pip_version = None if self.use_uv else get_pip_version(env) + self.env = env + + def build_wheel(self) -> Path: + build_options = self.build_options + build_frontend = build_options.build_frontend + + self.built_wheel_dir.mkdir() + + extra_flags = get_build_frontend_extra_flags( + build_frontend, + build_options.build_verbosity, + prepare_config_settings( + build_options.config_settings, + project=Path.cwd(), + package=build_options.package_dir, + ), + ) + + build_env = self.env.copy() + + match build_frontend.name: + case "pip": + # Path.resolve() is needed. Without it pip wheel may try to fetch package from pypi.org + # see https://github.com/pypa/cibuildwheel/pull/369 + call( + "python", + "-m", + "pip", + "wheel", + build_options.package_dir.resolve(), + f"--wheel-dir={self.built_wheel_dir}", + "--no-deps", + *extra_flags, + env=build_env, ) - repaired_wheel = compatible_wheel - else: - if build_options.before_build: - log.step("Running before_build...") - before_build_prepared = prepare_command( - build_options.before_build, project=".", package=build_options.package_dir - ) - shell(before_build_prepared, env=env) - - log.step("Building wheel...") - built_wheel_dir.mkdir() - - extra_flags = get_build_frontend_extra_flags( - build_frontend, - build_options.build_verbosity, - prepare_config_settings( - build_options.config_settings, - project=Path.cwd(), - package=build_options.package_dir, - ), + case "build" | "build[uv]": + if self.use_uv and "--no-isolation" not in extra_flags and "-n" not in extra_flags: + extra_flags.append("--installer=uv") + call( + "python", + "-m", + "build", + build_options.package_dir, + "--wheel", + f"--outdir={self.built_wheel_dir}", + *extra_flags, + env=build_env, ) + case "uv": + assert self.uv_path is not None + call( + self.uv_path, + "build", + f"--python={self.base_python}", + build_options.package_dir, + "--wheel", + f"--out-dir={self.built_wheel_dir}", + *extra_flags, + env=build_env, + ) + case "pyodide-build": + msg = "The 'pyodide-build' build frontend is not supported on this platform" + raise errors.FatalError(msg) + case _: + assert_never(build_frontend) + + try: + return next(self.built_wheel_dir.glob("*.whl")) + except StopIteration: + raise errors.BuildProducedNoWheelError() from None + + def repair_wheel(self, built_wheel: Path) -> list[Path]: + build_options = self.build_options + self.repaired_wheel_dir.mkdir(exist_ok=True) + + if build_options.repair_command: + if self.config_is_universal2: + delocate_archs = "x86_64,arm64" + elif self.config_is_arm64: + delocate_archs = "arm64" + else: + delocate_archs = "x86_64" + + repair_command_prepared = prepare_command( + build_options.repair_command, + wheel=built_wheel, + dest_dir=self.repaired_wheel_dir, + delocate_archs=delocate_archs, + package=build_options.package_dir, + project=".", + ) + shell(repair_command_prepared, env=self.env) + else: + shutil.move(str(built_wheel), self.repaired_wheel_dir) - build_env = env.copy() - - match build_frontend.name: - case "pip": - # Path.resolve() is needed. Without it pip wheel may try to fetch package from pypi.org - # see https://github.com/pypa/cibuildwheel/pull/369 - call( - "python", - "-m", - "pip", - "wheel", - build_options.package_dir.resolve(), - f"--wheel-dir={built_wheel_dir}", - "--no-deps", - *extra_flags, - env=build_env, - ) - case "build" | "build[uv]": - if ( - use_uv - and "--no-isolation" not in extra_flags - and "-n" not in extra_flags - ): - extra_flags.append("--installer=uv") - call( - "python", - "-m", - "build", - build_options.package_dir, - "--wheel", - f"--outdir={built_wheel_dir}", - *extra_flags, - env=build_env, + return list(self.repaired_wheel_dir.glob("*.whl")) + + def test_wheel(self, repaired_wheel: Path) -> None: + build_options = self.build_options + assert build_options.test_command is not None + + machine_arch = platform.machine() + testing_archs: list[Literal["x86_64", "arm64"]] + + if self.config_is_arm64: + testing_archs = ["arm64"] + elif self.config_is_universal2: + testing_archs = ["x86_64", "arm64"] + else: + testing_archs = ["x86_64"] + + for testing_arch in testing_archs: + if self.config_is_universal2: + arch_specific_identifier = f"{self.identifier}:{testing_arch}" + if not build_options.test_selector(arch_specific_identifier): + continue + + if machine_arch == "x86_64" and testing_arch == "arm64": + if self.config_is_arm64: + log.warning( + unwrap( + """ + While arm64 wheels can be built on x86_64, they cannot be + tested. Consider building arm64 wheels natively, if your CI + provider offers this. To silence this warning, set + `CIBW_TEST_SKIP: "*-macosx_arm64"`. + """ ) - case "uv": - assert uv_path is not None - call( - uv_path, - "build", - f"--python={base_python}", - build_options.package_dir, - "--wheel", - f"--out-dir={built_wheel_dir}", - *extra_flags, - env=build_env, + ) + elif self.config_is_universal2: + log.warning( + unwrap( + """ + While universal2 wheels can be built on x86_64, the arm64 part + of the wheel cannot be tested on x86_64. Consider building + universal2 wheels on an arm64 runner, if your CI provider offers + this. Notably, an arm64 runner can also test the x86_64 part of + the wheel, through Rosetta emulation. To silence this warning, + set `CIBW_TEST_SKIP: "*-macosx_universal2:arm64"`. + """ ) - case "pyodide-build": - msg = "The 'pyodide-build' build frontend is not supported on this platform" - raise errors.FatalError(msg) - case _: - assert_never(build_frontend) - - try: - built_wheel = next(built_wheel_dir.glob("*.whl")) - except StopIteration: - raise errors.BuildProducedNoWheelError() from None - - repaired_wheel_dir.mkdir() - - if built_wheel.name.endswith("none-any.whl"): - raise errors.NonPlatformWheelError() - - if build_options.repair_command: - log.step("Repairing wheel...") - - if config_is_universal2: - delocate_archs = "x86_64,arm64" - elif config_is_arm64: - delocate_archs = "arm64" - else: - delocate_archs = "x86_64" - - repair_command_prepared = prepare_command( - build_options.repair_command, - wheel=built_wheel, - dest_dir=repaired_wheel_dir, - delocate_archs=delocate_archs, - package=build_options.package_dir, - project=".", ) - shell(repair_command_prepared, env=env) else: - shutil.move(str(built_wheel), repaired_wheel_dir) - - try: - repaired_wheel = next(repaired_wheel_dir.glob("*.whl")) - except StopIteration: - raise errors.RepairStepProducedNoWheelError() from None + msg = "unreachable" + raise RuntimeError(msg) - if repaired_wheel.name in {wheel.name for wheel in built_wheels}: - raise errors.AlreadyBuiltWheelError(repaired_wheel.name) + # skip this test + continue - log.step_end() - - run_audit(tmp_dir=tmp_path, build_options=build_options, wheel=repaired_wheel) - - if build_options.test_command and build_options.test_selector(config.identifier): - machine_arch = platform.machine() - testing_archs: list[Literal["x86_64", "arm64"]] + log.step( + "Testing wheel..." + if testing_arch == machine_arch + else f"Testing wheel on {testing_arch}..." + ) - if config_is_arm64: - testing_archs = ["arm64"] - elif config_is_universal2: - testing_archs = ["x86_64", "arm64"] + arch_prefix = [] + uv_arch_args = [] + if testing_arch != machine_arch: + if machine_arch == "arm64" and testing_arch == "x86_64": + # rosetta2 will provide the emulation with just the arch prefix. + arch_prefix = ["arch", "-x86_64"] + uv_arch_args = ["--python-platform", "x86_64-apple-darwin"] else: - testing_archs = ["x86_64"] - - for testing_arch in testing_archs: - if config_is_universal2: - arch_specific_identifier = f"{config.identifier}:{testing_arch}" - if not build_options.test_selector(arch_specific_identifier): - continue - - if machine_arch == "x86_64" and testing_arch == "arm64": - if config_is_arm64: - log.warning( - unwrap( - """ - While arm64 wheels can be built on x86_64, they cannot be - tested. Consider building arm64 wheels natively, if your CI - provider offers this. To silence this warning, set - `CIBW_TEST_SKIP: "*-macosx_arm64"`. - """ - ) - ) - elif config_is_universal2: - log.warning( - unwrap( - """ - While universal2 wheels can be built on x86_64, the arm64 part - of the wheel cannot be tested on x86_64. Consider building - universal2 wheels on an arm64 runner, if your CI provider offers - this. Notably, an arm64 runner can also test the x86_64 part of - the wheel, through Rosetta emulation. To silence this warning, - set `CIBW_TEST_SKIP: "*-macosx_universal2:arm64"`. - """ - ) - ) - else: - msg = "unreachable" - raise RuntimeError(msg) - - # skip this test - continue - - log.step( - "Testing wheel..." - if testing_arch == machine_arch - else f"Testing wheel on {testing_arch}..." - ) + msg = f"don't know how to emulate {testing_arch} on {machine_arch}" + raise RuntimeError(msg) + + # define a custom 'call' function that adds the arch prefix each time + call_with_arch = functools.partial(call, *arch_prefix) + shell_with_arch = functools.partial(call, *arch_prefix, "/bin/sh", "-c") + + # set up a virtual environment to install and test from, to make sure + # there are no dependencies that were pulled in at build time. + venv_dir = self.tmp_dir / f"venv-test-{testing_arch}" + virtualenv_env = virtualenv( + self.config.version, + self.base_python, + venv_dir, + None, + use_uv=self.use_uv, + env=self.env, + pip_version=self.pip_version, + ) + if self.use_uv: + pip_install = functools.partial(call, *self.pip, "install", *uv_arch_args) + else: + pip_install = functools.partial(call_with_arch, *self.pip, "install") - arch_prefix = [] - uv_arch_args = [] - if testing_arch != machine_arch: - if machine_arch == "arm64" and testing_arch == "x86_64": - # rosetta2 will provide the emulation with just the arch prefix. - arch_prefix = ["arch", "-x86_64"] - uv_arch_args = ["--python-platform", "x86_64-apple-darwin"] - else: - msg = f"don't know how to emulate {testing_arch} on {machine_arch}" - raise RuntimeError(msg) - - # define a custom 'call' function that adds the arch prefix each time - call_with_arch = functools.partial(call, *arch_prefix) - shell_with_arch = functools.partial(call, *arch_prefix, "/bin/sh", "-c") - - # set up a virtual environment to install and test from, to make sure - # there are no dependencies that were pulled in at build time. - venv_dir = identifier_tmp_dir / f"venv-test-{testing_arch}" - virtualenv_env = virtualenv( - config.version, - base_python, - venv_dir, - None, - use_uv=use_uv, - env=env, - pip_version=pip_version, - ) - if use_uv: - pip_install = functools.partial(call, *pip, "install", *uv_arch_args) - else: - pip_install = functools.partial(call_with_arch, *pip, "install") + virtualenv_env["MACOSX_DEPLOYMENT_TARGET"] = get_test_macosx_deployment_target() - virtualenv_env["MACOSX_DEPLOYMENT_TARGET"] = get_test_macosx_deployment_target() + virtualenv_env = build_options.test_environment.as_dictionary( + prev_environment=virtualenv_env + ) - virtualenv_env = build_options.test_environment.as_dictionary( - prev_environment=virtualenv_env - ) + # check that we are using the Python from the virtual environment + call_with_arch("which", "python", env=virtualenv_env) - # check that we are using the Python from the virtual environment - call_with_arch("which", "python", env=virtualenv_env) + if build_options.before_test: + before_test_prepared = prepare_command( + build_options.before_test, + project=".", + package=build_options.package_dir, + ) + shell_with_arch(before_test_prepared, env=virtualenv_env) - if build_options.before_test: - before_test_prepared = prepare_command( - build_options.before_test, - project=".", - package=build_options.package_dir, - ) - shell_with_arch(before_test_prepared, env=virtualenv_env) + # install the wheel + pip_install( + f"{repaired_wheel}{build_options.test_extras}", + env=virtualenv_env, + ) - # install the wheel - pip_install( - f"{repaired_wheel}{build_options.test_extras}", - env=virtualenv_env, - ) + # test the wheel + if build_options.test_requires: + pip_install( + *build_options.test_requires, + env=virtualenv_env, + ) - # test the wheel - if build_options.test_requires: - pip_install( - *build_options.test_requires, - env=virtualenv_env, - ) + # run the tests from a temp dir, with an absolute path in the command + # (this ensures that Python runs the tests against the installed wheel + # and not the repo code) + test_command_prepared = prepare_command( + build_options.test_command, + project=Path.cwd(), + package=build_options.package_dir.resolve(), + wheel=repaired_wheel, + ) - # run the tests from a temp dir, with an absolute path in the command - # (this ensures that Python runs the tests against the installed wheel - # and not the repo code) - test_command_prepared = prepare_command( - build_options.test_command, - project=Path.cwd(), - package=build_options.package_dir.resolve(), - wheel=repaired_wheel, - ) + # only populate test_cwd if it doesn't already exist - it + # may have been created during a previous `testing_arch` + test_cwd = self.tmp_dir / "test_cwd" + if not test_cwd.exists(): + runner.prepare_test_cwd(test_cwd, build_options.test_sources) - test_cwd = identifier_tmp_dir / "test_cwd" - - if build_options.test_sources: - # only create test_cwd if it doesn't already exist - it - # may have been created during a previous `testing_arch` - if not test_cwd.exists(): - test_cwd.mkdir() - copy_test_sources( - build_options.test_sources, - Path.cwd(), - test_cwd, - ) - else: - # Use the test_fail.py file to raise a nice error if the user - # tries to run tests in the cwd - test_cwd.mkdir(exist_ok=True) - (test_cwd / "test_fail.py").write_text( - resources.TEST_FAIL_CWD_FILE.read_text() - ) + shell_with_arch(test_command_prepared, cwd=test_cwd, env=virtualenv_env) - shell_with_arch(test_command_prepared, cwd=test_cwd, env=virtualenv_env) - # we're all done here; move it to output (overwrite existing) - output_wheel = None - if compatible_wheel is None: - output_wheel = build_options.output_dir.joinpath(repaired_wheel.name) - moved_wheel = move_file(repaired_wheel, output_wheel) - if moved_wheel != output_wheel.resolve(): - log.warning( - f"{repaired_wheel} was moved to {moved_wheel} instead of {output_wheel}" - ) - built_wheels.append(output_wheel) +def build(options: Options, tmp_path: Path) -> None: + python_configurations = get_python_configurations( + options.globals.build_selector, options.globals.architectures + ) - # clean up - shutil.rmtree(identifier_tmp_dir) + if not python_configurations: + return - log.build_end(output_wheel) - except subprocess.CalledProcessError as error: - msg = f"Command {error.cmd} failed with code {error.returncode}. {error.stdout or ''}" - raise errors.FatalError(msg) from error + with runner.fatal_on_called_process_error(): + runner.run_before_all( + options, + python_configurations, + env_defaults={"MACOSX_DEPLOYMENT_TARGET": "10.9"}, + ) + runner.run_builds( + [ + MacOSBuilder( + config=config, + build_options=options.build_options(config.identifier), + tmp_dir=tmp_path / config.identifier, + session_tmp_dir=tmp_path, + ) + for config in python_configurations + ] + ) From 6b7bce2e333893c8b917f98bd833884d0bd29528 Mon Sep 17 00:00:00 2001 From: Henry Schreiner Date: Tue, 9 Jun 2026 17:18:49 -0400 Subject: [PATCH 4/9] refactor: port windows to the shared build runner The per-wheel loop becomes a WindowsBuilder. The ARM64-on-non-ARM64 test skip moves inside test_wheel(), so its warning now only appears when a test command is actually configured. A repair command producing several wheels is now an error, as on Linux. Assisted-by: ClaudeCode:claude-fable-5 --- cibuildwheel/platforms/windows.py | 468 +++++++++++++----------------- 1 file changed, 199 insertions(+), 269 deletions(-) diff --git a/cibuildwheel/platforms/windows.py b/cibuildwheel/platforms/windows.py index d7df473ef..50f68e04a 100644 --- a/cibuildwheel/platforms/windows.py +++ b/cibuildwheel/platforms/windows.py @@ -2,7 +2,6 @@ __lazy_modules__ = { "cibuildwheel.architecture", - "cibuildwheel.audit", "cibuildwheel.frontend", "cibuildwheel.logger", "cibuildwheel.util", @@ -14,17 +13,12 @@ "filelock", "pathlib", "platform", - "shutil", - "subprocess", "textwrap", "typing", } import dataclasses -import os import platform as platform_module -import shutil -import subprocess import textwrap from functools import cache from pathlib import Path @@ -34,25 +28,23 @@ from cibuildwheel import errors from cibuildwheel.architecture import Architecture -from cibuildwheel.audit import run_audit from cibuildwheel.frontend import ( BuildFrontendName, get_build_frontend_extra_flags, prepare_config_settings, ) from cibuildwheel.logger import log +from cibuildwheel.platforms import runner from cibuildwheel.util import resources from cibuildwheel.util.cmd import call, shell from cibuildwheel.util.file import ( CIBW_CACHE_PATH, - copy_test_sources, download, extract_zip, - move_file, remove_on_error, ) from cibuildwheel.util.helpers import prepare_command, unwrap -from cibuildwheel.util.packaging import find_compatible_wheel, get_pip_version +from cibuildwheel.util.packaging import get_pip_version from cibuildwheel.venv import constraint_flags, find_uv, target_marker_env, virtualenv TYPE_CHECKING = False @@ -60,7 +52,7 @@ from collections.abc import MutableMapping, Sequence, Set from cibuildwheel.environment import ParsedEnvironment - from cibuildwheel.options import Options + from cibuildwheel.options import BuildOptions, Options from cibuildwheel.selector import BuildSelector @@ -415,276 +407,214 @@ def setup_python( return base_python, env -def build(options: Options, tmp_path: Path) -> None: - python_configurations = get_python_configurations( - options.globals.build_selector, options.globals.architectures - ) - - if not python_configurations: - return +class WindowsBuilder(runner.HostBuilder): + def __init__( + self, + *, + config: PythonConfiguration, + build_options: BuildOptions, + tmp_dir: Path, + session_tmp_dir: Path, + ) -> None: + super().__init__( + identifier=config.identifier, + build_options=build_options, + tmp_dir=tmp_dir, + session_tmp_dir=session_tmp_dir, + ) + self.config = config - uv_path = find_uv() + def setup(self) -> None: + build_options = self.build_options + build_frontend = build_options.build_frontend - try: - before_all_options_identifier = python_configurations[0].identifier - before_all_options = options.build_options(before_all_options_identifier) + self.use_uv = build_frontend.name in {"build[uv]", "uv"} + self.uv_path = find_uv() - if before_all_options.before_all: - log.step("Running before_all...") - env = before_all_options.environment.as_dictionary(prev_environment=os.environ) - before_all_prepared = prepare_command( - before_all_options.before_all, project=".", package=options.globals.package_dir - ) - shell(before_all_prepared, env=env) + self.tmp_dir.mkdir() - built_wheels: list[Path] = [] + constraints_path = build_options.dependency_constraints.get_for_python_version( + version=self.config.version, tmp_dir=self.tmp_dir + ) - for config in python_configurations: - build_options = options.build_options(config.identifier) - build_frontend = build_options.build_frontend - use_uv = build_frontend.name in {"build[uv]", "uv"} - log.build_start(config.identifier) + # install Python + self.base_python, env = setup_python( + self.tmp_dir / "build", + self.config, + constraints_path, + build_options.environment, + build_frontend.name, + ) + env["CIBUILDWHEEL_BUILD_IDENTIFIER"] = self.identifier + self.pip_version = None if self.use_uv else get_pip_version(env) + self.env = env + + def build_wheel(self) -> Path: + build_options = self.build_options + build_frontend = build_options.build_frontend + + self.built_wheel_dir.mkdir() + + extra_flags = get_build_frontend_extra_flags( + build_frontend, + build_options.build_verbosity, + prepare_config_settings( + build_options.config_settings, + project=Path.cwd(), + package=build_options.package_dir, + ), + ) - identifier_tmp_dir = tmp_path / config.identifier - identifier_tmp_dir.mkdir() - built_wheel_dir = identifier_tmp_dir / "built_wheel" - repaired_wheel_dir = identifier_tmp_dir / "repaired_wheel" + match build_frontend.name: + case "pip": + # Path.resolve() is needed. Without it pip wheel may try to fetch package from pypi.org + # see https://github.com/pypa/cibuildwheel/pull/369 + call( + "python", + "-m", + "pip", + "wheel", + build_options.package_dir.resolve(), + f"--wheel-dir={self.built_wheel_dir}", + "--no-deps", + *extra_flags, + env=self.env, + ) + case "build" | "build[uv]": + if self.use_uv and "--no-isolation" not in extra_flags and "-n" not in extra_flags: + extra_flags.append("--installer=uv") - constraints_path = build_options.dependency_constraints.get_for_python_version( - version=config.version, - tmp_dir=identifier_tmp_dir, + call( + "python", + "-m", + "build", + build_options.package_dir, + "--wheel", + f"--outdir={self.built_wheel_dir}", + *extra_flags, + env=self.env, + ) + case "uv": + assert self.uv_path is not None + call( + self.uv_path, + "build", + f"--python={self.base_python}", + build_options.package_dir, + "--wheel", + f"--out-dir={self.built_wheel_dir}", + *extra_flags, + env=self.env, + ) + case "pyodide-build": + msg = "The 'pyodide-build' build frontend is not supported on this platform" + raise errors.FatalError(msg) + case _: + assert_never(build_frontend) + + try: + return next(self.built_wheel_dir.glob("*.whl")) + except StopIteration: + raise errors.BuildProducedNoWheelError() from None + + def test_wheel(self, repaired_wheel: Path) -> None: + build_options = self.build_options + assert build_options.test_command is not None + + if self.config.arch == "ARM64" != platform_module.machine(): + log.warning( + unwrap( + """ + While arm64 wheels can be built on other platforms, they cannot + be tested. An arm64 runner is required. To silence this warning, + set `CIBW_TEST_SKIP: "*-win_arm64"`. + """ + ) ) + # skip this test + return + + log.step("Testing wheel...") + # set up a virtual environment to install and test from, to make sure + # there are no dependencies that were pulled in at build time. + venv_dir = self.tmp_dir / "venv-test" + virtualenv_env = virtualenv( + self.config.version, + self.base_python, + venv_dir, + None, + use_uv=self.use_uv, + env=self.env, + pip_version=self.pip_version, + ) - # install Python - base_python, env = setup_python( - identifier_tmp_dir / "build", - config, - constraints_path, - build_options.environment, - build_frontend.name, + virtualenv_env = build_options.test_environment.as_dictionary( + prev_environment=virtualenv_env + ) + + # check that we are using the Python from the virtual environment + call("where", "python", env=virtualenv_env) + + if build_options.before_test: + before_test_prepared = prepare_command( + build_options.before_test, + project=".", + package=build_options.package_dir, ) - env["CIBUILDWHEEL_BUILD_IDENTIFIER"] = config.identifier - pip_version = None if use_uv else get_pip_version(env) - - compatible_wheel = find_compatible_wheel(built_wheels, config.identifier) - if compatible_wheel: - log.step_end() - print( - f"\nFound previously built wheel {compatible_wheel.name}, that's compatible with {config.identifier}. Skipping build step..." - ) - repaired_wheel = compatible_wheel - else: - # run the before_build command - if build_options.before_build: - log.step("Running before_build...") - before_build_prepared = prepare_command( - build_options.before_build, - project=".", - package=options.globals.package_dir, - ) - shell(before_build_prepared, env=env) - - log.step("Building wheel...") - built_wheel_dir.mkdir() - - extra_flags = get_build_frontend_extra_flags( - build_frontend, - build_options.build_verbosity, - prepare_config_settings( - build_options.config_settings, - project=Path.cwd(), - package=options.globals.package_dir, - ), - ) + shell(before_test_prepared, env=virtualenv_env) + + pip: Sequence[Path | str] + if self.use_uv: + assert self.uv_path is not None + pip = [self.uv_path, "pip"] + else: + pip = ["pip"] + + # install the wheel + call( + *pip, + "install", + str(repaired_wheel) + build_options.test_extras, + env=virtualenv_env, + ) - match build_frontend.name: - case "pip": - # Path.resolve() is needed. Without it pip wheel may try to fetch package from pypi.org - # see https://github.com/pypa/cibuildwheel/pull/369 - call( - "python", - "-m", - "pip", - "wheel", - options.globals.package_dir.resolve(), - f"--wheel-dir={built_wheel_dir}", - "--no-deps", - *extra_flags, - env=env, - ) - case "build" | "build[uv]": - if ( - use_uv - and "--no-isolation" not in extra_flags - and "-n" not in extra_flags - ): - extra_flags.append("--installer=uv") - - call( - "python", - "-m", - "build", - build_options.package_dir, - "--wheel", - f"--outdir={built_wheel_dir}", - *extra_flags, - env=env, - ) - case "uv": - assert uv_path is not None - call( - uv_path, - "build", - f"--python={base_python}", - build_options.package_dir, - "--wheel", - f"--out-dir={built_wheel_dir}", - *extra_flags, - env=env, - ) - case "pyodide-build": - msg = "The 'pyodide-build' build frontend is not supported on this platform" - raise errors.FatalError(msg) - case _: - assert_never(build_frontend) - - try: - built_wheel = next(built_wheel_dir.glob("*.whl")) - except StopIteration: - raise errors.BuildProducedNoWheelError() from None - - # repair the wheel - repaired_wheel_dir.mkdir() - - if built_wheel.name.endswith("none-any.whl"): - raise errors.NonPlatformWheelError() - - if build_options.repair_command: - log.step("Repairing wheel...") - repair_command_prepared = prepare_command( - build_options.repair_command, - wheel=built_wheel, - dest_dir=repaired_wheel_dir, - package=build_options.package_dir, - project=".", - ) - shell(repair_command_prepared, env=env) - else: - shutil.move(str(built_wheel), repaired_wheel_dir) - - try: - repaired_wheel = next(repaired_wheel_dir.glob("*.whl")) - except StopIteration: - raise errors.RepairStepProducedNoWheelError() from None - - if repaired_wheel.name in {wheel.name for wheel in built_wheels}: - raise errors.AlreadyBuiltWheelError(repaired_wheel.name) - - run_audit(tmp_dir=tmp_path, build_options=build_options, wheel=repaired_wheel) - - test_selected = options.globals.test_selector(config.identifier) - if test_selected and config.arch == "ARM64" != platform_module.machine(): - log.warning( - unwrap( - """ - While arm64 wheels can be built on other platforms, they cannot - be tested. An arm64 runner is required. To silence this warning, - set `CIBW_TEST_SKIP: "*-win_arm64"`. - """ - ) - ) - # skip this test - elif test_selected and build_options.test_command: - log.step("Testing wheel...") - # set up a virtual environment to install and test from, to make sure - # there are no dependencies that were pulled in at build time. - venv_dir = identifier_tmp_dir / "venv-test" - virtualenv_env = virtualenv( - config.version, - base_python, - venv_dir, - None, - use_uv=use_uv, - env=env, - pip_version=pip_version, - ) + # test the wheel + if build_options.test_requires: + call(*pip, "install", *build_options.test_requires, env=virtualenv_env) + + # run the tests from a temp dir, with an absolute path in the command + # (this ensures that Python runs the tests against the installed wheel + # and not the repo code) + test_cwd = self.tmp_dir / "test_cwd" + runner.prepare_test_cwd(test_cwd, build_options.test_sources) + + test_command_prepared = prepare_command( + build_options.test_command, + project=Path.cwd(), + package=build_options.package_dir.resolve(), + wheel=repaired_wheel, + ) + shell(test_command_prepared, cwd=test_cwd, env=virtualenv_env) - virtualenv_env = build_options.test_environment.as_dictionary( - prev_environment=virtualenv_env - ) - # check that we are using the Python from the virtual environment - call("where", "python", env=virtualenv_env) - - if build_options.before_test: - before_test_prepared = prepare_command( - build_options.before_test, - project=".", - package=build_options.package_dir, - ) - shell(before_test_prepared, env=virtualenv_env) - - pip: Sequence[Path | str] - if use_uv: - assert uv_path is not None - pip = [uv_path, "pip"] - else: - pip = ["pip"] - - # install the wheel - call( - *pip, - "install", - str(repaired_wheel) + build_options.test_extras, - env=virtualenv_env, - ) +def build(options: Options, tmp_path: Path) -> None: + python_configurations = get_python_configurations( + options.globals.build_selector, options.globals.architectures + ) - # test the wheel - if build_options.test_requires: - call(*pip, "install", *build_options.test_requires, env=virtualenv_env) - - # run the tests from a temp dir, with an absolute path in the command - # (this ensures that Python runs the tests against the installed wheel - # and not the repo code) - test_cwd = identifier_tmp_dir / "test_cwd" - test_cwd.mkdir() - - if build_options.test_sources: - copy_test_sources( - build_options.test_sources, - Path.cwd(), - test_cwd, - ) - else: - # Use the test_fail.py file to raise a nice error if the user - # tries to run tests in the cwd - (test_cwd / "test_fail.py").write_text(resources.TEST_FAIL_CWD_FILE.read_text()) - - test_command_prepared = prepare_command( - build_options.test_command, - project=Path.cwd(), - package=options.globals.package_dir.resolve(), - wheel=repaired_wheel, + if not python_configurations: + return + + with runner.fatal_on_called_process_error(): + runner.run_before_all(options, python_configurations) + runner.run_builds( + [ + WindowsBuilder( + config=config, + build_options=options.build_options(config.identifier), + tmp_dir=tmp_path / config.identifier, + session_tmp_dir=tmp_path, ) - shell(test_command_prepared, cwd=test_cwd, env=virtualenv_env) - - # we're all done here; move it to output (remove if already exists) - output_wheel = None - if compatible_wheel is None: - output_wheel = build_options.output_dir.joinpath(repaired_wheel.name) - moved_wheel = move_file(repaired_wheel, output_wheel) - if moved_wheel != output_wheel.resolve(): - log.warning( - f"{repaired_wheel} was moved to {moved_wheel} instead of {output_wheel}" - ) - built_wheels.append(output_wheel) - - # clean up - # (we ignore errors because occasionally Windows fails to unlink a file and we - # don't want to abort a build because of that) - shutil.rmtree(identifier_tmp_dir, ignore_errors=True) - - log.build_end(output_wheel) - except subprocess.CalledProcessError as error: - msg = f"Command {error.cmd} failed with code {error.returncode}. {error.stdout or ''}" - raise errors.FatalError(msg) from error + for config in python_configurations + ] + ) From dcd5cac31550c8909adfaafaa06aac057f865ba0 Mon Sep 17 00:00:00 2001 From: Henry Schreiner Date: Tue, 9 Jun 2026 17:21:17 -0400 Subject: [PATCH 5/9] refactor: port ios to the shared build runner The per-wheel loop becomes an IOSBuilder; the testbed-based test flow, the non-simulator/non-native skip steps, and the python-module test command validation stay inside test_wheel(). Behavior changes: - a test-suite failure now raises FatalError instead of calling sys.exit(1) directly (same exit code, consistent error reporting) - a repair command producing several wheels is now an error, as on Linux - the compatible-wheel reuse message now matches the other platforms - the per-identifier temp dir is removed with ignore_errors Assisted-by: ClaudeCode:claude-fable-5 --- cibuildwheel/platforms/ios.py | 594 +++++++++++++++------------------- 1 file changed, 268 insertions(+), 326 deletions(-) diff --git a/cibuildwheel/platforms/ios.py b/cibuildwheel/platforms/ios.py index 72ab35952..256990d84 100644 --- a/cibuildwheel/platforms/ios.py +++ b/cibuildwheel/platforms/ios.py @@ -1,7 +1,6 @@ from __future__ import annotations __lazy_modules__ = { - "cibuildwheel.audit", "cibuildwheel.frontend", "cibuildwheel.logger", "cibuildwheel.platforms.macos", @@ -9,7 +8,6 @@ "cibuildwheel.util.cmd", "cibuildwheel.util.file", "cibuildwheel.util.helpers", - "cibuildwheel.util.packaging", "cibuildwheel.venv", "filelock", "packaging", @@ -38,13 +36,13 @@ from packaging.version import Version from cibuildwheel import errors -from cibuildwheel.audit import run_audit from cibuildwheel.frontend import ( BuildFrontendName, get_build_frontend_extra_flags, prepare_config_settings, ) from cibuildwheel.logger import log +from cibuildwheel.platforms import runner from cibuildwheel.platforms.macos import install_cpython as install_build_cpython from cibuildwheel.util import resources from cibuildwheel.util.cmd import call, shell, split_command @@ -52,11 +50,9 @@ CIBW_CACHE_PATH, copy_test_sources, download, - move_file, remove_on_error, ) from cibuildwheel.util.helpers import prepare_command, unwrap_preserving_paragraphs -from cibuildwheel.util.packaging import find_compatible_wheel from cibuildwheel.venv import constraint_flags, virtualenv TYPE_CHECKING = False @@ -65,7 +61,7 @@ from cibuildwheel.architecture import Architecture from cibuildwheel.environment import ParsedEnvironment - from cibuildwheel.options import Options + from cibuildwheel.options import BuildOptions, Options from cibuildwheel.selector import BuildSelector @@ -472,345 +468,291 @@ def setup_python( return target_install_path, env -def build(options: Options, tmp_path: Path) -> None: - if sys.platform != "darwin": - msg = "iOS binaries can only be built on macOS" - raise errors.FatalError(msg) +class IOSBuilder(runner.HostBuilder): + def __init__( + self, + *, + config: PythonConfiguration, + build_options: BuildOptions, + tmp_dir: Path, + session_tmp_dir: Path, + ) -> None: + super().__init__( + identifier=config.identifier, + build_options=build_options, + tmp_dir=tmp_dir, + session_tmp_dir=session_tmp_dir, + ) + self.config = config - python_configurations = get_python_configurations( - build_selector=options.globals.build_selector, - architectures=options.globals.architectures, - ) + def setup(self) -> None: + build_options = self.build_options + build_frontend = build_options.build_frontend - if not python_configurations: - return + # uv doesn't support iOS + # Not using set because mypy can't narrow it + if build_frontend.name == "build[uv]" or build_frontend.name == "uv": # noqa: PLR1714 + msg = "uv doesn't support iOS" + raise errors.FatalError(msg) - try: - before_all_options_identifier = python_configurations[0].identifier - before_all_options = options.build_options(before_all_options_identifier) + self.tmp_dir.mkdir() - if before_all_options.before_all: - log.step("Running before_all...") - env = before_all_options.environment.as_dictionary(prev_environment=os.environ) - env.setdefault("IPHONEOS_DEPLOYMENT_TARGET", "13.0") - before_all_prepared = prepare_command( - before_all_options.before_all, - project=".", - package=before_all_options.package_dir, - ) - shell(before_all_prepared, env=env) + constraints_path = build_options.dependency_constraints.get_for_python_version( + version=self.config.version, tmp_dir=self.tmp_dir + ) - built_wheels: list[Path] = [] + self.target_install_path, env = setup_python( + self.tmp_dir / "build", + python_configuration=self.config, + dependency_constraint=constraints_path, + environment=build_options.environment, + build_frontend=build_frontend.name, + xbuild_tools=build_options.xbuild_tools, + ) + env["CIBUILDWHEEL_BUILD_IDENTIFIER"] = self.identifier + self.env = env + + def build_wheel(self) -> Path: + build_options = self.build_options + build_frontend = build_options.build_frontend + + self.built_wheel_dir.mkdir() + + extra_flags = get_build_frontend_extra_flags( + build_frontend, + build_options.build_verbosity, + prepare_config_settings( + build_options.config_settings, + project=Path.cwd(), + package=build_options.package_dir, + ), + ) - for config in python_configurations: - build_options = options.build_options(config.identifier) - build_frontend = build_options.build_frontend - # uv doesn't support iOS - # Not using set because mypy can't narrow it - if build_frontend.name == "build[uv]" or build_frontend.name == "uv": # noqa: PLR1714 - msg = "uv doesn't support iOS" + match build_frontend.name: + case "pip": + # Path.resolve() is needed. Without it pip wheel may try to + # fetch package from pypi.org. See + # https://github.com/pypa/cibuildwheel/pull/369 + call( + "python", + "-m", + "pip", + "wheel", + build_options.package_dir.resolve(), + f"--wheel-dir={self.built_wheel_dir}", + "--no-deps", + *extra_flags, + env=self.env, + ) + case "build": + call( + "python", + "-m", + "build", + build_options.package_dir, + "--wheel", + f"--outdir={self.built_wheel_dir}", + *extra_flags, + env=self.env, + ) + case _: + # the other frontends were rejected in setup() or option validation + msg = f"Unsupported build frontend {build_frontend.name!r} for iOS" raise errors.FatalError(msg) - log.build_start(config.identifier) + try: + return next(self.built_wheel_dir.glob("*.whl")) + except StopIteration: + raise errors.BuildProducedNoWheelError() from None + + def test_wheel(self, repaired_wheel: Path) -> None: + build_options = self.build_options + assert build_options.test_command is not None + + if not self.config.is_simulator: + log.step("Skipping tests on non-simulator SDK") + return + if self.config.arch != os.uname().machine: + log.step("Skipping tests on non-native simulator architecture") + return - identifier_tmp_dir = tmp_path / config.identifier - identifier_tmp_dir.mkdir() - built_wheel_dir = identifier_tmp_dir / "built_wheel" - repaired_wheel_dir = identifier_tmp_dir / "repaired_wheel" + test_env = build_options.test_environment.as_dictionary(prev_environment=self.env) - constraints_path = build_options.dependency_constraints.get_for_python_version( - version=config.version, tmp_dir=identifier_tmp_dir + if build_options.before_test: + before_test_prepared = prepare_command( + build_options.before_test, + project=".", + package=build_options.package_dir, ) + shell(before_test_prepared, env=test_env) + + log.step("Setting up test harness...") + # Clone the testbed project into the build directory + testbed_path = self.tmp_dir / "testbed" + call( + "python", + self.target_install_path / "testbed", + "clone", + testbed_path, + env=test_env, + ) + + testbed_app_path = testbed_path / "iOSTestbed" / "app" - target_install_path, env = setup_python( - identifier_tmp_dir / "build", - python_configuration=config, - dependency_constraint=constraints_path, - environment=build_options.environment, - build_frontend=build_frontend.name, - xbuild_tools=build_options.xbuild_tools, + # Copy the test sources to the testbed app + if build_options.test_sources: + copy_test_sources( + build_options.test_sources, + Path.cwd(), + testbed_app_path, ) - env["CIBUILDWHEEL_BUILD_IDENTIFIER"] = config.identifier - - compatible_wheel = find_compatible_wheel(built_wheels, config.identifier) - if compatible_wheel: - log.step_end() - print( - f"\nFound previously built wheel {compatible_wheel.name} " - f"that is compatible with {config.identifier}. " - "Skipping build step..." - ) - test_wheel = compatible_wheel - else: - if build_options.before_build: - log.step("Running before_build...") - before_build_prepared = prepare_command( - build_options.before_build, - project=".", - package=build_options.package_dir, - ) - shell(before_build_prepared, env=env) - - log.step("Building wheel...") - built_wheel_dir.mkdir() - - extra_flags = get_build_frontend_extra_flags( - build_frontend, - build_options.build_verbosity, - prepare_config_settings( - build_options.config_settings, - project=Path.cwd(), - package=build_options.package_dir, - ), - ) + else: + (testbed_app_path / "test_fail.py").write_text(resources.TEST_FAIL_CWD_FILE.read_text()) - match build_frontend.name: - case "pip": - # Path.resolve() is needed. Without it pip wheel may try to - # fetch package from pypi.org. See - # https://github.com/pypa/cibuildwheel/pull/369 - call( - "python", - "-m", - "pip", - "wheel", - build_options.package_dir.resolve(), - f"--wheel-dir={built_wheel_dir}", - "--no-deps", - *extra_flags, - env=env, - ) - case "build": - call( - "python", - "-m", - "build", - build_options.package_dir, - "--wheel", - f"--outdir={built_wheel_dir}", - *extra_flags, - env=env, - ) - case "pyodide-build": - msg = "The 'pyodide-build' build frontend is not supported on this platform" - raise errors.FatalError(msg) - case _: - assert_never(build_frontend) - - try: - built_wheel = next(built_wheel_dir.glob("*.whl")) - except StopIteration: - raise errors.BuildProducedNoWheelError() from None - - if built_wheel.name.endswith("none-any.whl"): - raise errors.NonPlatformWheelError() - - repaired_wheel_dir.mkdir() - if build_options.repair_command: - log.step("Repairing wheel...") - - repair_command_prepared = prepare_command( - build_options.repair_command, - wheel=built_wheel, - dest_dir=repaired_wheel_dir, - package=build_options.package_dir, - project=".", - ) - shell(repair_command_prepared, env=env) - else: - shutil.move(str(built_wheel), repaired_wheel_dir) - - try: - repaired_wheel = next(repaired_wheel_dir.glob("*.whl")) - except StopIteration: - raise errors.RepairStepProducedNoWheelError() from None - - if repaired_wheel.name in {wheel.name for wheel in built_wheels}: - raise errors.AlreadyBuiltWheelError(repaired_wheel.name) - - log.step_end() - - run_audit(tmp_dir=tmp_path, build_options=build_options, wheel=repaired_wheel) - - test_wheel = repaired_wheel - - if build_options.test_command and build_options.test_selector(config.identifier): - if not config.is_simulator: - log.step("Skipping tests on non-simulator SDK") - elif config.arch != os.uname().machine: - log.step("Skipping tests on non-native simulator architecture") - else: - test_env = build_options.test_environment.as_dictionary(prev_environment=env) - - if build_options.before_test: - before_test_prepared = prepare_command( - build_options.before_test, - project=".", - package=build_options.package_dir, - ) - shell(before_test_prepared, env=test_env) - - log.step("Setting up test harness...") - # Clone the testbed project into the build directory - testbed_path = identifier_tmp_dir / "testbed" - call( - "python", - target_install_path / "testbed", - "clone", - testbed_path, - env=test_env, - ) - - testbed_app_path = testbed_path / "iOSTestbed" / "app" - - # Copy the test sources to the testbed app - if build_options.test_sources: - copy_test_sources( - build_options.test_sources, - Path.cwd(), - testbed_app_path, - ) - else: - (testbed_app_path / "test_fail.py").write_text( - resources.TEST_FAIL_CWD_FILE.read_text() - ) + log.step("Installing test requirements...") + # Install the compiled wheel (with any test extras), plus + # the test requirements. Use the --platform tag to force + # the installation of iOS wheels; this requires the use of + # --only-binary=:all: + ios_version = test_env["IPHONEOS_DEPLOYMENT_TARGET"] + platform_tag = f"ios_{ios_version.replace('.', '_')}_{self.config.arch}_{self.config.sdk}" - log.step("Installing test requirements...") - # Install the compiled wheel (with any test extras), plus - # the test requirements. Use the --platform tag to force - # the installation of iOS wheels; this requires the use of - # --only-binary=:all: - ios_version = test_env["IPHONEOS_DEPLOYMENT_TARGET"] - platform_tag = f"ios_{ios_version.replace('.', '_')}_{config.arch}_{config.sdk}" - - call( - "python", - "-m", - "pip", - "install", - "--only-binary=:all:", - "--platform", - platform_tag, - "--target", - testbed_path / "iOSTestbed" / "app_packages", - f"{test_wheel}{build_options.test_extras}", - *build_options.test_requires, - env=test_env, - ) - - log.step("Running test suite...") - - # iOS doesn't support placeholders in the test command, - # because the source dir isn't visible on the simulator. - if ( - "{project}" in build_options.test_command - or "{package}" in build_options.test_command - ): - msg = unwrap_preserving_paragraphs( - f""" - iOS tests configured with a test command that uses the "{{project}}" or - "{{package}}" placeholder. iOS tests cannot use placeholders, because the - source directory is not visible on the simulator. + call( + "python", + "-m", + "pip", + "install", + "--only-binary=:all:", + "--platform", + platform_tag, + "--target", + testbed_path / "iOSTestbed" / "app_packages", + f"{repaired_wheel}{build_options.test_extras}", + *build_options.test_requires, + env=test_env, + ) + + log.step("Running test suite...") + + # iOS doesn't support placeholders in the test command, + # because the source dir isn't visible on the simulator. + if "{project}" in build_options.test_command or "{package}" in build_options.test_command: + msg = unwrap_preserving_paragraphs( + f""" + iOS tests configured with a test command that uses the "{{project}}" or + "{{package}}" placeholder. iOS tests cannot use placeholders, because the + source directory is not visible on the simulator. + + In addition, iOS tests must run as a Python module, so the test command + must begin with 'python -m'. + + Test command: {build_options.test_command!r} + """ + ) + raise errors.FatalError(msg) - In addition, iOS tests must run as a Python module, so the test command - must begin with 'python -m'. + test_command_list = shlex.split(build_options.test_command) + try: + for test_command_parts in split_command(test_command_list): + match test_command_parts: + case ["python", "-m", *rest]: + final_command = rest + case ["pytest", *rest]: + # pytest works exactly the same as a module, so we + # can just run it as a module. + msg = unwrap_preserving_paragraphs(f""" + iOS tests configured with a test command which doesn't start + with 'python -m'. iOS tests must execute python modules - other + entrypoints are not supported. + + cibuildwheel will try to execute it as if it started with + 'python -m'. If this works, all you need to do is add that to + your test command. Test command: {build_options.test_command!r} + """) + log.warning(msg) + final_command = ["pytest", *rest] + case _: + msg = unwrap_preserving_paragraphs( + f""" + iOS tests configured with a test command which doesn't start + with 'python -m'. iOS tests must execute python modules - other + entrypoints are not supported. + + Test command: {build_options.test_command!r} """ ) raise errors.FatalError(msg) - test_command_list = shlex.split(build_options.test_command) - try: - for test_command_parts in split_command(test_command_list): - match test_command_parts: - case ["python", "-m", *rest]: - final_command = rest - case ["pytest", *rest]: - # pytest works exactly the same as a module, so we - # can just run it as a module. - msg = unwrap_preserving_paragraphs(f""" - iOS tests configured with a test command which doesn't start - with 'python -m'. iOS tests must execute python modules - other - entrypoints are not supported. - - cibuildwheel will try to execute it as if it started with - 'python -m'. If this works, all you need to do is add that to - your test command. - - Test command: {build_options.test_command!r} - """) - log.warning(msg) - final_command = ["pytest", *rest] - case _: - msg = unwrap_preserving_paragraphs( - f""" - iOS tests configured with a test command which doesn't start - with 'python -m'. iOS tests must execute python modules - other - entrypoints are not supported. - - Test command: {build_options.test_command!r} - """ - ) - raise errors.FatalError(msg) - - test_runtime_args = build_options.test_runtime.args - - # 2025-10: The GitHub Actions macos-15 runner has a known issue where - # the default simulator won't start due to a disk performance issue; - # see https://github.com/actions/runner-images/issues/12777 for details. - # In the meantime, if it looks like we're running on a GitHub Actions - # macos-15 runner, use a simulator that is known to work, unless the - # user explicitly specifies a simulator. - os_version, _, arch = platform.mac_ver() - if ( - "GITHUB_ACTIONS" in os.environ - and os_version.startswith("15.") - and arch == "arm64" - and not any( - arg.startswith("--simulator") for arg in test_runtime_args - ) - ): - test_runtime_args = [ - "--simulator", - "iPhone 16e,OS=18.5", - *test_runtime_args, - ] - - call( - "python", - testbed_path, - "run", - *(["--verbose"] if build_options.build_verbosity > 0 else []), - *test_runtime_args, - "--", - *final_command, - env=test_env, - ) - except subprocess.CalledProcessError: - # catches the first test command failure in the loop, - # implementing short-circuiting - log.step_end(success=False) - log.error(f"Test suite failed on {config.identifier}") - sys.exit(1) - - log.step_end() - - # We're all done here; move it to output (overwrite existing) - output_wheel: Path | None = None - if compatible_wheel is None: - output_wheel = build_options.output_dir.joinpath(repaired_wheel.name) - moved_wheel = move_file(repaired_wheel, output_wheel) - if moved_wheel != output_wheel.resolve(): - log.warning( - f"{repaired_wheel} was moved to {moved_wheel} instead of {output_wheel}" - ) - built_wheels.append(output_wheel) - - # Clean up - shutil.rmtree(identifier_tmp_dir) - - log.build_end(output_wheel) - except subprocess.CalledProcessError as error: - msg = f"Command {error.cmd} failed with code {error.returncode}. {error.stdout or ''}" - raise errors.FatalError(msg) from error + test_runtime_args = build_options.test_runtime.args + + # 2025-10: The GitHub Actions macos-15 runner has a known issue where + # the default simulator won't start due to a disk performance issue; + # see https://github.com/actions/runner-images/issues/12777 for details. + # In the meantime, if it looks like we're running on a GitHub Actions + # macos-15 runner, use a simulator that is known to work, unless the + # user explicitly specifies a simulator. + os_version, _, arch = platform.mac_ver() + if ( + "GITHUB_ACTIONS" in os.environ + and os_version.startswith("15.") + and arch == "arm64" + and not any(arg.startswith("--simulator") for arg in test_runtime_args) + ): + test_runtime_args = [ + "--simulator", + "iPhone 16e,OS=18.5", + *test_runtime_args, + ] + + call( + "python", + testbed_path, + "run", + *(["--verbose"] if build_options.build_verbosity > 0 else []), + *test_runtime_args, + "--", + *final_command, + env=test_env, + ) + except subprocess.CalledProcessError: + # catches the first test command failure in the loop, + # implementing short-circuiting + log.step_end(success=False) + msg = f"Test suite failed on {self.identifier}" + raise errors.FatalError(msg) from None + + log.step_end() + + +def build(options: Options, tmp_path: Path) -> None: + if sys.platform != "darwin": + msg = "iOS binaries can only be built on macOS" + raise errors.FatalError(msg) + + python_configurations = get_python_configurations( + build_selector=options.globals.build_selector, + architectures=options.globals.architectures, + ) + + if not python_configurations: + return + + with runner.fatal_on_called_process_error(): + runner.run_before_all( + options, + python_configurations, + env_defaults={"IPHONEOS_DEPLOYMENT_TARGET": "13.0"}, + ) + runner.run_builds( + [ + IOSBuilder( + config=config, + build_options=options.build_options(config.identifier), + tmp_dir=tmp_path / config.identifier, + session_tmp_dir=tmp_path, + ) + for config in python_configurations + ] + ) From 81aa3e49f8d0102b588a0cadd80472f1d231b887 Mon Sep 17 00:00:00 2001 From: Henry Schreiner Date: Tue, 9 Jun 2026 17:24:32 -0400 Subject: [PATCH 6/9] refactor: port android to the shared build runner The BuildState dataclass and its step functions fold into an AndroidBuilder; setup_target_python/setup_env and the other environment helpers are unchanged. Behavior changes: - test-environment (CIBW_TEST_ENVIRONMENT_ANDROID) is now applied to the test steps (before-test, wheel install, testbed invocation); it was documented but previously ignored on Android - a wheel name colliding with an earlier build now raises AlreadyBuiltWheelError, like every other platform - the redundant post-repair none-any.whl check is dropped (the built wheel is still checked, like every other platform) - the compatible-wheel reuse message now matches the other platforms, and the moved-elsewhere warning from move_file is now emitted - a missing uv now raises FatalError instead of AssertionError Assisted-by: ClaudeCode:claude-fable-5 --- cibuildwheel/platforms/android.py | 565 ++++++++++++++---------------- 1 file changed, 256 insertions(+), 309 deletions(-) diff --git a/cibuildwheel/platforms/android.py b/cibuildwheel/platforms/android.py index de0c12b78..a208f49ea 100644 --- a/cibuildwheel/platforms/android.py +++ b/cibuildwheel/platforms/android.py @@ -4,13 +4,11 @@ "build", "build.env", "cibuildwheel.architecture", - "cibuildwheel.audit", "cibuildwheel.frontend", "cibuildwheel.logger", "cibuildwheel.util.cmd", "cibuildwheel.util.file", "cibuildwheel.util.helpers", - "cibuildwheel.util.packaging", "cibuildwheel.util.python_build_standalone", "cibuildwheel.venv", "filelock", @@ -23,7 +21,6 @@ "runpy", "shlex", "shutil", - "subprocess", "textwrap", "typing", } @@ -33,7 +30,6 @@ import re import shlex import shutil -import subprocess import sys from dataclasses import dataclass from pathlib import Path @@ -49,24 +45,21 @@ from cibuildwheel import errors, platforms # pylint: disable=cyclic-import from cibuildwheel.architecture import Architecture, arch_synonym -from cibuildwheel.audit import run_audit from cibuildwheel.frontend import ( get_build_frontend_extra_flags, parse_config_settings, prepare_config_settings, ) from cibuildwheel.logger import log +from cibuildwheel.platforms import runner from cibuildwheel.util import resources from cibuildwheel.util.cmd import call, shell from cibuildwheel.util.file import ( CIBW_CACHE_PATH, - copy_test_sources, download, - move_file, remove_on_error, ) from cibuildwheel.util.helpers import prepare_command -from cibuildwheel.util.packaging import find_compatible_wheel from cibuildwheel.util.python_build_standalone import create_python_build_standalone_environment from cibuildwheel.venv import constraint_flags, find_uv, virtualenv @@ -129,88 +122,6 @@ def shell_prepared(command: str, *, build_options: BuildOptions, env: dict[str, ) -def before_all(options: Options, python_configurations: list[PythonConfiguration]) -> None: - before_all_options = options.build_options(python_configurations[0].identifier) - if before_all_options.before_all: - log.step("Running before_all...") - shell_prepared( - before_all_options.before_all, - build_options=before_all_options, - env=before_all_options.environment.as_dictionary(os.environ), - ) - - -@dataclass(frozen=True) -class BuildState: - config: PythonConfiguration - options: BuildOptions - build_path: Path - python_dir: Path - build_env: dict[str, str] - android_env: dict[str, str] - - -def build(options: Options, tmp_path: Path) -> None: - if "ANDROID_HOME" not in os.environ: - msg = ( - "ANDROID_HOME environment variable is not set. For instructions, see " - "https://cibuildwheel.pypa.io/en/stable/platforms/#android" - ) - raise errors.FatalError(msg) - - configs = get_python_configurations( - options.globals.build_selector, options.globals.architectures - ) - if not configs: - return - - try: - before_all(options, configs) - - built_wheels: list[Path] = [] - for config in configs: - log.build_start(config.identifier) - build_options = options.build_options(config.identifier) - build_path = tmp_path / config.identifier - build_path.mkdir() - python_dir = setup_target_python(config, build_path) - build_env, android_env = setup_env(config, build_options, build_path, python_dir) - - state = BuildState( - config, build_options, build_path, python_dir, build_env, android_env - ) - setup_xbuild_files(state) - - compatible_wheel = find_compatible_wheel(built_wheels, config.identifier) - if compatible_wheel: - print( - f"\nFound previously built wheel {compatible_wheel.name} that is " - f"compatible with {config.identifier}. Skipping build step..." - ) - repaired_wheel = compatible_wheel - else: - before_build(state) - built_wheel = build_wheel(state) - repaired_wheel = repair_wheel(state, built_wheel) - run_audit(tmp_dir=tmp_path, build_options=build_options, wheel=repaired_wheel) - - test_wheel(state, repaired_wheel) - - output_wheel: Path | None = None - if compatible_wheel is None: - output_wheel = move_file( - repaired_wheel, build_options.output_dir / repaired_wheel.name - ) - built_wheels.append(output_wheel) - - shutil.rmtree(build_path) - log.build_end(output_wheel) - - except subprocess.CalledProcessError as error: - msg = f"Command {error.cmd} failed with code {error.returncode}. {error.stdout or ''}" - raise errors.FatalError(msg) from error - - def setup_target_python(config: PythonConfiguration, build_path: Path) -> Path: log.step("Installing target Python...") python_tgz = CIBW_CACHE_PATH / config.url.rpartition("/")[-1] @@ -516,41 +427,6 @@ def setup_fortran(env: dict[str, str]) -> None: env["FC"] = str(shim_out) -def setup_xbuild_files(state: BuildState) -> None: - _, pip = find_pip(state.options) - xbf_dir = state.build_path / "xbuild_files" - xbf_dir.mkdir() - - for requirement in call(*pip, "freeze", env=state.build_env, capture_stdout=True).splitlines(): - name, _, _ = requirement.strip().partition("==") - xbuild_files = state.options.xbuild_files.get(canonicalize_name(name), []) - if xbuild_files: - log.step(f"Installing xbuild-files for {name}...") - pip_install_android(state, xbf_dir, "--no-deps", requirement) - for xbf in xbuild_files: - if (xbf_dir / xbf).exists(): - shutil.copy( - xbf_dir / xbf, - find_site_packages(state.build_env) / xbf, - ) - else: - log.warning(f"{xbf_dir / xbf} does not exist") - - -def pip_install_android(state: BuildState, target: Path, *args: PathOrStr) -> None: - use_uv, pip = find_pip(state.options) - call( - *pip, - "install", - "--only-binary=:all:", - *(["--python-platform", android_triplet(state.config.identifier)] if use_uv else []), - "--target", - target, - *args, - env=state.android_env, - ) - - def find_site_packages(env: dict[str, str]) -> Path: return glob1(Path(env["VIRTUAL_ENV"]), "lib/python*/site-packages") @@ -568,216 +444,287 @@ def find_pip(build_options: BuildOptions) -> tuple[bool, list[str]]: uv_path = find_uv() if use_uv and uv_path is None: msg = "uv not found" - raise AssertionError(msg) + raise errors.FatalError(msg) pip = ["pip"] if not use_uv else [str(uv_path), "pip"] return use_uv, pip -def before_build(state: BuildState) -> None: - if state.options.before_build: - log.step("Running before_build...") - shell_prepared( - state.options.before_build, - build_options=state.options, - env=state.android_env, +class AndroidBuilder(runner.HostBuilder): + def __init__( + self, + *, + config: PythonConfiguration, + build_options: BuildOptions, + tmp_dir: Path, + session_tmp_dir: Path, + ) -> None: + super().__init__( + identifier=config.identifier, + build_options=build_options, + tmp_dir=tmp_dir, + session_tmp_dir=session_tmp_dir, + ) + self.config = config + + def setup(self) -> None: + self.tmp_dir.mkdir() + self.python_dir = setup_target_python(self.config, self.tmp_dir) + self.build_env, self.android_env = setup_env( + self.config, self.build_options, self.tmp_dir, self.python_dir + ) + self.setup_xbuild_files() + + def setup_xbuild_files(self) -> None: + _, pip = find_pip(self.build_options) + xbf_dir = self.tmp_dir / "xbuild_files" + xbf_dir.mkdir() + + for requirement in call( + *pip, "freeze", env=self.build_env, capture_stdout=True + ).splitlines(): + name, _, _ = requirement.strip().partition("==") + xbuild_files = self.build_options.xbuild_files.get(canonicalize_name(name), []) + if xbuild_files: + log.step(f"Installing xbuild-files for {name}...") + self.pip_install_android(self.android_env, xbf_dir, "--no-deps", requirement) + for xbf in xbuild_files: + if (xbf_dir / xbf).exists(): + shutil.copy( + xbf_dir / xbf, + find_site_packages(self.build_env) / xbf, + ) + else: + log.warning(f"{xbf_dir / xbf} does not exist") + + def pip_install_android(self, env: dict[str, str], target: Path, *args: PathOrStr) -> None: + use_uv, pip = find_pip(self.build_options) + call( + *pip, + "install", + "--only-binary=:all:", + *(["--python-platform", android_triplet(self.identifier)] if use_uv else []), + "--target", + target, + *args, + env=env, ) + def before_build(self) -> None: + assert self.build_options.before_build is not None + shell_prepared( + self.build_options.before_build, + build_options=self.build_options, + env=self.android_env, + ) -def build_wheel(state: BuildState) -> Path: - log.step("Building wheel...") - built_wheel_dir = state.build_path / "built_wheel" - match state.options.build_frontend.name: - case "build" | "build[uv]": - call( - "python", - "-m", - "build", - state.options.package_dir, - "--wheel", - "--no-isolation", - "--skip-dependency-check", - f"--outdir={built_wheel_dir}", - *get_build_frontend_extra_flags( - state.options.build_frontend, - state.options.build_verbosity, - prepare_config_settings( - state.options.config_settings, - project=Path.cwd(), - package=state.options.package_dir, + def build_wheel(self) -> Path: + build_options = self.build_options + match build_options.build_frontend.name: + case "build" | "build[uv]": + call( + "python", + "-m", + "build", + build_options.package_dir, + "--wheel", + "--no-isolation", + "--skip-dependency-check", + f"--outdir={self.built_wheel_dir}", + *get_build_frontend_extra_flags( + build_options.build_frontend, + build_options.build_verbosity, + prepare_config_settings( + build_options.config_settings, + project=Path.cwd(), + package=build_options.package_dir, + ), ), - ), - env=state.android_env, - ) - case "uv": - uv_path = find_uv() - assert uv_path is not None - call( - uv_path, - "build", - state.options.package_dir, - "--wheel", - "--no-build-isolation", - f"--out-dir={built_wheel_dir}", - *get_build_frontend_extra_flags( - state.options.build_frontend, - state.options.build_verbosity, - prepare_config_settings( - state.options.config_settings, - project=Path.cwd(), - package=state.options.package_dir, + env=self.android_env, + ) + case "uv": + uv_path = find_uv() + assert uv_path is not None + call( + uv_path, + "build", + build_options.package_dir, + "--wheel", + "--no-build-isolation", + f"--out-dir={self.built_wheel_dir}", + *get_build_frontend_extra_flags( + build_options.build_frontend, + build_options.build_verbosity, + prepare_config_settings( + build_options.config_settings, + project=Path.cwd(), + package=build_options.package_dir, + ), ), + env=self.android_env, + ) + case x: + msg = f"Android requires the build frontend to be 'build' or 'uv', not {x!r}" + raise errors.FatalError(msg) + + return glob1(self.built_wheel_dir, "*.whl") + + def repair_wheel(self, built_wheel: Path) -> list[Path]: + self.repaired_wheel_dir.mkdir() + + if self.build_options.repair_command: + # Tell auditwheel the locations of compiler libraries. + toolchain = Path(self.android_env["CC"]).parent.parent + triplet = android_triplet(self.identifier) + ldpaths = ":".join( + str(glob1(toolchain, pattern)) + for pattern in [ + f"lib/clang/*/lib/linux/{triplet.split('-')[0]}", # libomp + f"sysroot/usr/lib/{triplet}", # libc++_shared + ] + ) + shell( + prepare_command( + self.build_options.repair_command, + ldpaths=ldpaths, + wheel=built_wheel, + dest_dir=self.repaired_wheel_dir, + package=self.build_options.package_dir, + project=".", ), - env=state.android_env, + env=self.build_env, ) - case x: - msg = f"Android requires the build frontend to be 'build' or 'uv', not {x!r}" - raise errors.FatalError(msg) + else: + shutil.move(built_wheel, self.repaired_wheel_dir) - built_wheel = glob1(built_wheel_dir, "*.whl") - if built_wheel.name.endswith("none-any.whl"): - raise errors.NonPlatformWheelError() - return built_wheel - - -def repair_wheel(state: BuildState, built_wheel: Path) -> Path: - log.step("Repairing wheel...") - repaired_wheel_dir = state.build_path / "repaired_wheel" - repaired_wheel_dir.mkdir() - - if state.options.repair_command: - # Tell auditwheel the locations of compiler libraries. - toolchain = Path(state.android_env["CC"]).parent.parent - triplet = android_triplet(state.config.identifier) - ldpaths = ":".join( - str(glob1(toolchain, pattern)) - for pattern in [ - f"lib/clang/*/lib/linux/{triplet.split('-')[0]}", # libomp - f"sysroot/usr/lib/{triplet}", # libc++_shared - ] - ) - shell( - prepare_command( - state.options.repair_command, - ldpaths=ldpaths, - wheel=built_wheel, - dest_dir=repaired_wheel_dir, - package=state.options.package_dir, - project=".", - ), - env=state.build_env, + return list(self.repaired_wheel_dir.glob("*.whl")) + + def test_wheel(self, repaired_wheel: Path) -> None: + build_options = self.build_options + test_command = build_options.test_command + assert test_command is not None + + log.step("Testing wheel...") + native_arch = arch_synonym(platform.machine(), platforms.native_platform(), "android") + if self.config.arch != native_arch: + log.warning( + f"Skipping tests for {self.config.arch}, as the build machine only " + f"supports {native_arch}" + ) + return + + # The test steps run in the environments below; the testbed doesn't + # automatically propagate arbitrary variables into the emulator process. + android_test_env = build_options.test_environment.as_dictionary( + prev_environment=self.android_env ) - else: - shutil.move(built_wheel, repaired_wheel_dir) - - repaired_wheels = list(repaired_wheel_dir.glob("*.whl")) - if len(repaired_wheels) == 0: - raise errors.RepairStepProducedNoWheelError() - if len(repaired_wheels) != 1: - raise errors.RepairStepProducedMultipleWheelsError( - [rw.name for rw in repaired_wheels], + build_test_env = build_options.test_environment.as_dictionary( + prev_environment=self.build_env ) - repaired_wheel = repaired_wheels[0] - if repaired_wheel.name.endswith("none-any.whl"): - raise errors.NonPlatformWheelError() - return repaired_wheel + if build_options.before_test: + shell_prepared( + build_options.before_test, + build_options=build_options, + env=android_test_env, + ) + # Install the wheel and test-requires. + site_packages_dir = self.tmp_dir / "site-packages" + site_packages_dir.mkdir() + self.pip_install_android( + android_test_env, + site_packages_dir, + f"{repaired_wheel}{build_options.test_extras}", + *build_options.test_requires, + ) -def test_wheel(state: BuildState, wheel: Path) -> None: - test_command = state.options.test_command - if not (test_command and state.options.test_selector(state.config.identifier)): - return + # Copy test-sources. + cwd_dir = self.tmp_dir / "cwd" + runner.prepare_test_cwd(cwd_dir, build_options.test_sources) - log.step("Testing wheel...") - native_arch = arch_synonym(platform.machine(), platforms.native_platform(), "android") - if state.config.arch != native_arch: - log.warning( - f"Skipping tests for {state.config.arch}, as the build machine only " - f"supports {native_arch}" - ) - return + # Android doesn't support placeholders in the test command. + if any(("{" + placeholder + "}") in test_command for placeholder in ("project", "package")): + msg = ( + f"Test command {test_command!r} with a " + "'{project}' or '{package}' placeholder is not supported on Android, " + "because the source directory is not visible on the emulator." + ) + raise errors.FatalError(msg) - if state.options.before_test: - shell_prepared( - state.options.before_test, - build_options=state.options, - env=state.android_env, - ) + # Parse test-command. + test_args = shlex.split(test_command) + if test_args[0] in {"python", "python3"} and any(arg in test_args for arg in ("-c", "-m")): + # Forward the args to the CPython testbed script. We require '-c' or '-m' + # to be in the command, because without those flags, the testbed script + # will prepend '-m test', which will run Python's own test suite. + del test_args[0] + elif test_args[0] == "pytest": + # We transform some commands into the `python -m` form, but this is deprecated. + msg = ( + f"Test command {test_command!r} is not supported on Android. " + "cibuildwheel will try to execute it as if it started with 'python -m'. " + "If this works, all you need to do is add that to your test command." + ) + log.warning(msg) + test_args.insert(0, "-m") + else: + msg = ( + f"Test command {test_command!r} is not supported on Android. " + f"Command must begin with 'python' or 'python3', and contain '-m' or '-c'." + ) + raise errors.FatalError(msg) - # Install the wheel and test-requires. - site_packages_dir = state.build_path / "site-packages" - site_packages_dir.mkdir() - pip_install_android( - state, - site_packages_dir, - f"{wheel}{state.options.test_extras}", - *state.options.test_requires, - ) + # By default, run on a testbed managed emulator running the newest supported + # Android version. However, if the user specifies a --managed or --connected + # test execution argument, that argument takes precedence. + test_runtime_args = build_options.test_runtime.args - # Copy test-sources. - cwd_dir = state.build_path / "cwd" - cwd_dir.mkdir() - if state.options.test_sources: - copy_test_sources(state.options.test_sources, Path.cwd(), cwd_dir) - else: - (cwd_dir / "test_fail.py").write_text( - resources.TEST_FAIL_CWD_FILE.read_text(), + if any(arg.startswith(("--managed", "--connected")) for arg in test_runtime_args): + emulator_args = [] + else: + emulator_args = ["--managed", "maxVersion"] + + # Run the test app. + call( + self.python_dir / "android.py", + "test", + "--site-packages", + site_packages_dir, + "--cwd", + cwd_dir, + *emulator_args, + *(["-v"] if build_options.build_verbosity > 0 else []), + *test_runtime_args, + "--", + *test_args, + env=build_test_env, ) - # Android doesn't support placeholders in the test command. - if any(("{" + placeholder + "}") in test_command for placeholder in ("project", "package")): - msg = ( - f"Test command {test_command!r} with a " - "'{project}' or '{package}' placeholder is not supported on Android, " - "because the source directory is not visible on the emulator." - ) - raise errors.FatalError(msg) - # Parse test-command. - test_args = shlex.split(test_command) - if test_args[0] in {"python", "python3"} and any(arg in test_args for arg in ("-c", "-m")): - # Forward the args to the CPython testbed script. We require '-c' or '-m' - # to be in the command, because without those flags, the testbed script - # will prepend '-m test', which will run Python's own test suite. - del test_args[0] - elif test_args[0] == "pytest": - # We transform some commands into the `python -m` form, but this is deprecated. - msg = ( - f"Test command {test_command!r} is not supported on Android. " - "cibuildwheel will try to execute it as if it started with 'python -m'. " - "If this works, all you need to do is add that to your test command." - ) - log.warning(msg) - test_args.insert(0, "-m") - else: +def build(options: Options, tmp_path: Path) -> None: + if "ANDROID_HOME" not in os.environ: msg = ( - f"Test command {test_command!r} is not supported on Android. " - f"Command must begin with 'python' or 'python3', and contain '-m' or '-c'." + "ANDROID_HOME environment variable is not set. For instructions, see " + "https://cibuildwheel.pypa.io/en/stable/platforms/#android" ) raise errors.FatalError(msg) - # By default, run on a testbed managed emulator running the newest supported - # Android version. However, if the user specifies a --managed or --connected - # test execution argument, that argument takes precedence. - test_runtime_args = state.options.test_runtime.args - - if any(arg.startswith(("--managed", "--connected")) for arg in test_runtime_args): - emulator_args = [] - else: - emulator_args = ["--managed", "maxVersion"] - - # Run the test app. - call( - state.python_dir / "android.py", - "test", - "--site-packages", - site_packages_dir, - "--cwd", - cwd_dir, - *emulator_args, - *(["-v"] if state.options.build_verbosity > 0 else []), - *test_runtime_args, - "--", - *test_args, - env=state.build_env, + configs = get_python_configurations( + options.globals.build_selector, options.globals.architectures ) + if not configs: + return + + with runner.fatal_on_called_process_error(): + runner.run_before_all(options, configs) + runner.run_builds( + [ + AndroidBuilder( + config=config, + build_options=options.build_options(config.identifier), + tmp_dir=tmp_path / config.identifier, + session_tmp_dir=tmp_path, + ) + for config in configs + ] + ) From db751f89a8ea670fb6e16d44a9d1e0a25d3a864e Mon Sep 17 00:00:00 2001 From: Henry Schreiner Date: Tue, 9 Jun 2026 17:28:00 -0400 Subject: [PATCH 7/9] refactor: port linux to the shared build runner MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The per-config loop in build_in_container() becomes a LinuxBuilder (generic over PurePosixPath — wheels stay container-side, and the existing copy-out and host-side audit behavior is preserved as overridden steps). build() keeps the container grouping, engine check, and OCIContainer lifecycle; the CalledProcessError wrapper keeps its per-container scope and troubleshoot() hook via fatal_on_called_process_error. build_in_container() keeps its signature, since tests mock it. Also fixes a latent bug: the host-side per-identifier temp dir was never created, so dependency-versions with packages: would have crashed writing constraints.txt on Linux. Assisted-by: ClaudeCode:claude-fable-5 --- cibuildwheel/platforms/linux.py | 572 +++++++++++++++++--------------- 1 file changed, 303 insertions(+), 269 deletions(-) diff --git a/cibuildwheel/platforms/linux.py b/cibuildwheel/platforms/linux.py index 537d43277..b36d8a0da 100644 --- a/cibuildwheel/platforms/linux.py +++ b/cibuildwheel/platforms/linux.py @@ -7,10 +7,9 @@ "cibuildwheel.util", "cibuildwheel.util.file", "cibuildwheel.util.helpers", - "cibuildwheel.util.packaging", "collections", "contextlib", - "pathlib", + "functools", "shutil", "subprocess", "textwrap", @@ -19,6 +18,7 @@ import contextlib import dataclasses +import functools import shutil import subprocess import sys @@ -33,10 +33,10 @@ from cibuildwheel.frontend import get_build_frontend_extra_flags, prepare_config_settings from cibuildwheel.logger import log from cibuildwheel.oci_container import OCIContainer, OCIContainerEngineConfig, OCIPlatform +from cibuildwheel.platforms import runner from cibuildwheel.util import resources from cibuildwheel.util.file import copy_test_sources from cibuildwheel.util.helpers import prepare_command, unwrap -from cibuildwheel.util.packaging import find_compatible_wheel TYPE_CHECKING = False if TYPE_CHECKING: @@ -186,56 +186,47 @@ def check_all_python_exist( raise errors.FatalError(message) -def build_in_container( - *, - options: Options, - platform_configs: Sequence[PythonConfiguration], - container: OCIContainer, - container_project_path: PurePath, - container_package_dir: PurePath, - local_tmp_dir: Path, -) -> None: - container_output_dir = PurePosixPath("/output") - - check_all_python_exist(platform_configs=platform_configs, container=container) - - log.step("Copying project into container...") - container.copy_into(Path.cwd(), container_project_path) - - before_all_options_identifier = platform_configs[0].identifier - before_all_options = options.build_options(before_all_options_identifier) - - if before_all_options.before_all: - log.step("Running before_all...") - - env = container.get_environment() - env["PATH"] = f"/opt/python/cp39-cp39/bin:{env['PATH']}" - env["PIP_DISABLE_PIP_VERSION_CHECK"] = "1" - env["PIP_ROOT_USER_ACTION"] = "ignore" - env = before_all_options.environment.as_dictionary( - env, executor=container.environment_executor - ) - - before_all_prepared = prepare_command( - before_all_options.before_all, - project=container_project_path, - package=container_package_dir, - ) - container.call(["sh", "-c", before_all_prepared], env=env) - - built_wheels: list[PurePosixPath] = [] - - for config in platform_configs: - log.build_start(config.identifier) - local_identifier_tmp_dir = local_tmp_dir / config.identifier - build_options = options.build_options(config.identifier) - build_frontend = build_options.build_frontend - use_uv = build_frontend.name in {"build[uv]", "uv"} - pip = ["uv", "pip"] if use_uv else ["pip"] +class LinuxBuilder(runner.Builder[PurePosixPath]): + """A Builder that runs every step inside an OCI container. Wheel paths + are container-side paths; the wheels are copied back to the host once, + after every build in the container has finished.""" + + def __init__( + self, + *, + config: PythonConfiguration, + build_options: BuildOptions, + container: OCIContainer, + container_project_path: PurePath, + container_package_dir: PurePath, + local_tmp_dir: Path, + ) -> None: + self.identifier = config.identifier + self.build_options = build_options + self.config = config + self.container = container + self.container_project_path = container_project_path + self.container_package_dir = container_package_dir + # host-side scratch space, shared across identifiers + self.local_tmp_dir = local_tmp_dir + self.container_output_dir = PurePosixPath("/output") + container_tmp_dir = PurePosixPath("/tmp/cibuildwheel") + self.built_wheel_dir = container_tmp_dir / "built_wheel" + self.repaired_wheel_dir = container_tmp_dir / "repaired_wheel" + + def setup(self) -> None: + container = self.container + config = self.config + build_options = self.build_options + + self.use_uv = build_options.build_frontend.name in {"build[uv]", "uv"} + self.pip: list[str] = ["uv", "pip"] if self.use_uv else ["pip"] log.step("Setting up build environment...") - dependency_constraint_flags: list[PathOrStr] = [] + self.dependency_constraint_flags: list[PathOrStr] = [] + local_identifier_tmp_dir = self.local_tmp_dir / self.identifier + local_identifier_tmp_dir.mkdir(parents=True, exist_ok=True) local_constraints_file = build_options.dependency_constraints.get_for_python_version( version=config.version, tmp_dir=local_identifier_tmp_dir, @@ -243,267 +234,315 @@ def build_in_container( if local_constraints_file: container_constraints_file = PurePosixPath("/constraints.txt") container.copy_into(local_constraints_file, container_constraints_file) - dependency_constraint_flags = ["-c", container_constraints_file] + self.dependency_constraint_flags = ["-c", container_constraints_file] env = container.get_environment() env["PIP_DISABLE_PIP_VERSION_CHECK"] = "1" env["PIP_ROOT_USER_ACTION"] = "ignore" # put this config's python top of the list - python_bin = config.path / "bin" - env["PATH"] = f"{python_bin}:{env['PATH']}" + self.python_bin = config.path / "bin" + env["PATH"] = f"{self.python_bin}:{env['PATH']}" env = build_options.environment.as_dictionary(env, executor=container.environment_executor) - env["CIBUILDWHEEL_BUILD_IDENTIFIER"] = config.identifier + env["CIBUILDWHEEL_BUILD_IDENTIFIER"] = self.identifier # check config python is still on PATH which_python = container.call(["which", "python"], env=env, capture_output=True).strip() - if PurePosixPath(which_python) != python_bin / "python": + if PurePosixPath(which_python) != self.python_bin / "python": msg = "python available on PATH doesn't match our installed instance. If you have modified PATH, ensure that you don't overwrite cibuildwheel's entry or insert python above it." raise errors.FatalError(msg) container.call(["python", "-V", "-V"], env=env) - if use_uv: + if self.use_uv: which_uv = container.call(["which", "uv"], env=env, capture_output=True).strip() if not which_uv: msg = "uv not found on PATH. You must use a supported manylinux or musllinux environment with uv." raise errors.FatalError(msg) else: which_pip = container.call(["which", "pip"], env=env, capture_output=True).strip() - if PurePosixPath(which_pip) != python_bin / "pip": + if PurePosixPath(which_pip) != self.python_bin / "pip": msg = "pip available on PATH doesn't match our installed instance. If you have modified PATH, ensure that you don't overwrite cibuildwheel's entry or insert pip above it." raise errors.FatalError(msg) - compatible_wheel = find_compatible_wheel(built_wheels, config.identifier) - if compatible_wheel: - log.step_end() - print( - f"\nFound previously built wheel {compatible_wheel.name}, that's compatible with {config.identifier}. Skipping build step..." - ) - repaired_wheel = compatible_wheel - else: - if build_options.before_build: - log.step("Running before_build...") - before_build_prepared = prepare_command( - build_options.before_build, - project=container_project_path, - package=container_package_dir, - ) - before_build_env = env.copy() - if use_uv: - # On Linux, no virtualenv is created for the build environment - # (unlike macOS/Windows, where one is set up before before_build - # runs). uv requires either an active venv or an explicit Python - # target to install packages. Pin UV_PYTHON to the exact interpreter - # for this build so that `uv pip install` works in before_build - # without requiring users to pass --system. - before_build_env["UV_PYTHON"] = str(python_bin / "python") - container.call(["sh", "-c", before_build_prepared], env=before_build_env) - - log.step("Building wheel...") - - temp_dir = PurePosixPath("/tmp/cibuildwheel") - built_wheel_dir = temp_dir / "built_wheel" - container.call(["rm", "-rf", built_wheel_dir]) - container.call(["mkdir", "-p", built_wheel_dir]) - - extra_flags = get_build_frontend_extra_flags( - build_frontend, - build_options.build_verbosity, - prepare_config_settings( - build_options.config_settings, - project=container_project_path, - package=container_package_dir, - ), - ) + self.env = env - match build_frontend.name: - case "pip": - container.call( - [ - "python", - "-m", - "pip", - "wheel", - container_package_dir, - f"--wheel-dir={built_wheel_dir}", - "--no-deps", - *extra_flags, - ], - env=env, - ) - case "build" | "build[uv]": - if use_uv and "--no-isolation" not in extra_flags and "-n" not in extra_flags: - extra_flags += ["--installer=uv"] - container.call( - [ - "python", - "-m", - "build", - container_package_dir, - "--wheel", - f"--outdir={built_wheel_dir}", - *extra_flags, - ], - env=env, - ) - case "uv": - container.call( - [ - "uv", - "build", - f"--python={python_bin / 'python'}", - container_package_dir, - "--wheel", - f"--out-dir={built_wheel_dir}", - *extra_flags, - ], - env=env, - ) - case "pyodide-build": - msg = "The 'pyodide-build' build frontend is not supported on this platform" - raise errors.FatalError(msg) - case _: - assert_never(build_frontend) + def before_build(self) -> None: + build_options = self.build_options + assert build_options.before_build is not None - try: - built_wheel = container.glob(built_wheel_dir, "*.whl")[0] - except IndexError: - raise errors.BuildProducedNoWheelError() from None - - repaired_wheel_dir = temp_dir / "repaired_wheel" - container.call(["rm", "-rf", repaired_wheel_dir]) - container.call(["mkdir", "-p", repaired_wheel_dir]) - - if built_wheel.name.endswith("none-any.whl"): - raise errors.NonPlatformWheelError() - - if build_options.repair_command: - log.step("Repairing wheel...") - repair_command_prepared = prepare_command( - build_options.repair_command, - wheel=built_wheel, - dest_dir=repaired_wheel_dir, - package=container_package_dir, - project=container_project_path, + before_build_prepared = prepare_command( + build_options.before_build, + project=self.container_project_path, + package=self.container_package_dir, + ) + before_build_env = self.env.copy() + if self.use_uv: + # On Linux, no virtualenv is created for the build environment + # (unlike macOS/Windows, where one is set up before before_build + # runs). uv requires either an active venv or an explicit Python + # target to install packages. Pin UV_PYTHON to the exact interpreter + # for this build so that `uv pip install` works in before_build + # without requiring users to pass --system. + before_build_env["UV_PYTHON"] = str(self.python_bin / "python") + self.container.call(["sh", "-c", before_build_prepared], env=before_build_env) + + def build_wheel(self) -> PurePosixPath: + container = self.container + build_options = self.build_options + build_frontend = build_options.build_frontend + + container.call(["rm", "-rf", self.built_wheel_dir]) + container.call(["mkdir", "-p", self.built_wheel_dir]) + + extra_flags = get_build_frontend_extra_flags( + build_frontend, + build_options.build_verbosity, + prepare_config_settings( + build_options.config_settings, + project=self.container_project_path, + package=self.container_package_dir, + ), + ) + + match build_frontend.name: + case "pip": + container.call( + [ + "python", + "-m", + "pip", + "wheel", + self.container_package_dir, + f"--wheel-dir={self.built_wheel_dir}", + "--no-deps", + *extra_flags, + ], + env=self.env, + ) + case "build" | "build[uv]": + if self.use_uv and "--no-isolation" not in extra_flags and "-n" not in extra_flags: + extra_flags += ["--installer=uv"] + container.call( + [ + "python", + "-m", + "build", + self.container_package_dir, + "--wheel", + f"--outdir={self.built_wheel_dir}", + *extra_flags, + ], + env=self.env, ) - container.call(["sh", "-c", repair_command_prepared], env=env) - else: - container.call(["mv", built_wheel, repaired_wheel_dir]) - - match container.glob(repaired_wheel_dir, "*.whl"): - case []: - raise errors.RepairStepProducedNoWheelError() - case [repaired_wheel]: - pass - case too_many: - raise errors.RepairStepProducedMultipleWheelsError([p.name for p in too_many]) - - if repaired_wheel.name in {wheel.name for wheel in built_wheels}: - raise errors.AlreadyBuiltWheelError(repaired_wheel.name) - - log.step_end() - - if needs_audit(build_options.audit_command, repaired_wheel.name): - local_abi3audit_dir = local_identifier_tmp_dir / "audit" - local_abi3audit_dir.mkdir(parents=True, exist_ok=True) - try: - container.copy_out(repaired_wheel_dir, local_abi3audit_dir) - local_wheel = local_abi3audit_dir / repaired_wheel.name - run_audit(tmp_dir=local_tmp_dir, build_options=build_options, wheel=local_wheel) - finally: - shutil.rmtree(local_abi3audit_dir, ignore_errors=True) - - if build_options.test_command and build_options.test_selector(config.identifier): - log.step("Testing wheel...") - - # set up a virtual environment to install and test from, to make sure - # there are no dependencies that were pulled in at build time. - if not use_uv: + case "uv": container.call( - ["pip", "install", "virtualenv", *dependency_constraint_flags], env=env + [ + "uv", + "build", + f"--python={self.python_bin / 'python'}", + self.container_package_dir, + "--wheel", + f"--out-dir={self.built_wheel_dir}", + *extra_flags, + ], + env=self.env, ) + case "pyodide-build": + msg = "The 'pyodide-build' build frontend is not supported on this platform" + raise errors.FatalError(msg) + case _: + assert_never(build_frontend) - testing_temp_dir = PurePosixPath( - container.call(["mktemp", "-d"], capture_output=True).strip() + try: + return container.glob(self.built_wheel_dir, "*.whl")[0] + except IndexError: + raise errors.BuildProducedNoWheelError() from None + + def repair_wheel(self, built_wheel: PurePosixPath) -> list[PurePosixPath]: + container = self.container + build_options = self.build_options + + container.call(["rm", "-rf", self.repaired_wheel_dir]) + container.call(["mkdir", "-p", self.repaired_wheel_dir]) + + if build_options.repair_command: + repair_command_prepared = prepare_command( + build_options.repair_command, + wheel=built_wheel, + dest_dir=self.repaired_wheel_dir, + package=self.container_package_dir, + project=self.container_project_path, ) - venv_dir = testing_temp_dir / "venv" - - if use_uv: - container.call(["uv", "venv", venv_dir, "--python", python_bin / "python"], env=env) - else: - # Use embedded dependencies from virtualenv to ensure determinism - venv_args = ["--no-periodic-update", "--pip=embed", "--no-setuptools"] - if "38" in config.identifier: - venv_args.append("--no-wheel") - container.call(["python", "-m", "virtualenv", *venv_args, venv_dir], env=env) - - virtualenv_env = env.copy() - virtualenv_env["PATH"] = f"{venv_dir / 'bin'}:{virtualenv_env['PATH']}" - virtualenv_env["VIRTUAL_ENV"] = str(venv_dir) - virtualenv_env = build_options.test_environment.as_dictionary( - prev_environment=virtualenv_env + container.call(["sh", "-c", repair_command_prepared], env=self.env) + else: + container.call(["mv", built_wheel, self.repaired_wheel_dir]) + + return container.glob(self.repaired_wheel_dir, "*.whl") + + def audit_wheel(self, repaired_wheel: PurePosixPath) -> None: + build_options = self.build_options + if not needs_audit(build_options.audit_command, repaired_wheel.name): + return + + # the wheel lives in the container, but the audit tools run on the + # host; copy it out to a temporary directory first + local_abi3audit_dir = self.local_tmp_dir / self.identifier / "audit" + local_abi3audit_dir.mkdir(parents=True, exist_ok=True) + try: + self.container.copy_out(self.repaired_wheel_dir, local_abi3audit_dir) + local_wheel = local_abi3audit_dir / repaired_wheel.name + run_audit(tmp_dir=self.local_tmp_dir, build_options=build_options, wheel=local_wheel) + finally: + shutil.rmtree(local_abi3audit_dir, ignore_errors=True) + + def test_wheel(self, repaired_wheel: PurePosixPath) -> None: + container = self.container + build_options = self.build_options + assert build_options.test_command is not None + + log.step("Testing wheel...") + + # set up a virtual environment to install and test from, to make sure + # there are no dependencies that were pulled in at build time. + if not self.use_uv: + container.call( + ["pip", "install", "virtualenv", *self.dependency_constraint_flags], env=self.env ) - if build_options.before_test: - before_test_prepared = prepare_command( - build_options.before_test, - project=container_project_path, - package=container_package_dir, - ) - container.call(["sh", "-c", before_test_prepared], env=virtualenv_env) + testing_temp_dir = PurePosixPath( + container.call(["mktemp", "-d"], capture_output=True).strip() + ) + venv_dir = testing_temp_dir / "venv" - # Install the wheel we just built + if self.use_uv: container.call( - [*pip, "install", str(repaired_wheel) + build_options.test_extras], - env=virtualenv_env, + ["uv", "venv", venv_dir, "--python", self.python_bin / "python"], env=self.env ) + else: + # Use embedded dependencies from virtualenv to ensure determinism + venv_args = ["--no-periodic-update", "--pip=embed", "--no-setuptools"] + if "38" in self.identifier: + venv_args.append("--no-wheel") + container.call(["python", "-m", "virtualenv", *venv_args, venv_dir], env=self.env) + + virtualenv_env = self.env.copy() + virtualenv_env["PATH"] = f"{venv_dir / 'bin'}:{virtualenv_env['PATH']}" + virtualenv_env["VIRTUAL_ENV"] = str(venv_dir) + virtualenv_env = build_options.test_environment.as_dictionary( + prev_environment=virtualenv_env + ) + + if build_options.before_test: + before_test_prepared = prepare_command( + build_options.before_test, + project=self.container_project_path, + package=self.container_package_dir, + ) + container.call(["sh", "-c", before_test_prepared], env=virtualenv_env) - # Install any requirements to run the tests - if build_options.test_requires: - container.call([*pip, "install", *build_options.test_requires], env=virtualenv_env) + # Install the wheel we just built + container.call( + [*self.pip, "install", str(repaired_wheel) + build_options.test_extras], + env=virtualenv_env, + ) + + # Install any requirements to run the tests + if build_options.test_requires: + container.call([*self.pip, "install", *build_options.test_requires], env=virtualenv_env) + + # Run the tests from a different directory + test_command_prepared = prepare_command( + build_options.test_command, + project=self.container_project_path, + package=self.container_package_dir, + wheel=repaired_wheel, + ) - # Run the tests from a different directory - test_command_prepared = prepare_command( - build_options.test_command, - project=container_project_path, - package=container_package_dir, - wheel=repaired_wheel, + test_cwd = testing_temp_dir / "test_cwd" + container.call(["mkdir", "-p", test_cwd]) + + if build_options.test_sources: + copy_test_sources( + build_options.test_sources, + Path.cwd(), + test_cwd, + copy_into=container.copy_into, ) + else: + # Use the test_fail.py file to raise a nice error if the user + # tries to run tests in the cwd + container.copy_into(resources.TEST_FAIL_CWD_FILE, test_cwd / "test_fail.py") - test_cwd = testing_temp_dir / "test_cwd" - container.call(["mkdir", "-p", test_cwd]) + container.call(["sh", "-c", test_command_prepared], cwd=test_cwd, env=virtualenv_env) - if build_options.test_sources: - copy_test_sources( - build_options.test_sources, - Path.cwd(), - test_cwd, - copy_into=container.copy_into, - ) - else: - # Use the test_fail.py file to raise a nice error if the user - # tries to run tests in the cwd - container.copy_into(resources.TEST_FAIL_CWD_FILE, test_cwd / "test_fail.py") + # clean up test environment + container.call(["rm", "-rf", testing_temp_dir]) + + def move_to_output(self, repaired_wheel: PurePosixPath) -> PurePosixPath: + self.container.call(["mkdir", "-p", self.container_output_dir]) + self.container.call(["mv", repaired_wheel, self.container_output_dir]) + return self.container_output_dir / repaired_wheel.name + + def cleanup(self) -> None: + # nothing to do: the container is removed when the build step ends, + # and the host temp dir is removed at the end of the run + pass + + +def build_in_container( + *, + options: Options, + platform_configs: Sequence[PythonConfiguration], + container: OCIContainer, + container_project_path: PurePath, + container_package_dir: PurePath, + local_tmp_dir: Path, +) -> None: + check_all_python_exist(platform_configs=platform_configs, container=container) - container.call(["sh", "-c", test_command_prepared], cwd=test_cwd, env=virtualenv_env) + log.step("Copying project into container...") + container.copy_into(Path.cwd(), container_project_path) - # clean up test environment - container.call(["rm", "-rf", testing_temp_dir]) + # before_all runs once per container, not once per session + before_all_options_identifier = platform_configs[0].identifier + before_all_options = options.build_options(before_all_options_identifier) - # move repaired wheel to output - output_wheel: Path | None = None - if compatible_wheel is None: - container.call(["mkdir", "-p", container_output_dir]) - container.call(["mv", repaired_wheel, container_output_dir]) - built_wheels.append(container_output_dir / repaired_wheel.name) - output_wheel = options.globals.output_dir / repaired_wheel.name + if before_all_options.before_all: + log.step("Running before_all...") - log.build_end(output_wheel) + env = container.get_environment() + env["PATH"] = f"/opt/python/cp39-cp39/bin:{env['PATH']}" + env["PIP_DISABLE_PIP_VERSION_CHECK"] = "1" + env["PIP_ROOT_USER_ACTION"] = "ignore" + env = before_all_options.environment.as_dictionary( + env, executor=container.environment_executor + ) + + before_all_prepared = prepare_command( + before_all_options.before_all, + project=container_project_path, + package=container_package_dir, + ) + container.call(["sh", "-c", before_all_prepared], env=env) + + runner.run_builds( + [ + LinuxBuilder( + config=config, + build_options=options.build_options(config.identifier), + container=container, + container_project_path=container_project_path, + container_package_dir=container_package_dir, + local_tmp_dir=local_tmp_dir, + ) + for config in platform_configs + ] + ) log.step("Copying wheels back to host...") # copy the output back into the host - container.copy_out(container_output_dir, options.globals.output_dir) + container.copy_out(PurePosixPath("/output"), options.globals.output_dir) log.step_end() @@ -541,7 +580,7 @@ def build(options: Options, tmp_path: Path) -> None: ) raise errors.ConfigurationError(msg) from error - try: + with runner.fatal_on_called_process_error(functools.partial(troubleshoot, options)): ids_to_build = [x.identifier for x in build_step.platform_configs] log.step(f"Starting container image {build_step.container_image}...") @@ -563,11 +602,6 @@ def build(options: Options, tmp_path: Path) -> None: local_tmp_dir=tmp_path, ) - except subprocess.CalledProcessError as error: - troubleshoot(options, error) - msg = f"Command {error.cmd} failed with code {error.returncode}. {error.stdout or ''}" - raise errors.FatalError(msg) from error - def _matches_prepared_command(error_cmd: Sequence[str], command_template: str) -> bool: if len(error_cmd) < 3 or error_cmd[0:2] != ["sh", "-c"]: From 9798ec6360598641b2f49557f5bd9d4a285e7915 Mon Sep 17 00:00:00 2001 From: Henry Schreiner Date: Thu, 9 Jul 2026 02:11:10 -0400 Subject: [PATCH 8/9] refactor: use a Protocol for builders instead of inheritance Rework the builder design in response to review feedback: - Builder is now a Protocol rather than an ABC; platform builders are plain frozen dataclasses with no base class, checked structurally. - A builder is only constructed once its setup has completed. run_builds() takes BuildSpec(identifier, setup) pairs, where setup() prepares the environment and returns the ready-to-use builder, so every attribute on a builder is set from the moment it exists and the dataclass fields document exactly what state each platform carries. - The step bodies previously inherited from the HostBuilder base class are now module functions (runner.host_before_build() etc.) taking a small HostBuilder protocol; platforms delegate to them explicitly, so each platform file lists every step it performs. The two functions that run user commands take env explicitly, which also makes Android's dual-environment usage visible at the call sites. Assisted-by: ClaudeCode:claude-fable-5 --- cibuildwheel/platforms/android.py | 138 +++++++++++++-------- cibuildwheel/platforms/ios.py | 131 ++++++++++++-------- cibuildwheel/platforms/linux.py | 192 ++++++++++++++++------------- cibuildwheel/platforms/macos.py | 132 ++++++++++++-------- cibuildwheel/platforms/pyodide.py | 151 +++++++++++++---------- cibuildwheel/platforms/runner.py | 193 ++++++++++++++++-------------- cibuildwheel/platforms/windows.py | 130 +++++++++++++------- unit_test/runner_test.py | 25 ++-- 8 files changed, 652 insertions(+), 440 deletions(-) diff --git a/cibuildwheel/platforms/android.py b/cibuildwheel/platforms/android.py index a208f49ea..b609c01bc 100644 --- a/cibuildwheel/platforms/android.py +++ b/cibuildwheel/platforms/android.py @@ -12,6 +12,7 @@ "cibuildwheel.util.python_build_standalone", "cibuildwheel.venv", "filelock", + "functools", "packaging", "packaging.utils", "pathlib", @@ -25,6 +26,7 @@ "typing", } +import functools import os import platform import re @@ -449,52 +451,25 @@ def find_pip(build_options: BuildOptions) -> tuple[bool, list[str]]: return use_uv, pip -class AndroidBuilder(runner.HostBuilder): - def __init__( - self, - *, - config: PythonConfiguration, - build_options: BuildOptions, - tmp_dir: Path, - session_tmp_dir: Path, - ) -> None: - super().__init__( - identifier=config.identifier, - build_options=build_options, - tmp_dir=tmp_dir, - session_tmp_dir=session_tmp_dir, - ) - self.config = config +@dataclass(frozen=True, kw_only=True) +class AndroidBuilder: + """Steps to build one Android wheel, invoked by runner.run_builds(). - def setup(self) -> None: - self.tmp_dir.mkdir() - self.python_dir = setup_target_python(self.config, self.tmp_dir) - self.build_env, self.android_env = setup_env( - self.config, self.build_options, self.tmp_dir, self.python_dir - ) - self.setup_xbuild_files() - - def setup_xbuild_files(self) -> None: - _, pip = find_pip(self.build_options) - xbf_dir = self.tmp_dir / "xbuild_files" - xbf_dir.mkdir() - - for requirement in call( - *pip, "freeze", env=self.build_env, capture_stdout=True - ).splitlines(): - name, _, _ = requirement.strip().partition("==") - xbuild_files = self.build_options.xbuild_files.get(canonicalize_name(name), []) - if xbuild_files: - log.step(f"Installing xbuild-files for {name}...") - self.pip_install_android(self.android_env, xbf_dir, "--no-deps", requirement) - for xbf in xbuild_files: - if (xbf_dir / xbf).exists(): - shutil.copy( - xbf_dir / xbf, - find_site_packages(self.build_env) / xbf, - ) - else: - log.warning(f"{xbf_dir / xbf} does not exist") + Android is a cross-compilation target, so there are two environments: + build_env for tools that run on the build machine, and android_env for + anything that must see the target's configuration. + """ + + identifier: str + build_options: BuildOptions + config: PythonConfiguration + tmp_dir: Path + session_tmp_dir: Path + built_wheel_dir: Path + repaired_wheel_dir: Path + python_dir: Path + build_env: dict[str, str] + android_env: dict[str, str] def pip_install_android(self, env: dict[str, str], target: Path, *args: PathOrStr) -> None: use_uv, pip = find_pip(self.build_options) @@ -598,6 +573,9 @@ def repair_wheel(self, built_wheel: Path) -> list[Path]: return list(self.repaired_wheel_dir.glob("*.whl")) + def audit_wheel(self, repaired_wheel: Path) -> None: + runner.host_audit_wheel(self, repaired_wheel) + def test_wheel(self, repaired_wheel: Path) -> None: build_options = self.build_options test_command = build_options.test_command @@ -700,6 +678,62 @@ def test_wheel(self, repaired_wheel: Path) -> None: env=build_test_env, ) + def move_to_output(self, repaired_wheel: Path) -> Path: + return runner.host_move_to_output(self, repaired_wheel) + + def cleanup(self) -> None: + runner.host_cleanup(self) + + +def install_xbuild_files(builder: AndroidBuilder) -> None: + _, pip = find_pip(builder.build_options) + xbf_dir = builder.tmp_dir / "xbuild_files" + xbf_dir.mkdir() + + for requirement in call( + *pip, "freeze", env=builder.build_env, capture_stdout=True + ).splitlines(): + name, _, _ = requirement.strip().partition("==") + xbuild_files = builder.build_options.xbuild_files.get(canonicalize_name(name), []) + if xbuild_files: + log.step(f"Installing xbuild-files for {name}...") + builder.pip_install_android(builder.android_env, xbf_dir, "--no-deps", requirement) + for xbf in xbuild_files: + if (xbf_dir / xbf).exists(): + shutil.copy( + xbf_dir / xbf, + find_site_packages(builder.build_env) / xbf, + ) + else: + log.warning(f"{xbf_dir / xbf} does not exist") + + +def setup_builder( + config: PythonConfiguration, + build_options: BuildOptions, + tmp_dir: Path, + session_tmp_dir: Path, +) -> AndroidBuilder: + """Install Python and prepare the build environments for one identifier.""" + tmp_dir.mkdir() + python_dir = setup_target_python(config, tmp_dir) + build_env, android_env = setup_env(config, build_options, tmp_dir, python_dir) + + builder = AndroidBuilder( + identifier=config.identifier, + build_options=build_options, + config=config, + tmp_dir=tmp_dir, + session_tmp_dir=session_tmp_dir, + built_wheel_dir=tmp_dir / "built_wheel", + repaired_wheel_dir=tmp_dir / "repaired_wheel", + python_dir=python_dir, + build_env=build_env, + android_env=android_env, + ) + install_xbuild_files(builder) + return builder + def build(options: Options, tmp_path: Path) -> None: if "ANDROID_HOME" not in os.environ: @@ -719,11 +753,15 @@ def build(options: Options, tmp_path: Path) -> None: runner.run_before_all(options, configs) runner.run_builds( [ - AndroidBuilder( - config=config, - build_options=options.build_options(config.identifier), - tmp_dir=tmp_path / config.identifier, - session_tmp_dir=tmp_path, + runner.BuildSpec( + identifier=config.identifier, + setup=functools.partial( + setup_builder, + config=config, + build_options=options.build_options(config.identifier), + tmp_dir=tmp_path / config.identifier, + session_tmp_dir=tmp_path, + ), ) for config in configs ] diff --git a/cibuildwheel/platforms/ios.py b/cibuildwheel/platforms/ios.py index 256990d84..d85068973 100644 --- a/cibuildwheel/platforms/ios.py +++ b/cibuildwheel/platforms/ios.py @@ -10,6 +10,7 @@ "cibuildwheel.util.helpers", "cibuildwheel.venv", "filelock", + "functools", "packaging", "packaging.version", "pathlib", @@ -22,6 +23,7 @@ } import dataclasses +import functools import os import platform import shlex @@ -468,49 +470,22 @@ def setup_python( return target_install_path, env -class IOSBuilder(runner.HostBuilder): - def __init__( - self, - *, - config: PythonConfiguration, - build_options: BuildOptions, - tmp_dir: Path, - session_tmp_dir: Path, - ) -> None: - super().__init__( - identifier=config.identifier, - build_options=build_options, - tmp_dir=tmp_dir, - session_tmp_dir=session_tmp_dir, - ) - self.config = config - - def setup(self) -> None: - build_options = self.build_options - build_frontend = build_options.build_frontend - - # uv doesn't support iOS - # Not using set because mypy can't narrow it - if build_frontend.name == "build[uv]" or build_frontend.name == "uv": # noqa: PLR1714 - msg = "uv doesn't support iOS" - raise errors.FatalError(msg) - - self.tmp_dir.mkdir() - - constraints_path = build_options.dependency_constraints.get_for_python_version( - version=self.config.version, tmp_dir=self.tmp_dir - ) +@dataclasses.dataclass(frozen=True, kw_only=True) +class IOSBuilder: + """Steps to build one iOS wheel, invoked by runner.run_builds().""" - self.target_install_path, env = setup_python( - self.tmp_dir / "build", - python_configuration=self.config, - dependency_constraint=constraints_path, - environment=build_options.environment, - build_frontend=build_frontend.name, - xbuild_tools=build_options.xbuild_tools, - ) - env["CIBUILDWHEEL_BUILD_IDENTIFIER"] = self.identifier - self.env = env + identifier: str + build_options: BuildOptions + config: PythonConfiguration + tmp_dir: Path + session_tmp_dir: Path + built_wheel_dir: Path + repaired_wheel_dir: Path + env: dict[str, str] + target_install_path: Path + + def before_build(self) -> None: + runner.host_before_build(self, env=self.env) def build_wheel(self) -> Path: build_options = self.build_options @@ -556,7 +531,7 @@ def build_wheel(self) -> Path: env=self.env, ) case _: - # the other frontends were rejected in setup() or option validation + # the other frontends were rejected in setup_builder() or option validation msg = f"Unsupported build frontend {build_frontend.name!r} for iOS" raise errors.FatalError(msg) @@ -565,6 +540,12 @@ def build_wheel(self) -> Path: except StopIteration: raise errors.BuildProducedNoWheelError() from None + def repair_wheel(self, built_wheel: Path) -> list[Path]: + return runner.host_repair_wheel(self, built_wheel, env=self.env) + + def audit_wheel(self, repaired_wheel: Path) -> None: + runner.host_audit_wheel(self, repaired_wheel) + def test_wheel(self, repaired_wheel: Path) -> None: build_options = self.build_options assert build_options.test_command is not None @@ -725,6 +706,56 @@ def test_wheel(self, repaired_wheel: Path) -> None: log.step_end() + def move_to_output(self, repaired_wheel: Path) -> Path: + return runner.host_move_to_output(self, repaired_wheel) + + def cleanup(self) -> None: + runner.host_cleanup(self) + + +def setup_builder( + config: PythonConfiguration, + build_options: BuildOptions, + tmp_dir: Path, + session_tmp_dir: Path, +) -> IOSBuilder: + """Install Python and prepare the build environment for one identifier.""" + build_frontend = build_options.build_frontend + + # uv doesn't support iOS + # Not using set because mypy can't narrow it + if build_frontend.name == "build[uv]" or build_frontend.name == "uv": # noqa: PLR1714 + msg = "uv doesn't support iOS" + raise errors.FatalError(msg) + + tmp_dir.mkdir() + + constraints_path = build_options.dependency_constraints.get_for_python_version( + version=config.version, tmp_dir=tmp_dir + ) + + target_install_path, env = setup_python( + tmp_dir / "build", + python_configuration=config, + dependency_constraint=constraints_path, + environment=build_options.environment, + build_frontend=build_frontend.name, + xbuild_tools=build_options.xbuild_tools, + ) + env["CIBUILDWHEEL_BUILD_IDENTIFIER"] = config.identifier + + return IOSBuilder( + identifier=config.identifier, + build_options=build_options, + config=config, + tmp_dir=tmp_dir, + session_tmp_dir=session_tmp_dir, + built_wheel_dir=tmp_dir / "built_wheel", + repaired_wheel_dir=tmp_dir / "repaired_wheel", + env=env, + target_install_path=target_install_path, + ) + def build(options: Options, tmp_path: Path) -> None: if sys.platform != "darwin": @@ -747,11 +778,15 @@ def build(options: Options, tmp_path: Path) -> None: ) runner.run_builds( [ - IOSBuilder( - config=config, - build_options=options.build_options(config.identifier), - tmp_dir=tmp_path / config.identifier, - session_tmp_dir=tmp_path, + runner.BuildSpec( + identifier=config.identifier, + setup=functools.partial( + setup_builder, + config=config, + build_options=options.build_options(config.identifier), + tmp_dir=tmp_path / config.identifier, + session_tmp_dir=tmp_path, + ), ) for config in python_configurations ] diff --git a/cibuildwheel/platforms/linux.py b/cibuildwheel/platforms/linux.py index b36d8a0da..5fec7ee15 100644 --- a/cibuildwheel/platforms/linux.py +++ b/cibuildwheel/platforms/linux.py @@ -10,6 +10,7 @@ "collections", "contextlib", "functools", + "pathlib", "shutil", "subprocess", "textwrap", @@ -25,7 +26,7 @@ import textwrap from collections import OrderedDict from pathlib import Path, PurePath, PurePosixPath -from typing import assert_never +from typing import ClassVar, assert_never from cibuildwheel import errors from cibuildwheel.architecture import Architecture @@ -186,86 +187,31 @@ def check_all_python_exist( raise errors.FatalError(message) -class LinuxBuilder(runner.Builder[PurePosixPath]): - """A Builder that runs every step inside an OCI container. Wheel paths - are container-side paths; the wheels are copied back to the host once, - after every build in the container has finished.""" - - def __init__( - self, - *, - config: PythonConfiguration, - build_options: BuildOptions, - container: OCIContainer, - container_project_path: PurePath, - container_package_dir: PurePath, - local_tmp_dir: Path, - ) -> None: - self.identifier = config.identifier - self.build_options = build_options - self.config = config - self.container = container - self.container_project_path = container_project_path - self.container_package_dir = container_package_dir - # host-side scratch space, shared across identifiers - self.local_tmp_dir = local_tmp_dir - self.container_output_dir = PurePosixPath("/output") - container_tmp_dir = PurePosixPath("/tmp/cibuildwheel") - self.built_wheel_dir = container_tmp_dir / "built_wheel" - self.repaired_wheel_dir = container_tmp_dir / "repaired_wheel" - - def setup(self) -> None: - container = self.container - config = self.config - build_options = self.build_options - - self.use_uv = build_options.build_frontend.name in {"build[uv]", "uv"} - self.pip: list[str] = ["uv", "pip"] if self.use_uv else ["pip"] - - log.step("Setting up build environment...") - - self.dependency_constraint_flags: list[PathOrStr] = [] - local_identifier_tmp_dir = self.local_tmp_dir / self.identifier - local_identifier_tmp_dir.mkdir(parents=True, exist_ok=True) - local_constraints_file = build_options.dependency_constraints.get_for_python_version( - version=config.version, - tmp_dir=local_identifier_tmp_dir, - ) - if local_constraints_file: - container_constraints_file = PurePosixPath("/constraints.txt") - container.copy_into(local_constraints_file, container_constraints_file) - self.dependency_constraint_flags = ["-c", container_constraints_file] - - env = container.get_environment() - env["PIP_DISABLE_PIP_VERSION_CHECK"] = "1" - env["PIP_ROOT_USER_ACTION"] = "ignore" - - # put this config's python top of the list - self.python_bin = config.path / "bin" - env["PATH"] = f"{self.python_bin}:{env['PATH']}" - - env = build_options.environment.as_dictionary(env, executor=container.environment_executor) - env["CIBUILDWHEEL_BUILD_IDENTIFIER"] = self.identifier - - # check config python is still on PATH - which_python = container.call(["which", "python"], env=env, capture_output=True).strip() - if PurePosixPath(which_python) != self.python_bin / "python": - msg = "python available on PATH doesn't match our installed instance. If you have modified PATH, ensure that you don't overwrite cibuildwheel's entry or insert python above it." - raise errors.FatalError(msg) - container.call(["python", "-V", "-V"], env=env) +@dataclasses.dataclass(frozen=True, kw_only=True) +class LinuxBuilder: + """Steps to build one Linux wheel, invoked by runner.run_builds(). - if self.use_uv: - which_uv = container.call(["which", "uv"], env=env, capture_output=True).strip() - if not which_uv: - msg = "uv not found on PATH. You must use a supported manylinux or musllinux environment with uv." - raise errors.FatalError(msg) - else: - which_pip = container.call(["which", "pip"], env=env, capture_output=True).strip() - if PurePosixPath(which_pip) != self.python_bin / "pip": - msg = "pip available on PATH doesn't match our installed instance. If you have modified PATH, ensure that you don't overwrite cibuildwheel's entry or insert pip above it." - raise errors.FatalError(msg) + Every step runs inside an OCI container, so wheel paths are container-side + paths; the wheels are copied back to the host once, after every build in + the container has finished. + """ - self.env = env + identifier: str + build_options: BuildOptions + config: PythonConfiguration + container: OCIContainer + container_project_path: PurePath + container_package_dir: PurePath + # host-side scratch space, shared across identifiers + local_tmp_dir: Path + env: dict[str, str] + python_bin: PurePosixPath + use_uv: bool + pip: list[str] + dependency_constraint_flags: list[PathOrStr] + container_output_dir: ClassVar[PurePosixPath] = PurePosixPath("/output") + built_wheel_dir: ClassVar[PurePosixPath] = PurePosixPath("/tmp/cibuildwheel/built_wheel") + repaired_wheel_dir: ClassVar[PurePosixPath] = PurePosixPath("/tmp/cibuildwheel/repaired_wheel") def before_build(self) -> None: build_options = self.build_options @@ -490,6 +436,76 @@ def cleanup(self) -> None: pass +def setup_builder( + config: PythonConfiguration, + build_options: BuildOptions, + container: OCIContainer, + container_project_path: PurePath, + container_package_dir: PurePath, + local_tmp_dir: Path, +) -> LinuxBuilder: + """Prepare the container's build environment for one identifier.""" + use_uv = build_options.build_frontend.name in {"build[uv]", "uv"} + + log.step("Setting up build environment...") + + dependency_constraint_flags: list[PathOrStr] = [] + local_identifier_tmp_dir = local_tmp_dir / config.identifier + local_identifier_tmp_dir.mkdir(parents=True, exist_ok=True) + local_constraints_file = build_options.dependency_constraints.get_for_python_version( + version=config.version, + tmp_dir=local_identifier_tmp_dir, + ) + if local_constraints_file: + container_constraints_file = PurePosixPath("/constraints.txt") + container.copy_into(local_constraints_file, container_constraints_file) + dependency_constraint_flags = ["-c", container_constraints_file] + + env = container.get_environment() + env["PIP_DISABLE_PIP_VERSION_CHECK"] = "1" + env["PIP_ROOT_USER_ACTION"] = "ignore" + + # put this config's python top of the list + python_bin = config.path / "bin" + env["PATH"] = f"{python_bin}:{env['PATH']}" + + env = build_options.environment.as_dictionary(env, executor=container.environment_executor) + env["CIBUILDWHEEL_BUILD_IDENTIFIER"] = config.identifier + + # check config python is still on PATH + which_python = container.call(["which", "python"], env=env, capture_output=True).strip() + if PurePosixPath(which_python) != python_bin / "python": + msg = "python available on PATH doesn't match our installed instance. If you have modified PATH, ensure that you don't overwrite cibuildwheel's entry or insert python above it." + raise errors.FatalError(msg) + container.call(["python", "-V", "-V"], env=env) + + if use_uv: + which_uv = container.call(["which", "uv"], env=env, capture_output=True).strip() + if not which_uv: + msg = "uv not found on PATH. You must use a supported manylinux or musllinux environment with uv." + raise errors.FatalError(msg) + else: + which_pip = container.call(["which", "pip"], env=env, capture_output=True).strip() + if PurePosixPath(which_pip) != python_bin / "pip": + msg = "pip available on PATH doesn't match our installed instance. If you have modified PATH, ensure that you don't overwrite cibuildwheel's entry or insert pip above it." + raise errors.FatalError(msg) + + return LinuxBuilder( + identifier=config.identifier, + build_options=build_options, + config=config, + container=container, + container_project_path=container_project_path, + container_package_dir=container_package_dir, + local_tmp_dir=local_tmp_dir, + env=env, + python_bin=python_bin, + use_uv=use_uv, + pip=["uv", "pip"] if use_uv else ["pip"], + dependency_constraint_flags=dependency_constraint_flags, + ) + + def build_in_container( *, options: Options, @@ -528,13 +544,17 @@ def build_in_container( runner.run_builds( [ - LinuxBuilder( - config=config, - build_options=options.build_options(config.identifier), - container=container, - container_project_path=container_project_path, - container_package_dir=container_package_dir, - local_tmp_dir=local_tmp_dir, + runner.BuildSpec( + identifier=config.identifier, + setup=functools.partial( + setup_builder, + config=config, + build_options=options.build_options(config.identifier), + container=container, + container_project_path=container_project_path, + container_package_dir=container_package_dir, + local_tmp_dir=local_tmp_dir, + ), ) for config in platform_configs ] diff --git a/cibuildwheel/platforms/macos.py b/cibuildwheel/platforms/macos.py index 2ef92eb0c..fe6d81797 100644 --- a/cibuildwheel/platforms/macos.py +++ b/cibuildwheel/platforms/macos.py @@ -453,22 +453,23 @@ def setup_python( return base_python, env -class MacOSBuilder(runner.HostBuilder): - def __init__( - self, - *, - config: PythonConfiguration, - build_options: BuildOptions, - tmp_dir: Path, - session_tmp_dir: Path, - ) -> None: - super().__init__( - identifier=config.identifier, - build_options=build_options, - tmp_dir=tmp_dir, - session_tmp_dir=session_tmp_dir, - ) - self.config = config +@dataclasses.dataclass(frozen=True, kw_only=True) +class MacOSBuilder: + """Steps to build one macOS wheel, invoked by runner.run_builds().""" + + identifier: str + build_options: BuildOptions + config: PythonConfiguration + tmp_dir: Path + session_tmp_dir: Path + built_wheel_dir: Path + repaired_wheel_dir: Path + env: dict[str, str] + base_python: Path + use_uv: bool + uv_path: Path | None + pip: list[str] + pip_version: str | None @property def config_is_arm64(self) -> bool: @@ -478,33 +479,8 @@ def config_is_arm64(self) -> bool: def config_is_universal2(self) -> bool: return self.identifier.endswith("universal2") - def setup(self) -> None: - build_options = self.build_options - build_frontend = build_options.build_frontend - - self.use_uv = build_frontend.name in {"build[uv]", "uv"} - self.uv_path = find_uv() - if self.use_uv and self.uv_path is None: - msg = "uv not found" - raise errors.FatalError(msg) - self.pip: list[str] = ["pip"] if not self.use_uv else [str(self.uv_path), "pip"] - - self.tmp_dir.mkdir() - - constraints_path = build_options.dependency_constraints.get_for_python_version( - version=self.config.version, tmp_dir=self.tmp_dir - ) - - self.base_python, env = setup_python( - self.tmp_dir / "build", - self.config, - constraints_path, - build_options.environment, - build_frontend.name, - ) - env["CIBUILDWHEEL_BUILD_IDENTIFIER"] = self.identifier - self.pip_version = None if self.use_uv else get_pip_version(env) - self.env = env + def before_build(self) -> None: + runner.host_before_build(self, env=self.env) def build_wheel(self) -> Path: build_options = self.build_options @@ -601,6 +577,9 @@ def repair_wheel(self, built_wheel: Path) -> list[Path]: return list(self.repaired_wheel_dir.glob("*.whl")) + def audit_wheel(self, repaired_wheel: Path) -> None: + runner.host_audit_wheel(self, repaired_wheel) + def test_wheel(self, repaired_wheel: Path) -> None: build_options = self.build_options assert build_options.test_command is not None @@ -739,6 +718,59 @@ def test_wheel(self, repaired_wheel: Path) -> None: shell_with_arch(test_command_prepared, cwd=test_cwd, env=virtualenv_env) + def move_to_output(self, repaired_wheel: Path) -> Path: + return runner.host_move_to_output(self, repaired_wheel) + + def cleanup(self) -> None: + runner.host_cleanup(self) + + +def setup_builder( + config: PythonConfiguration, + build_options: BuildOptions, + tmp_dir: Path, + session_tmp_dir: Path, +) -> MacOSBuilder: + """Install Python and prepare the build environment for one identifier.""" + build_frontend = build_options.build_frontend + + use_uv = build_frontend.name in {"build[uv]", "uv"} + uv_path = find_uv() + if use_uv and uv_path is None: + msg = "uv not found" + raise errors.FatalError(msg) + + tmp_dir.mkdir() + + constraints_path = build_options.dependency_constraints.get_for_python_version( + version=config.version, tmp_dir=tmp_dir + ) + + base_python, env = setup_python( + tmp_dir / "build", + config, + constraints_path, + build_options.environment, + build_frontend.name, + ) + env["CIBUILDWHEEL_BUILD_IDENTIFIER"] = config.identifier + + return MacOSBuilder( + identifier=config.identifier, + build_options=build_options, + config=config, + tmp_dir=tmp_dir, + session_tmp_dir=session_tmp_dir, + built_wheel_dir=tmp_dir / "built_wheel", + repaired_wheel_dir=tmp_dir / "repaired_wheel", + env=env, + base_python=base_python, + use_uv=use_uv, + uv_path=uv_path, + pip=["pip"] if not use_uv else [str(uv_path), "pip"], + pip_version=None if use_uv else get_pip_version(env), + ) + def build(options: Options, tmp_path: Path) -> None: python_configurations = get_python_configurations( @@ -756,11 +788,15 @@ def build(options: Options, tmp_path: Path) -> None: ) runner.run_builds( [ - MacOSBuilder( - config=config, - build_options=options.build_options(config.identifier), - tmp_dir=tmp_path / config.identifier, - session_tmp_dir=tmp_path, + runner.BuildSpec( + identifier=config.identifier, + setup=functools.partial( + setup_builder, + config=config, + build_options=options.build_options(config.identifier), + tmp_dir=tmp_path / config.identifier, + session_tmp_dir=tmp_path, + ), ) for config in python_configurations ] diff --git a/cibuildwheel/platforms/pyodide.py b/cibuildwheel/platforms/pyodide.py index 184650467..3352b03ca 100644 --- a/cibuildwheel/platforms/pyodide.py +++ b/cibuildwheel/platforms/pyodide.py @@ -393,62 +393,22 @@ def get_python_configurations( return [c for c in all_python_configurations() if build_selector(c.identifier)] -class PyodideBuilder(runner.HostBuilder): - def __init__( - self, - *, - config: PythonConfiguration, - build_options: BuildOptions, - tmp_dir: Path, - session_tmp_dir: Path, - ) -> None: - super().__init__( - identifier=config.identifier, - build_options=build_options, - tmp_dir=tmp_dir, - session_tmp_dir=session_tmp_dir, - ) - self.config = config - - def setup(self) -> None: - build_options = self.build_options - - self.tmp_dir.mkdir() - self.built_wheel_dir.mkdir() - - constraints_path = build_options.dependency_constraints.get_for_python_version( - version=self.config.version, variant="pyodide", tmp_dir=self.tmp_dir - ) +@dataclasses.dataclass(frozen=True, kw_only=True) +class PyodideBuilder: + """Steps to build one Pyodide wheel, invoked by runner.run_builds().""" - env = setup_python( - tmp=self.tmp_dir / "build", - python_configuration=self.config, - constraints_path=constraints_path, - environment=build_options.environment, - user_pyodide_version=build_options.pyodide_version, - ) - env["CIBUILDWHEEL_BUILD_IDENTIFIER"] = self.identifier - self.pip_version = get_pip_version(env) - - # The Pyodide command line runner mounts all directories in the host - # filesystem into the Pyodide file system, except for the custom - # file systems /dev, /lib, /proc, and /tmp. Mounting the mount - # points for alternate file systems causes some mysterious failure - # of the process (it just quits without any clear error). - # - # Because of this, by default Pyodide can't see anything under /tmp. - # This environment variable tells it also to mount our temp - # directory. - oldmounts = "" - extra_mounts = [str(self.tmp_dir)] - if Path.cwd().is_relative_to("/tmp"): - extra_mounts.append(str(Path.cwd())) - - if "_PYODIDE_EXTRA_MOUNTS" in env: - oldmounts = env["_PYODIDE_EXTRA_MOUNTS"] + ":" - env["_PYODIDE_EXTRA_MOUNTS"] = oldmounts + ":".join(extra_mounts) - - self.env = env + identifier: str + build_options: BuildOptions + config: PythonConfiguration + tmp_dir: Path + session_tmp_dir: Path + built_wheel_dir: Path + repaired_wheel_dir: Path + env: dict[str, str] + pip_version: str + + def before_build(self) -> None: + runner.host_before_build(self, env=self.env) def build_wheel(self) -> Path: build_options = self.build_options @@ -476,6 +436,12 @@ def build_wheel(self) -> Path: except StopIteration: raise errors.BuildProducedNoWheelError() from None + def repair_wheel(self, built_wheel: Path) -> list[Path]: + return runner.host_repair_wheel(self, built_wheel, env=self.env) + + def audit_wheel(self, repaired_wheel: Path) -> None: + runner.host_audit_wheel(self, repaired_wheel) + def test_wheel(self, repaired_wheel: Path) -> None: build_options = self.build_options assert build_options.test_command is not None @@ -552,6 +518,67 @@ def test_wheel(self, repaired_wheel: Path) -> None: shell(test_command_prepared, cwd=test_cwd, env=virtualenv_env) + def move_to_output(self, repaired_wheel: Path) -> Path: + return runner.host_move_to_output(self, repaired_wheel) + + def cleanup(self) -> None: + runner.host_cleanup(self) + + +def setup_builder( + config: PythonConfiguration, + build_options: BuildOptions, + tmp_dir: Path, + session_tmp_dir: Path, +) -> PyodideBuilder: + """Install Python and prepare the build environment for one identifier.""" + tmp_dir.mkdir() + built_wheel_dir = tmp_dir / "built_wheel" + built_wheel_dir.mkdir() + + constraints_path = build_options.dependency_constraints.get_for_python_version( + version=config.version, variant="pyodide", tmp_dir=tmp_dir + ) + + env = setup_python( + tmp=tmp_dir / "build", + python_configuration=config, + constraints_path=constraints_path, + environment=build_options.environment, + user_pyodide_version=build_options.pyodide_version, + ) + env["CIBUILDWHEEL_BUILD_IDENTIFIER"] = config.identifier + + # The Pyodide command line runner mounts all directories in the host + # filesystem into the Pyodide file system, except for the custom + # file systems /dev, /lib, /proc, and /tmp. Mounting the mount + # points for alternate file systems causes some mysterious failure + # of the process (it just quits without any clear error). + # + # Because of this, by default Pyodide can't see anything under /tmp. + # This environment variable tells it also to mount our temp + # directory. + oldmounts = "" + extra_mounts = [str(tmp_dir)] + if Path.cwd().is_relative_to("/tmp"): + extra_mounts.append(str(Path.cwd())) + + if "_PYODIDE_EXTRA_MOUNTS" in env: + oldmounts = env["_PYODIDE_EXTRA_MOUNTS"] + ":" + env["_PYODIDE_EXTRA_MOUNTS"] = oldmounts + ":".join(extra_mounts) + + return PyodideBuilder( + identifier=config.identifier, + build_options=build_options, + config=config, + tmp_dir=tmp_dir, + session_tmp_dir=session_tmp_dir, + built_wheel_dir=built_wheel_dir, + repaired_wheel_dir=tmp_dir / "repaired_wheel", + env=env, + pip_version=get_pip_version(env), + ) + def build(options: Options, tmp_path: Path) -> None: python_configurations = get_python_configurations( @@ -565,11 +592,15 @@ def build(options: Options, tmp_path: Path) -> None: runner.run_before_all(options, python_configurations) runner.run_builds( [ - PyodideBuilder( - config=config, - build_options=options.build_options(config.identifier), - tmp_dir=tmp_path / config.identifier, - session_tmp_dir=tmp_path, + runner.BuildSpec( + identifier=config.identifier, + setup=functools.partial( + setup_builder, + config=config, + build_options=options.build_options(config.identifier), + tmp_dir=tmp_path / config.identifier, + session_tmp_dir=tmp_path, + ), ) for config in python_configurations ] diff --git a/cibuildwheel/platforms/runner.py b/cibuildwheel/platforms/runner.py index 8bbb29012..8da045848 100644 --- a/cibuildwheel/platforms/runner.py +++ b/cibuildwheel/platforms/runner.py @@ -1,16 +1,23 @@ """ The shared build loop, used by every platform module. -Each platform defines a Builder subclass whose methods implement the -platform-specific steps; run_builds() drives those steps for each build -identifier, in a fixed order: +Each platform defines a builder class implementing the Builder protocol, whose +methods are the platform-specific steps of one wheel build. run_builds() takes +a BuildSpec per build identifier and drives the steps in a fixed order: setup -> before_build -> build_wheel -> repair_wheel -> audit_wheel -> test_wheel -> move_to_output -> cleanup +A builder is only constructed once its environment is fully set up: each +BuildSpec carries a setup function that does the work and returns the +ready-to-use builder, so every attribute on a builder is set from the moment +it exists. + run_builds() also owns everything that's identical across platforms: logging, reuse of compatible wheels, validation of the built/repaired wheels, the test -gate, and moving wheels to the output directory. +gate, and moving wheels to the output directory. The host_*() functions are +the step implementations shared by the platforms that build directly on the +host (everything except Linux); builders use them by delegation. """ from __future__ import annotations @@ -28,12 +35,12 @@ } import contextlib +import dataclasses import os import shutil import subprocess -from abc import ABC, abstractmethod from pathlib import Path, PurePath -from typing import Generic, TypeVar +from typing import Generic, Protocol, TypeVar from cibuildwheel import errors from cibuildwheel.audit import run_audit @@ -54,44 +61,38 @@ PathT = TypeVar("PathT", bound=PurePath) -class Builder(ABC, Generic[PathT]): +class Builder(Protocol[PathT]): """ The platform-specific steps to build one wheel (one build identifier). - Constructing a Builder must not do any work; all work happens in the step - methods, which run_builds() calls in a fixed order. PathT is the type of - the paths the wheels live at while building: Path on the host platforms, - PurePosixPath inside the Linux container. + Builders are plain classes (typically frozen dataclasses) satisfying this + protocol structurally; they are created by a BuildSpec's setup function + with all their state already prepared. run_builds() calls the steps in a + fixed order. PathT is the type of the paths the wheels live at while + building: Path on the host platforms, PurePosixPath inside the Linux + container. """ - identifier: str - build_options: BuildOptions + @property + def identifier(self) -> str: ... - @abstractmethod - def setup(self) -> None: - """Install Python and prepare the build environment, storing any state - the later steps need (e.g. self.env) on the instance. Logs its own - log.step()s and may leave the last one open.""" + @property + def build_options(self) -> BuildOptions: ... - @abstractmethod def before_build(self) -> None: """Run the user's before-build command. Only called if one is set.""" - @abstractmethod def build_wheel(self) -> PathT: """Build the wheel with the configured frontend and return its path.""" - @abstractmethod def repair_wheel(self, built_wheel: PathT) -> list[PathT]: """Run the user's repair command (or, if none is set, move the wheel as-is) and return the contents of the repaired-wheel directory. The runner validates that exactly one wheel was produced.""" - @abstractmethod def audit_wheel(self, repaired_wheel: PathT) -> None: """Run the audit command, if one is configured.""" - @abstractmethod def test_wheel(self, repaired_wheel: PathT) -> None: """Test the wheel: set up a test environment, run before-test, install the wheel and its test requirements, and run the test command. Only @@ -99,98 +100,106 @@ def test_wheel(self, repaired_wheel: PathT) -> None: conditions that can only be determined at build time (e.g. an arch that can't be tested on this machine) live here too.""" - @abstractmethod def move_to_output(self, repaired_wheel: PathT) -> PathT: """Move the wheel to the output directory, returning the path that later builds can reuse it from (and install it from, in tests).""" - @abstractmethod def cleanup(self) -> None: """Remove this identifier's temporary files.""" -class HostBuilder(Builder[Path]): - """A Builder whose wheels live on the host filesystem; implements the - steps that are common to every platform except Linux (which builds inside - a container).""" - - env: dict[str, str] # set by setup() in each subclass - - def __init__( - self, - *, - identifier: str, - build_options: BuildOptions, - tmp_dir: Path, - session_tmp_dir: Path, - ) -> None: - self.identifier = identifier - self.build_options = build_options - # per-identifier scratch space, removed by cleanup() - self.tmp_dir = tmp_dir - # shared across identifiers; run_audit() reuses its venv between calls - self.session_tmp_dir = session_tmp_dir - self.built_wheel_dir = tmp_dir / "built_wheel" - self.repaired_wheel_dir = tmp_dir / "repaired_wheel" +@dataclasses.dataclass(frozen=True, kw_only=True) +class BuildSpec(Generic[PathT]): + """One wheel to build: its identifier, plus a setup function that installs + Python, prepares the build environment, and returns the builder that runs + the remaining steps. setup logs its own log.step()s and may leave the last + one open.""" - def before_build(self) -> None: - assert self.build_options.before_build is not None - before_build_prepared = prepare_command( - self.build_options.before_build, + identifier: str + setup: Callable[[], Builder[PathT]] + + +class HostBuilder(Builder[Path], Protocol): + """The state the shared host_*() step implementations need on a builder + that builds directly on the host (every platform except Linux).""" + + @property + def tmp_dir(self) -> Path: + """Per-identifier scratch space, removed by cleanup().""" + + @property + def session_tmp_dir(self) -> Path: + """Shared across identifiers; run_audit() reuses its venv between + calls.""" + + @property + def repaired_wheel_dir(self) -> Path: ... + + +def host_before_build(builder: HostBuilder, *, env: dict[str, str]) -> None: + assert builder.build_options.before_build is not None + before_build_prepared = prepare_command( + builder.build_options.before_build, + project=".", + package=builder.build_options.package_dir, + ) + shell(before_build_prepared, env=env) + + +def host_repair_wheel( + builder: HostBuilder, built_wheel: Path, *, env: dict[str, str] +) -> list[Path]: + builder.repaired_wheel_dir.mkdir(exist_ok=True) + if builder.build_options.repair_command: + repair_command_prepared = prepare_command( + builder.build_options.repair_command, + wheel=built_wheel, + dest_dir=builder.repaired_wheel_dir, + package=builder.build_options.package_dir, project=".", - package=self.build_options.package_dir, ) - shell(before_build_prepared, env=self.env) - - def repair_wheel(self, built_wheel: Path) -> list[Path]: - self.repaired_wheel_dir.mkdir(exist_ok=True) - if self.build_options.repair_command: - repair_command_prepared = prepare_command( - self.build_options.repair_command, - wheel=built_wheel, - dest_dir=self.repaired_wheel_dir, - package=self.build_options.package_dir, - project=".", - ) - shell(repair_command_prepared, env=self.env) - else: - shutil.move(str(built_wheel), self.repaired_wheel_dir) - return list(self.repaired_wheel_dir.glob("*.whl")) + shell(repair_command_prepared, env=env) + else: + shutil.move(str(built_wheel), builder.repaired_wheel_dir) + return list(builder.repaired_wheel_dir.glob("*.whl")) - def audit_wheel(self, repaired_wheel: Path) -> None: - run_audit( - tmp_dir=self.session_tmp_dir, build_options=self.build_options, wheel=repaired_wheel - ) - def move_to_output(self, repaired_wheel: Path) -> Path: - output_wheel = self.build_options.output_dir / repaired_wheel.name - moved_wheel = move_file(repaired_wheel, output_wheel) - if moved_wheel != output_wheel.resolve(): - log.warning(f"{repaired_wheel} was moved to {moved_wheel} instead of {output_wheel}") - return output_wheel +def host_audit_wheel(builder: HostBuilder, repaired_wheel: Path) -> None: + run_audit( + tmp_dir=builder.session_tmp_dir, build_options=builder.build_options, wheel=repaired_wheel + ) - def cleanup(self) -> None: - # ignore_errors: occasionally Windows fails to unlink a file, and we - # don't want to abort a build because of a leftover temp dir - shutil.rmtree(self.tmp_dir, ignore_errors=True) +def host_move_to_output(builder: HostBuilder, repaired_wheel: Path) -> Path: + output_wheel = builder.build_options.output_dir / repaired_wheel.name + moved_wheel = move_file(repaired_wheel, output_wheel) + if moved_wheel != output_wheel.resolve(): + log.warning(f"{repaired_wheel} was moved to {moved_wheel} instead of {output_wheel}") + return output_wheel -def run_builds(builders: Sequence[Builder[PathT]]) -> None: - """Build a wheel for each builder, in order.""" + +def host_cleanup(builder: HostBuilder) -> None: + # ignore_errors: occasionally Windows fails to unlink a file, and we + # don't want to abort a build because of a leftover temp dir + shutil.rmtree(builder.tmp_dir, ignore_errors=True) + + +def run_builds(specs: Sequence[BuildSpec[PathT]]) -> None: + """Build a wheel for each spec, in order.""" built_wheels: list[PathT] = [] - for builder in builders: - build_options = builder.build_options - log.build_start(builder.identifier) + for spec in specs: + log.build_start(spec.identifier) - builder.setup() + builder = spec.setup() + build_options = builder.build_options - compatible_wheel = find_compatible_wheel(built_wheels, builder.identifier) + compatible_wheel = find_compatible_wheel(built_wheels, spec.identifier) if compatible_wheel is not None: log.step_end() print( f"\nFound previously built wheel {compatible_wheel.name}, that's compatible with " - f"{builder.identifier}. Skipping build step..." + f"{spec.identifier}. Skipping build step..." ) repaired_wheel = compatible_wheel else: @@ -222,7 +231,7 @@ def run_builds(builders: Sequence[Builder[PathT]]) -> None: builder.audit_wheel(repaired_wheel) - if build_options.test_command and build_options.test_selector(builder.identifier): + if build_options.test_command and build_options.test_selector(spec.identifier): builder.test_wheel(repaired_wheel) output_wheel: Path | None = None diff --git a/cibuildwheel/platforms/windows.py b/cibuildwheel/platforms/windows.py index 50f68e04a..c8b5ed7d3 100644 --- a/cibuildwheel/platforms/windows.py +++ b/cibuildwheel/platforms/windows.py @@ -20,7 +20,7 @@ import dataclasses import platform as platform_module import textwrap -from functools import cache +from functools import cache, partial from pathlib import Path from typing import assert_never @@ -407,47 +407,25 @@ def setup_python( return base_python, env -class WindowsBuilder(runner.HostBuilder): - def __init__( - self, - *, - config: PythonConfiguration, - build_options: BuildOptions, - tmp_dir: Path, - session_tmp_dir: Path, - ) -> None: - super().__init__( - identifier=config.identifier, - build_options=build_options, - tmp_dir=tmp_dir, - session_tmp_dir=session_tmp_dir, - ) - self.config = config - - def setup(self) -> None: - build_options = self.build_options - build_frontend = build_options.build_frontend - - self.use_uv = build_frontend.name in {"build[uv]", "uv"} - self.uv_path = find_uv() - - self.tmp_dir.mkdir() - - constraints_path = build_options.dependency_constraints.get_for_python_version( - version=self.config.version, tmp_dir=self.tmp_dir - ) +@dataclasses.dataclass(frozen=True, kw_only=True) +class WindowsBuilder: + """Steps to build one Windows wheel, invoked by runner.run_builds().""" - # install Python - self.base_python, env = setup_python( - self.tmp_dir / "build", - self.config, - constraints_path, - build_options.environment, - build_frontend.name, - ) - env["CIBUILDWHEEL_BUILD_IDENTIFIER"] = self.identifier - self.pip_version = None if self.use_uv else get_pip_version(env) - self.env = env + identifier: str + build_options: BuildOptions + config: PythonConfiguration + tmp_dir: Path + session_tmp_dir: Path + built_wheel_dir: Path + repaired_wheel_dir: Path + env: dict[str, str] + base_python: Path + use_uv: bool + uv_path: Path | None + pip_version: str | None + + def before_build(self) -> None: + runner.host_before_build(self, env=self.env) def build_wheel(self) -> Path: build_options = self.build_options @@ -517,6 +495,12 @@ def build_wheel(self) -> Path: except StopIteration: raise errors.BuildProducedNoWheelError() from None + def repair_wheel(self, built_wheel: Path) -> list[Path]: + return runner.host_repair_wheel(self, built_wheel, env=self.env) + + def audit_wheel(self, repaired_wheel: Path) -> None: + runner.host_audit_wheel(self, repaired_wheel) + def test_wheel(self, repaired_wheel: Path) -> None: build_options = self.build_options assert build_options.test_command is not None @@ -596,6 +580,56 @@ def test_wheel(self, repaired_wheel: Path) -> None: ) shell(test_command_prepared, cwd=test_cwd, env=virtualenv_env) + def move_to_output(self, repaired_wheel: Path) -> Path: + return runner.host_move_to_output(self, repaired_wheel) + + def cleanup(self) -> None: + runner.host_cleanup(self) + + +def setup_builder( + config: PythonConfiguration, + build_options: BuildOptions, + tmp_dir: Path, + session_tmp_dir: Path, +) -> WindowsBuilder: + """Install Python and prepare the build environment for one identifier.""" + build_frontend = build_options.build_frontend + + use_uv = build_frontend.name in {"build[uv]", "uv"} + uv_path = find_uv() + + tmp_dir.mkdir() + + constraints_path = build_options.dependency_constraints.get_for_python_version( + version=config.version, tmp_dir=tmp_dir + ) + + # install Python + base_python, env = setup_python( + tmp_dir / "build", + config, + constraints_path, + build_options.environment, + build_frontend.name, + ) + env["CIBUILDWHEEL_BUILD_IDENTIFIER"] = config.identifier + + return WindowsBuilder( + identifier=config.identifier, + build_options=build_options, + config=config, + tmp_dir=tmp_dir, + session_tmp_dir=session_tmp_dir, + built_wheel_dir=tmp_dir / "built_wheel", + repaired_wheel_dir=tmp_dir / "repaired_wheel", + env=env, + base_python=base_python, + use_uv=use_uv, + uv_path=uv_path, + pip_version=None if use_uv else get_pip_version(env), + ) + def build(options: Options, tmp_path: Path) -> None: python_configurations = get_python_configurations( @@ -609,11 +643,15 @@ def build(options: Options, tmp_path: Path) -> None: runner.run_before_all(options, python_configurations) runner.run_builds( [ - WindowsBuilder( - config=config, - build_options=options.build_options(config.identifier), - tmp_dir=tmp_path / config.identifier, - session_tmp_dir=tmp_path, + runner.BuildSpec( + identifier=config.identifier, + setup=partial( + setup_builder, + config=config, + build_options=options.build_options(config.identifier), + tmp_dir=tmp_path / config.identifier, + session_tmp_dir=tmp_path, + ), ) for config in python_configurations ] diff --git a/unit_test/runner_test.py b/unit_test/runner_test.py index b29da8268..d01f5fbca 100644 --- a/unit_test/runner_test.py +++ b/unit_test/runner_test.py @@ -56,7 +56,7 @@ def build_options(tmp_path: Path) -> BuildOptions: ) -class FakeBuilder(runner.Builder[Path]): +class FakeBuilder: """Records the steps the runner invokes; touches no real files.""" def __init__( @@ -77,8 +77,13 @@ def __init__( def record(self, step: str) -> None: self.calls.append((self.identifier, step)) - def setup(self) -> None: + def setup(self) -> FakeBuilder: self.record("setup") + return self + + @property + def spec(self) -> runner.BuildSpec[Path]: + return runner.BuildSpec(identifier=self.identifier, setup=self.setup) def before_build(self) -> None: self.record("before_build") @@ -123,7 +128,7 @@ def test_step_order(build_options: BuildOptions) -> None: built_wheel_name=wheel_name, ) - runner.run_builds([builder]) + runner.run_builds([builder.spec]) assert calls == [ ("cp311-macosx_arm64", "setup"), @@ -141,7 +146,7 @@ def test_no_before_build_no_test(build_options: BuildOptions) -> None: calls: list[tuple[str, str]] = [] builder = FakeBuilder(identifier="cp311-macosx_arm64", build_options=build_options, calls=calls) - runner.run_builds([builder]) + runner.run_builds([builder.spec]) steps = [step for _, step in calls] assert "before_build" not in steps @@ -174,7 +179,7 @@ def test_compatible_wheel_reuse(build_options: BuildOptions) -> None: ), ] - runner.run_builds(builders) + runner.run_builds([b.spec for b in builders]) second = [step for identifier, step in calls if identifier == "cp312-macosx_arm64"] # build/repair/audit/move are skipped, but the reused wheel is still tested @@ -190,7 +195,7 @@ def test_repair_produced_no_wheel(build_options: BuildOptions) -> None: ) with pytest.raises(errors.RepairStepProducedNoWheelError): - runner.run_builds([builder]) + runner.run_builds([builder.spec]) def test_repair_produced_multiple_wheels(build_options: BuildOptions) -> None: @@ -205,7 +210,7 @@ def test_repair_produced_multiple_wheels(build_options: BuildOptions) -> None: ) with pytest.raises(errors.RepairStepProducedMultipleWheelsError): - runner.run_builds([builder]) + runner.run_builds([builder.spec]) def test_already_built_wheel(build_options: BuildOptions) -> None: @@ -228,7 +233,7 @@ def test_already_built_wheel(build_options: BuildOptions) -> None: ] with pytest.raises(errors.AlreadyBuiltWheelError): - runner.run_builds(builders) + runner.run_builds([b.spec for b in builders]) def test_none_any_wheel_rejected(build_options: BuildOptions) -> None: @@ -240,7 +245,7 @@ def test_none_any_wheel_rejected(build_options: BuildOptions) -> None: ) with pytest.raises(errors.NonPlatformWheelError): - runner.run_builds([builder]) + runner.run_builds([builder.spec]) def test_test_skipped_by_selector(build_options: BuildOptions) -> None: @@ -254,7 +259,7 @@ def test_test_skipped_by_selector(build_options: BuildOptions) -> None: calls: list[tuple[str, str]] = [] builder = FakeBuilder(identifier="cp311-macosx_arm64", build_options=build_options, calls=calls) - runner.run_builds([builder]) + runner.run_builds([builder.spec]) assert not any(step.startswith("test_wheel") for _, step in calls) From 1bcfe72a28fea364729590b9015649c69871eeb3 Mon Sep 17 00:00:00 2001 From: Henry Schreiner Date: Thu, 9 Jul 2026 08:48:51 -0400 Subject: [PATCH 9/9] fix: reject a pure Python wheel from repair on every platform Android checked the repaired wheel for none-any and its test suite asserts that; the other platforms only checked the built wheel, so a repair command emitting a pure wheel slipped through silently. Rather than dropping Android's check, extend it: run_builds() now validates the repaired wheel too, on all platforms. Assisted-by: ClaudeCode:claude-fable-5 --- cibuildwheel/platforms/runner.py | 3 +++ unit_test/runner_test.py | 12 ++++++++++++ 2 files changed, 15 insertions(+) diff --git a/cibuildwheel/platforms/runner.py b/cibuildwheel/platforms/runner.py index 8da045848..42fe50f6a 100644 --- a/cibuildwheel/platforms/runner.py +++ b/cibuildwheel/platforms/runner.py @@ -225,6 +225,9 @@ def run_builds(specs: Sequence[BuildSpec[PathT]]) -> None: [wheel.name for wheel in many_wheels] ) + if repaired_wheel.name.endswith("none-any.whl"): + raise errors.NonPlatformWheelError() + if repaired_wheel.name in {wheel.name for wheel in built_wheels}: raise errors.AlreadyBuiltWheelError(repaired_wheel.name) log.step_end() diff --git a/unit_test/runner_test.py b/unit_test/runner_test.py index d01f5fbca..a34cbf8e0 100644 --- a/unit_test/runner_test.py +++ b/unit_test/runner_test.py @@ -248,6 +248,18 @@ def test_none_any_wheel_rejected(build_options: BuildOptions) -> None: runner.run_builds([builder.spec]) +def test_none_any_repaired_wheel_rejected(build_options: BuildOptions) -> None: + builder = FakeBuilder( + identifier="cp311-macosx_arm64", + build_options=build_options, + calls=[], + repaired_wheel_names=("spam-0.1.0-py3-none-any.whl",), + ) + + with pytest.raises(errors.NonPlatformWheelError): + runner.run_builds([builder.spec]) + + def test_test_skipped_by_selector(build_options: BuildOptions) -> None: build_options = dataclasses.replace( build_options,