From 3a3caec646ac1e0e4e81b3b879bf853f0edc1520 Mon Sep 17 00:00:00 2001 From: Malcolm Smith Date: Sat, 5 Apr 2025 00:12:43 +0100 Subject: [PATCH 01/57] Add Android to resource files --- bin/generate_schema.py | 1 + bin/update_pythons.py | 58 +++++++++++++++--- cibuildwheel/resources/build-platforms.toml | 6 ++ .../resources/cibuildwheel.schema.json | 59 ++++++++++++++++++- cibuildwheel/resources/defaults.toml | 2 + 5 files changed, 116 insertions(+), 10 deletions(-) diff --git a/bin/generate_schema.py b/bin/generate_schema.py index b82a91136..d7a472752 100755 --- a/bin/generate_schema.py +++ b/bin/generate_schema.py @@ -322,6 +322,7 @@ def as_object(d: dict[str, Any]) -> dict[str, Any]: "windows": as_object(not_linux), "macos": as_object(not_linux), "pyodide": as_object(not_linux), + "android": as_object(not_linux), "ios": as_object(not_linux), } diff --git a/bin/update_pythons.py b/bin/update_pythons.py index 7b8ea1474..ce6466c8c 100755 --- a/bin/update_pythons.py +++ b/bin/update_pythons.py @@ -9,6 +9,7 @@ from collections.abc import Mapping, MutableMapping from pathlib import Path from typing import Any, Final, Literal, TypedDict +from xml.etree import ElementTree as ET import click import requests @@ -50,7 +51,13 @@ class ConfigApple(TypedDict): url: str -AnyConfig = ConfigWinCP | ConfigWinPP | ConfigApple +class ConfigAndroid(TypedDict): + identifier: str + version: str + url: str + + +AnyConfig = ConfigWinCP | ConfigWinPP | ConfigApple | ConfigAndroid # The following set of "Versions" classes allow the initial call to the APIs to @@ -232,6 +239,41 @@ def update_version_macos( return None +class AndroidVersions: + MAVEN_URL = "https://repo.maven.apache.org/maven2/com/chaquo/python/python" + + def __init__(self) -> None: + response = requests.get(f"{self.MAVEN_URL}/maven-metadata.xml") + response.raise_for_status() + root = ET.fromstring(response.text) + + self.versions: list[Version] = [] + for version_elem in root.findall("./versioning/versions/version"): + version_str = version_elem.text + assert isinstance(version_str, str), version_str + self.versions.append(Version(version_str)) + + def update_version_android( + self, identifier: str, version: Version, spec: Specifier + ) -> ConfigAndroid | None: + sorted_versions = sorted(spec.filter(self.versions), reverse=True) + + # Return a config using the highest version for the given specifier. + if sorted_versions: + max_version = sorted_versions[0] + triplet = { + "arm64_v8a": "aarch64-linux-android", + "x86_64": "x86_64-linux-android", + }[identifier.split("_", 1)[1]] + return ConfigAndroid( + identifier=identifier, + version=str(version), + url=f"{self.MAVEN_URL}/{max_version}/python-{max_version}-{triplet}.tar.gz", + ) + else: + return None + + class CPythonIOSVersions: def __init__(self) -> None: response = requests.get( @@ -292,6 +334,7 @@ def __init__(self) -> None: self.macos_pypy = PyPyVersions("64") self.macos_pypy_arm64 = PyPyVersions("ARM64") + self.android = AndroidVersions() self.ios_cpython = CPythonIOSVersions() def update_config(self, config: MutableMapping[str, str]) -> None: @@ -326,6 +369,8 @@ def update_config(self, config: MutableMapping[str, str]) -> None: config_update = self.windows_t_arm64.update_version_windows(spec) elif "win_arm64" in identifier and identifier.startswith("cp"): config_update = self.windows_arm64.update_version_windows(spec) + elif "android" in identifier: + config_update = self.android.update_version_android(identifier, version, spec) elif "ios" in identifier: config_update = self.ios_cpython.update_version_ios(identifier, version) @@ -357,14 +402,9 @@ def update_pythons(force: bool, level: str) -> None: with toml_file_path.open("rb") as f: configs = tomllib.load(f) - for config in configs["windows"]["python_configurations"]: - all_versions.update_config(config) - - for config in configs["macos"]["python_configurations"]: - all_versions.update_config(config) - - for config in configs["ios"]["python_configurations"]: - all_versions.update_config(config) + for platform in ["windows", "macos", "android", "ios"]: + for config in configs[platform]["python_configurations"]: + all_versions.update_config(config) result_toml = dump_python_configurations(configs) diff --git a/cibuildwheel/resources/build-platforms.toml b/cibuildwheel/resources/build-platforms.toml index 18838f15c..be29af4e3 100644 --- a/cibuildwheel/resources/build-platforms.toml +++ b/cibuildwheel/resources/build-platforms.toml @@ -164,6 +164,12 @@ python_configurations = [ { identifier = "cp312-pyodide_wasm32", version = "3.12", pyodide_version = "0.27.0", pyodide_build_version = "0.29.2", emscripten_version = "3.1.58", node_version = "v20" }, ] +[android] +python_configurations = [ + { identifier = "cp313-android_arm64_v8a", version = "3.13", url = "https://repo.maven.apache.org/maven2/com/chaquo/python/python/3.13.2/python-3.13.2-aarch64-linux-android.tar.gz" }, + { identifier = "cp313-android_x86_64", version = "3.13", url = "https://repo.maven.apache.org/maven2/com/chaquo/python/python/3.13.2/python-3.13.2-x86_64-linux-android.tar.gz" }, +] + [ios] python_configurations = [ { identifier = "cp313-ios_arm64_iphoneos", version = "3.13", url = "https://github.com/beeware/Python-Apple-support/releases/download/3.13-b6/Python-3.13-iOS-support.b6.tar.gz" }, diff --git a/cibuildwheel/resources/cibuildwheel.schema.json b/cibuildwheel/resources/cibuildwheel.schema.json index 4a84606cf..d0833caac 100644 --- a/cibuildwheel/resources/cibuildwheel.schema.json +++ b/cibuildwheel/resources/cibuildwheel.schema.json @@ -957,6 +957,57 @@ } } }, + "android": { + "type": "object", + "additionalProperties": false, + "properties": { + "archs": { + "$ref": "#/properties/archs" + }, + "before-all": { + "$ref": "#/properties/before-all" + }, + "before-build": { + "$ref": "#/properties/before-build" + }, + "before-test": { + "$ref": "#/properties/before-test" + }, + "build-frontend": { + "$ref": "#/properties/build-frontend" + }, + "build-verbosity": { + "$ref": "#/properties/build-verbosity" + }, + "config-settings": { + "$ref": "#/properties/config-settings" + }, + "dependency-versions": { + "$ref": "#/properties/dependency-versions" + }, + "environment": { + "$ref": "#/properties/environment" + }, + "repair-wheel-command": { + "$ref": "#/properties/repair-wheel-command" + }, + "test-command": { + "$ref": "#/properties/test-command" + }, + "test-extras": { + "$ref": "#/properties/test-extras" + }, + "test-sources": { + "$ref": "#/properties/test-sources" + }, + "test-groups": { + "$ref": "#/properties/test-groups" + }, + "test-requires": { + "$ref": "#/properties/test-requires" + } + } + }, "ios": { "type": "object", "additionalProperties": false, @@ -997,10 +1048,16 @@ "test-extras": { "$ref": "#/properties/test-extras" }, + "test-sources": { + "$ref": "#/properties/test-sources" + }, + "test-groups": { + "$ref": "#/properties/test-groups" + }, "test-requires": { "$ref": "#/properties/test-requires" } - } + } } } } diff --git a/cibuildwheel/resources/defaults.toml b/cibuildwheel/resources/defaults.toml index bd17245b4..545a3be91 100644 --- a/cibuildwheel/resources/defaults.toml +++ b/cibuildwheel/resources/defaults.toml @@ -51,6 +51,8 @@ repair-wheel-command = "delocate-wheel --require-archs {delocate_archs} -w {dest [tool.cibuildwheel.windows] +[tool.cibuildwheel.android] + [tool.cibuildwheel.ios] [tool.cibuildwheel.pyodide] From 578c1811cb9c8ab04e7074e668aae0a108f7eafa Mon Sep 17 00:00:00 2001 From: Malcolm Smith Date: Sat, 5 Apr 2025 00:23:15 +0100 Subject: [PATCH 02/57] Add Android to miscellaneous places --- cibuildwheel/__main__.py | 8 +++++--- cibuildwheel/architecture.py | 11 ++++++++--- cibuildwheel/logger.py | 2 ++ cibuildwheel/platforms/__init__.py | 3 ++- cibuildwheel/typing.py | 2 +- cibuildwheel/util/packaging.py | 6 +++--- 6 files changed, 21 insertions(+), 11 deletions(-) diff --git a/cibuildwheel/__main__.py b/cibuildwheel/__main__.py index eded443de..088286ac1 100644 --- a/cibuildwheel/__main__.py +++ b/cibuildwheel/__main__.py @@ -90,14 +90,14 @@ def main_inner(global_options: GlobalOptions) -> None: parser.add_argument( "--platform", - choices=["auto", "linux", "macos", "windows", "pyodide", "ios"], + choices=["auto", "linux", "macos", "windows", "pyodide", "android", "ios"], default=None, help=""" Platform to build for. Use this option to override the auto-detected platform. Specifying "macos" or "windows" only works on that operating system. "linux" works on any desktop OS, as long as - Docker/Podman is installed. "pyodide" only works on linux and macOS. - "ios" only work on macOS. Default: auto. + Docker/Podman is installed. "pyodide" and "android" only work on + Linux and macOS. "ios" only works on macOS. Default: auto. """, ) @@ -238,6 +238,8 @@ def _compute_platform_only(only: str) -> PlatformName: return "windows" if "pyodide_" in only: return "pyodide" + if "android_" in only: + return "android" if "ios_" in only: return "ios" msg = f"Invalid --only='{only}', must be a build selector with a known platform" diff --git a/cibuildwheel/architecture.py b/cibuildwheel/architecture.py index de17c75dd..6d33e53f5 100644 --- a/cibuildwheel/architecture.py +++ b/cibuildwheel/architecture.py @@ -17,13 +17,14 @@ "macos": "macOS", "windows": "Windows", "pyodide": "Pyodide", + "android": "Android", "ios": "iOS", } ARCH_SYNONYMS: Final[list[dict[PlatformName, str | None]]] = [ - {"linux": "x86_64", "macos": "x86_64", "windows": "AMD64"}, + {"linux": "x86_64", "macos": "x86_64", "windows": "AMD64", "android": "x86_64"}, {"linux": "i686", "macos": None, "windows": "x86"}, - {"linux": "aarch64", "macos": "arm64", "windows": "ARM64"}, + {"linux": "aarch64", "macos": "arm64", "windows": "ARM64", "android": "arm64_v8a"}, ] @@ -42,7 +43,7 @@ def _check_aarch32_el0() -> bool: @typing.final class Architecture(StrEnum): - # mac/linux archs + # mac/linux/android archs x86_64 = auto() # linux archs @@ -64,6 +65,9 @@ class Architecture(StrEnum): # WebAssembly wasm32 = auto() + # android archs + arm64_v8a = auto() + # iOS "multiarch" architectures that include both # the CPU architecture and the ABI. arm64_iphoneos = auto() @@ -171,6 +175,7 @@ def all_archs(platform: PlatformName) -> "set[Architecture]": "macos": {Architecture.x86_64, Architecture.arm64, Architecture.universal2}, "windows": {Architecture.x86, Architecture.AMD64, Architecture.ARM64}, "pyodide": {Architecture.wasm32}, + "android": {Architecture.x86_64, Architecture.arm64_v8a}, "ios": { Architecture.x86_64_iphonesimulator, Architecture.arm64_iphonesimulator, diff --git a/cibuildwheel/logger.py b/cibuildwheel/logger.py index 391786a04..8d9aa5bd4 100644 --- a/cibuildwheel/logger.py +++ b/cibuildwheel/logger.py @@ -35,6 +35,8 @@ "macosx_universal2": "macOS Universal 2 - x86_64 and arm64", "macosx_arm64": "macOS arm64 - Apple Silicon", "pyodide_wasm32": "Pyodide", + "android_arm64_v8a": "Android arm64_v8a", + "android_x86_64": "Android x86_64", "ios_arm64_iphoneos": "iOS Device (ARM64)", "ios_arm64_iphonesimulator": "iOS Simulator (ARM64)", "ios_x86_64_iphonesimulator": "iOS Simulator (x86_64)", diff --git a/cibuildwheel/platforms/__init__.py b/cibuildwheel/platforms/__init__.py index a56fb9036..ef78f1407 100644 --- a/cibuildwheel/platforms/__init__.py +++ b/cibuildwheel/platforms/__init__.py @@ -6,7 +6,7 @@ from cibuildwheel.architecture import Architecture from cibuildwheel.options import Options -from cibuildwheel.platforms import ios, linux, macos, pyodide, windows +from cibuildwheel.platforms import android, ios, linux, macos, pyodide, windows from cibuildwheel.selector import BuildSelector from cibuildwheel.typing import GenericPythonConfiguration, PlatformName @@ -28,6 +28,7 @@ def build(self, options: Options, tmp_path: Path) -> None: ... "windows": windows, "macos": macos, "pyodide": pyodide, + "android": android, "ios": ios, } diff --git a/cibuildwheel/typing.py b/cibuildwheel/typing.py index 420ef05f6..53b0887da 100644 --- a/cibuildwheel/typing.py +++ b/cibuildwheel/typing.py @@ -12,7 +12,7 @@ PathOrStr = str | os.PathLike[str] -PlatformName = Literal["linux", "macos", "windows", "pyodide", "ios"] +PlatformName = Literal["linux", "macos", "windows", "pyodide", "android", "ios"] PLATFORMS: Final[frozenset[PlatformName]] = frozenset(typing.get_args(PlatformName)) diff --git a/cibuildwheel/util/packaging.py b/cibuildwheel/util/packaging.py index f6619dd92..573a1786b 100644 --- a/cibuildwheel/util/packaging.py +++ b/cibuildwheel/util/packaging.py @@ -157,9 +157,9 @@ def find_compatible_wheel(wheels: Sequence[T], identifier: str) -> T | None: # If a minor version number is given, it has to be lower than the current one. continue - if platform.startswith(("manylinux", "musllinux", "macosx", "ios")): - # Linux, macOS, and iOS require the beginning and ending match - # (macos/manylinux/iOS version number doesn't need to match) + if platform.startswith(("manylinux", "musllinux", "macosx", "android", "ios")): + # On these platforms the wheel tag includes a platform version number, which we + # should ignore. os_, arch = platform.split("_", 1) if not tag.platform.startswith(os_): continue From c720392578d128bbb03585c02f288178365c7ba2 Mon Sep 17 00:00:00 2001 From: Malcolm Smith Date: Tue, 15 Apr 2025 23:22:16 +0100 Subject: [PATCH 03/57] Add Android documentation --- README.md | 45 +++++++++++----------- docs/options.md | 78 +++++++++++++++++++-------------------- docs/platforms/android.md | 73 ++++++++++++++++++++++++++++++++++++ docs/setup.md | 3 +- mkdocs.yml | 1 + 5 files changed, 138 insertions(+), 62 deletions(-) create mode 100644 docs/platforms/android.md diff --git a/README.md b/README.md index c70d40f9b..e298589b1 100644 --- a/README.md +++ b/README.md @@ -24,18 +24,18 @@ What does it do? While cibuildwheel itself requires a recent Python version to run (we support the last three releases), it can target the following versions to build wheels: -| | macOS Intel | macOS Apple Silicon | Windows 64bit | Windows 32bit | Windows Arm64 | manylinux
musllinux x86_64 | manylinux
musllinux i686 | manylinux
musllinux aarch64 | manylinux
musllinux ppc64le | manylinux
musllinux s390x | manylinux
musllinux armv7l | iOS | Pyodide | -|----------------|----|-----|-----|-----|-----|----|-----|----|-----|-----|---|-----|-----| -| CPython 3.8 | ✅ | ✅ | ✅ | ✅ | N/A | ✅ | ✅ | ✅ | ✅ | ✅ | ✅⁵ | N/A | N/A | -| CPython 3.9 | ✅ | ✅ | ✅ | ✅ | ✅² | ✅ | ✅ | ✅ | ✅ | ✅ | ✅⁵ | N/A | N/A | -| CPython 3.10 | ✅ | ✅ | ✅ | ✅ | ✅² | ✅ | ✅ | ✅ | ✅ | ✅ | ✅⁵ | N/A | N/A | -| CPython 3.11 | ✅ | ✅ | ✅ | ✅ | ✅² | ✅ | ✅ | ✅ | ✅ | ✅ | ✅⁵ | N/A | N/A | -| CPython 3.12 | ✅ | ✅ | ✅ | ✅ | ✅² | ✅ | ✅ | ✅ | ✅ | ✅ | ✅⁵ | N/A | ✅⁴ | -| CPython 3.13³ | ✅ | ✅ | ✅ | ✅ | ✅² | ✅ | ✅ | ✅ | ✅ | ✅ | ✅⁵ | ✅ | N/A | -| PyPy 3.8 v7.3 | ✅ | ✅ | ✅ | N/A | N/A | ✅¹ | ✅¹ | ✅¹ | N/A | N/A | N/A | N/A | N/A | -| PyPy 3.9 v7.3 | ✅ | ✅ | ✅ | N/A | N/A | ✅¹ | ✅¹ | ✅¹ | N/A | N/A | N/A | N/A | N/A | -| PyPy 3.10 v7.3 | ✅ | ✅ | ✅ | N/A | N/A | ✅¹ | ✅¹ | ✅¹ | N/A | N/A | N/A | N/A | N/A | -| PyPy 3.11 v7.3 | ✅ | ✅ | ✅ | N/A | N/A | ✅¹ | ✅¹ | ✅¹ | N/A | N/A | N/A | N/A | N/A | +| | macOS Intel | macOS Apple Silicon | Windows 64bit | Windows 32bit | Windows Arm64 | manylinux
musllinux x86_64 | manylinux
musllinux i686 | manylinux
musllinux aarch64 | manylinux
musllinux ppc64le | manylinux
musllinux s390x | manylinux
musllinux armv7l | Android | iOS | Pyodide | +|----------------|----|-----|----|-----|-----|----|-----|----|-----|-----|---|-----|-----|-----| +| CPython 3.8 | ✅ | ✅ | ✅ | ✅ | N/A | ✅ | ✅ | ✅ | ✅ | ✅ | ✅⁵ | N/A | N/A | N/A | +| CPython 3.9 | ✅ | ✅ | ✅ | ✅ | ✅² | ✅ | ✅ | ✅ | ✅ | ✅ | ✅⁵ | N/A | N/A | N/A | +| CPython 3.10 | ✅ | ✅ | ✅ | ✅ | ✅² | ✅ | ✅ | ✅ | ✅ | ✅ | ✅⁵ | N/A | N/A | N/A | +| CPython 3.11 | ✅ | ✅ | ✅ | ✅ | ✅² | ✅ | ✅ | ✅ | ✅ | ✅ | ✅⁵ | N/A | N/A | N/A | +| CPython 3.12 | ✅ | ✅ | ✅ | ✅ | ✅² | ✅ | ✅ | ✅ | ✅ | ✅ | ✅⁵ | N/A | N/A | ✅⁴ | +| CPython 3.13³ | ✅ | ✅ | ✅ | ✅ | ✅² | ✅ | ✅ | ✅ | ✅ | ✅ | ✅⁵ | ✅ | ✅ | N/A | +| PyPy 3.8 v7.3 | ✅ | ✅ | ✅ | N/A | N/A | ✅¹ | ✅¹ | ✅¹ | N/A | N/A | N/A | N/A | N/A | N/A | +| PyPy 3.9 v7.3 | ✅ | ✅ | ✅ | N/A | N/A | ✅¹ | ✅¹ | ✅¹ | N/A | N/A | N/A | N/A | N/A | N/A | +| PyPy 3.10 v7.3 | ✅ | ✅ | ✅ | N/A | N/A | ✅¹ | ✅¹ | ✅¹ | N/A | N/A | N/A | N/A | N/A | N/A | +| PyPy 3.11 v7.3 | ✅ | ✅ | ✅ | N/A | N/A | ✅¹ | ✅¹ | ✅¹ | N/A | N/A | N/A | N/A | N/A | N/A | ¹ PyPy is only supported for manylinux wheels.
² Windows arm64 support is experimental.
@@ -55,20 +55,21 @@ Usage `cibuildwheel` runs inside a CI service. Supported platforms depend on which service you're using: -| | Linux | macOS | Windows | Linux ARM | macOS ARM | Windows ARM | iOS | -|-----------------|-------|-------|---------|-----------|-----------|-------------|-----| -| GitHub Actions | ✅ | ✅ | ✅ | ✅ | ✅ | ✅² | ✅³ | -| Azure Pipelines | ✅ | ✅ | ✅ | | ✅ | ✅² | ✅³ | -| Travis CI | ✅ | | ✅ | ✅ | | | | -| AppVeyor | ✅ | ✅ | ✅ | | ✅ | ✅² | ✅³ | -| CircleCI | ✅ | ✅ | | ✅ | ✅ | | ✅³ | -| Gitlab CI | ✅ | ✅ | ✅ | ✅¹ | ✅ | | ✅³ | -| Cirrus CI | ✅ | ✅ | ✅ | ✅ | ✅ | | ✅³ | +| | Linux | macOS | Windows | Linux ARM | macOS ARM | Windows ARM | Android | iOS | +|-----------------|-------|-------|---------|-----------|-----------|-------------|---------|-----| +| GitHub Actions | ✅ | ✅ | ✅ | ✅ | ✅ | ✅² | ✅⁴ | ✅³ | +| Azure Pipelines | ✅ | ✅ | ✅ | | ✅ | ✅² | ✅⁴ | ✅³ | +| Travis CI | ✅ | | ✅ | ✅ | | | ✅⁴ | | +| AppVeyor | ✅ | ✅ | ✅ | | ✅ | ✅² | ✅⁴ | ✅³ | +| CircleCI | ✅ | ✅ | | ✅ | ✅ | | ✅⁴ | ✅³ | +| Gitlab CI | ✅ | ✅ | ✅ | ✅¹ | ✅ | | ✅⁴ | ✅³ | +| Cirrus CI | ✅ | ✅ | ✅ | ✅ | ✅ | | ✅⁴ | ✅³ | ¹ [Requires emulation](https://cibuildwheel.pypa.io/en/stable/faq/#emulation), distributed separately. Other services may also support Linux ARM through emulation or third-party build hosts, but these are not tested in our CI.
² [Uses cross-compilation](https://cibuildwheel.pypa.io/en/stable/faq/#windows-arm64). It is not possible to test `arm64` on this CI platform.
³ Requires a macOS runner; runs tests on the simulator for the runner's architecture. - +⁴ Requires runner to be Linux x86_64, macOS ARM64 or macOS x86_64. Runs tests on the emulator +for the runner's architecture. Example setup diff --git a/docs/options.md b/docs/options.md index 0c4ade208..bf3e5374e 100644 --- a/docs/options.md +++ b/docs/options.md @@ -247,7 +247,7 @@ environment variables will completely override any TOML configuration. > Override the auto-detected target platform -Options: `auto` `linux` `macos` `windows` `ios` `pyodide` +Options: `auto` `linux` `macos` `windows` `android` `ios` `pyodide` Default: `auto` @@ -255,6 +255,7 @@ Default: `auto` - For `linux`, you need [Docker or Podman](#container-engine) running, on Linux, macOS, or Windows. - For `macos` and `windows`, you need to be running on the respective system, with a working compiler toolchain installed - Xcode Command Line tools for macOS, and MSVC for Windows. +- For `android` you need to be running on Linux or macOS, with an Android SDK installed. See [here](android.md) for more details. - For `ios` you need to be running on macOS, with Xcode and the iOS simulator installed. - For `pyodide` you need to be on an x86-64 linux runner and `python3.12` must be available in `PATH`. @@ -289,18 +290,18 @@ When setting the options, you can use shell-style globbing syntax, as per [fnmat
-| | macOS | Windows | Linux Intel | Linux Other | iOS | -|---------------|------------------------------------------------------------------------|-----------------------------------------------------|-----------------------------------------------------------------------------------------------------|-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|---------------------------------------------------------------------------------------------------| -| Python 3.8 | cp38-macosx_x86_64
cp38-macosx_universal2
cp38-macosx_arm64 | cp38-win_amd64
cp38-win32 | cp38-manylinux_x86_64
cp38-manylinux_i686
cp38-musllinux_x86_64
cp38-musllinux_i686 | cp38-manylinux_aarch64
cp38-manylinux_ppc64le
cp38-manylinux_s390x
cp38-manylinux_armv7l
cp38-musllinux_aarch64
cp38-musllinux_ppc64le
cp38-musllinux_s390x
cp38-musllinux_armv7l | | -| Python 3.9 | cp39-macosx_x86_64
cp39-macosx_universal2
cp39-macosx_arm64 | cp39-win_amd64
cp39-win32
cp39-win_arm64 | cp39-manylinux_x86_64
cp39-manylinux_i686
cp39-musllinux_x86_64
cp39-musllinux_i686 | cp39-manylinux_aarch64
cp39-manylinux_ppc64le
cp39-manylinux_s390x
cp39-manylinux_armv7l
cp39-musllinux_aarch64
cp39-musllinux_ppc64le
cp39-musllinux_s390x
cp39-musllinux_armv7l | | -| Python 3.10 | cp310-macosx_x86_64
cp310-macosx_universal2
cp310-macosx_arm64 | cp310-win_amd64
cp310-win32
cp310-win_arm64 | cp310-manylinux_x86_64
cp310-manylinux_i686
cp310-musllinux_x86_64
cp310-musllinux_i686 | cp310-manylinux_aarch64
cp310-manylinux_ppc64le
cp310-manylinux_s390x
cp310-manylinux_armv7l
cp310-musllinux_aarch64
cp310-musllinux_ppc64le
cp310-musllinux_s390x
cp310-musllinux_armv7l | | -| Python 3.11 | cp311-macosx_x86_64
cp311-macosx_universal2
cp311-macosx_arm64 | cp311-win_amd64
cp311-win32
cp311-win_arm64 | cp311-manylinux_x86_64
cp311-manylinux_i686
cp311-musllinux_x86_64
cp311-musllinux_i686 | cp311-manylinux_aarch64
cp311-manylinux_ppc64le
cp311-manylinux_s390x
cp311-manylinux_armv7l
cp311-musllinux_aarch64
cp311-musllinux_ppc64le
cp311-musllinux_s390x
cp311-musllinux_armv7l | | -| Python 3.12 | cp312-macosx_x86_64
cp312-macosx_universal2
cp312-macosx_arm64 | cp312-win_amd64
cp312-win32
cp312-win_arm64 | cp312-manylinux_x86_64
cp312-manylinux_i686
cp312-musllinux_x86_64
cp312-musllinux_i686 | cp312-manylinux_aarch64
cp312-manylinux_ppc64le
cp312-manylinux_s390x
cp312-musllinux_armv7l
cp312-musllinux_ppc64le
cp312-musllinux_s390x
cp312-musllinux_armv7l | | -| Python 3.13 | cp313-macosx_x86_64
cp313-macosx_universal2
cp313-macosx_arm64 | cp313-win_amd64
cp313-win32
cp313-win_arm64 | cp313-manylinux_x86_64
cp313-manylinux_i686
cp313-musllinux_x86_64
cp313-musllinux_i686 | cp313-manylinux_aarch64
cp313-manylinux_ppc64le
cp313-manylinux_s390x
cp313-manylinux_armv7l
cp313-musllinux_aarch64
cp313-musllinux_ppc64le
cp313-musllinux_s390x
cp313-musllinux_armv7l | cp313-ios_arm64_iphoneos
cp313-ios_arm64_iphonesimulator
cp313-ios_x86_64_iphonesimulator | -| PyPy3.8 v7.3 | pp38-macosx_x86_64
pp38-macosx_arm64 | pp38-win_amd64 | pp38-manylinux_x86_64
pp38-manylinux_i686 | pp38-manylinux_aarch64 | | -| PyPy3.9 v7.3 | pp39-macosx_x86_64
pp39-macosx_arm64 | pp39-win_amd64 | pp39-manylinux_x86_64
pp39-manylinux_i686 | pp39-manylinux_aarch64 | | -| PyPy3.10 v7.3 | pp310-macosx_x86_64
pp310-macosx_arm64 | pp310-win_amd64 | pp310-manylinux_x86_64
pp310-manylinux_i686 | pp310-manylinux_aarch64 | | -| PyPy3.11 v7.3 | pp311-macosx_x86_64
pp311-macosx_arm64 | pp311-win_amd64 | pp311-manylinux_x86_64
pp311-manylinux_i686 | pp311-manylinux_aarch64 | | +| | macOS | Windows | Linux Intel | Linux Other | Android | iOS | +|---------------|------------------------------------------------------------------------|-----------------------------------------------------|-----------------------------------------------------------------------------------------------------|-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|--------------------------------------------------|---------------------------------------------------------------------------------------------------| +| Python 3.8 | cp38-macosx_x86_64
cp38-macosx_universal2
cp38-macosx_arm64 | cp38-win_amd64
cp38-win32 | cp38-manylinux_x86_64
cp38-manylinux_i686
cp38-musllinux_x86_64
cp38-musllinux_i686 | cp38-manylinux_aarch64
cp38-manylinux_ppc64le
cp38-manylinux_s390x
cp38-manylinux_armv7l
cp38-musllinux_aarch64
cp38-musllinux_ppc64le
cp38-musllinux_s390x
cp38-musllinux_armv7l | | | +| Python 3.9 | cp39-macosx_x86_64
cp39-macosx_universal2
cp39-macosx_arm64 | cp39-win_amd64
cp39-win32
cp39-win_arm64 | cp39-manylinux_x86_64
cp39-manylinux_i686
cp39-musllinux_x86_64
cp39-musllinux_i686 | cp39-manylinux_aarch64
cp39-manylinux_ppc64le
cp39-manylinux_s390x
cp39-manylinux_armv7l
cp39-musllinux_aarch64
cp39-musllinux_ppc64le
cp39-musllinux_s390x
cp39-musllinux_armv7l | | | +| Python 3.10 | cp310-macosx_x86_64
cp310-macosx_universal2
cp310-macosx_arm64 | cp310-win_amd64
cp310-win32
cp310-win_arm64 | cp310-manylinux_x86_64
cp310-manylinux_i686
cp310-musllinux_x86_64
cp310-musllinux_i686 | cp310-manylinux_aarch64
cp310-manylinux_ppc64le
cp310-manylinux_s390x
cp310-manylinux_armv7l
cp310-musllinux_aarch64
cp310-musllinux_ppc64le
cp310-musllinux_s390x
cp310-musllinux_armv7l | | | +| Python 3.11 | cp311-macosx_x86_64
cp311-macosx_universal2
cp311-macosx_arm64 | cp311-win_amd64
cp311-win32
cp311-win_arm64 | cp311-manylinux_x86_64
cp311-manylinux_i686
cp311-musllinux_x86_64
cp311-musllinux_i686 | cp311-manylinux_aarch64
cp311-manylinux_ppc64le
cp311-manylinux_s390x
cp311-manylinux_armv7l
cp311-musllinux_aarch64
cp311-musllinux_ppc64le
cp311-musllinux_s390x
cp311-musllinux_armv7l | | | +| Python 3.12 | cp312-macosx_x86_64
cp312-macosx_universal2
cp312-macosx_arm64 | cp312-win_amd64
cp312-win32
cp312-win_arm64 | cp312-manylinux_x86_64
cp312-manylinux_i686
cp312-musllinux_x86_64
cp312-musllinux_i686 | cp312-manylinux_aarch64
cp312-manylinux_ppc64le
cp312-manylinux_s390x
cp312-musllinux_armv7l
cp312-musllinux_ppc64le
cp312-musllinux_s390x
cp312-musllinux_armv7l | | | +| Python 3.13 | cp313-macosx_x86_64
cp313-macosx_universal2
cp313-macosx_arm64 | cp313-win_amd64
cp313-win32
cp313-win_arm64 | cp313-manylinux_x86_64
cp313-manylinux_i686
cp313-musllinux_x86_64
cp313-musllinux_i686 | cp313-manylinux_aarch64
cp313-manylinux_ppc64le
cp313-manylinux_s390x
cp313-manylinux_armv7l
cp313-musllinux_aarch64
cp313-musllinux_ppc64le
cp313-musllinux_s390x
cp313-musllinux_armv7l | cp313-android_arm64_v8a
cp313-android_x86_64 | cp313-ios_arm64_iphoneos
cp313-ios_arm64_iphonesimulator
cp313-ios_x86_64_iphonesimulator | +| PyPy3.8 v7.3 | pp38-macosx_x86_64
pp38-macosx_arm64 | pp38-win_amd64 | pp38-manylinux_x86_64
pp38-manylinux_i686 | pp38-manylinux_aarch64 | | | +| PyPy3.9 v7.3 | pp39-macosx_x86_64
pp39-macosx_arm64 | pp39-win_amd64 | pp39-manylinux_x86_64
pp39-manylinux_i686 | pp39-manylinux_aarch64 | | | +| PyPy3.10 v7.3 | pp310-macosx_x86_64
pp310-macosx_arm64 | pp310-win_amd64 | pp310-manylinux_x86_64
pp310-manylinux_i686 | pp310-manylinux_aarch64 | | | +| PyPy3.11 v7.3 | pp311-macosx_x86_64
pp311-macosx_arm64 | pp311-win_amd64 | pp311-manylinux_x86_64
pp311-manylinux_i686 | pp311-manylinux_aarch64 | | | The list of supported and currently selected build identifiers can also be retrieved by passing the `--print-build-identifiers` flag to cibuildwheel. The format is `python_tag-platform_tag`, with tags similar to those in [PEP 425](https://www.python.org/dev/peps/pep-0425/#details). @@ -424,6 +425,7 @@ Options: - macOS: `x86_64` `arm64` `universal2` - Windows: `AMD64` `x86` `ARM64` - Pyodide: `wasm32` +- Android: `arm64_v8a` `x86_64` - iOS: `arm64_iphoneos` `arm64_iphonesimulator` `x86_64_iphonesimulator` - `auto`: The default archs for your machine - see the table below. - `auto64`: Just the 64-bit auto archs @@ -451,7 +453,7 @@ If not listed above, `auto` is the same as `native`. [binfmt]: https://hub.docker.com/r/tonistiigi/binfmt Platform-specific environment variables are also available:
- `CIBW_ARCHS_MACOS` | `CIBW_ARCHS_WINDOWS` | `CIBW_ARCHS_LINUX` | `CIBW_ARCHS_IOS` +`CIBW_ARCHS_MACOS` | `CIBW_ARCHS_WINDOWS` | `CIBW_ARCHS_LINUX` | `CIBW_ARCHS_ANDROID` | `CIBW_ARCHS_IOS` This option can also be set using the [command-line option](#command-line) `--archs`. This option cannot be set in an `overrides` section in `pyproject.toml`. @@ -666,18 +668,18 @@ possible, both through `--installer=uv` passed to build, as well as when making all build and test environments. This will generally speed up cibuildwheel. Make sure you have an external uv on Windows and macOS, either by pre-installing it, or installing cibuildwheel with the uv extra, -`cibuildwheel[uv]`. You cannot use uv currently on Windows for ARM, for -musllinux on s390x, or for iOS, as binaries are not provided by uv. Legacy dependencies like +`cibuildwheel[uv]`. uv currently does not support Windows ARM, +musllinux s390x, Android, or iOS. Legacy dependencies like setuptools on Python < 3.12 and pip are not installed if using uv. -Pyodide ignores this setting, as only "build" is supported. +On Android and Pyodide, only "build" is supported. You can specify extra arguments to pass to `pip wheel` or `build` using the optional `args` option. !!! tip - Until v2.0.0, [pip][] was the only way to build wheels, and is still the - default. However, we expect that at some point in the future, cibuildwheel + Until v2.0.0, [pip][] was the only way to build wheels, and is still the default + on most platforms. However, we expect that at some point in the future, cibuildwheel will change the default to [build][], in line with the PyPA's recommendation. If you want to try `build` before this, you can use this option. @@ -744,7 +746,7 @@ a table of items, including arrays. single values. Platform-specific environment variables also available:
-`CIBW_CONFIG_SETTINGS_MACOS` | `CIBW_CONFIG_SETTINGS_WINDOWS` | `CIBW_CONFIG_SETTINGS_LINUX` | `CIBW_CONFIG_SETTINGS_IOS` | `CIBW_CONFIG_SETTINGS_PYODIDE` +`CIBW_CONFIG_SETTINGS_MACOS` | `CIBW_CONFIG_SETTINGS_WINDOWS` | `CIBW_CONFIG_SETTINGS_LINUX` | `CIBW_CONFIG_SETTINGS_ANDROID` | `CIBW_CONFIG_SETTINGS_IOS` | `CIBW_CONFIG_SETTINGS_PYODIDE` #### Examples @@ -775,7 +777,7 @@ You can use `$PATH` syntax to insert other variables, or the `$(pwd)` syntax to To specify more than one environment variable, separate the assignments by spaces. Platform-specific environment variables are also available:
-`CIBW_ENVIRONMENT_MACOS` | `CIBW_ENVIRONMENT_WINDOWS` | `CIBW_ENVIRONMENT_LINUX` | `CIBW_ENVIRONMENT_IOS` | `CIBW_ENVIRONMENT_PYODIDE` +`CIBW_ENVIRONMENT_MACOS` | `CIBW_ENVIRONMENT_WINDOWS` | `CIBW_ENVIRONMENT_LINUX` | `CIBW_ENVIRONMENT_ANDROID` | `CIBW_ENVIRONMENT_IOS` | `CIBW_ENVIRONMENT_PYODIDE` #### Examples @@ -907,7 +909,7 @@ On linux, overriding it triggers a new container launch. It cannot be overridden on macOS and Windows. Platform-specific environment variables also available:
-`CIBW_BEFORE_ALL_MACOS` | `CIBW_BEFORE_ALL_WINDOWS` | `CIBW_BEFORE_ALL_LINUX` | `CIBW_BEFORE_ALL_IOS` | `CIBW_BEFORE_ALL_PYODIDE` +`CIBW_BEFORE_ALL_MACOS` | `CIBW_BEFORE_ALL_WINDOWS` | `CIBW_BEFORE_ALL_LINUX` | `CIBW_BEFORE_ALL_ANDROID` | `CIBW_BEFORE_ALL_IOS` | `CIBW_BEFORE_ALL_PYODIDE` !!! note @@ -971,7 +973,7 @@ The active Python binary can be accessed using `python`, and pip with `pip`; cib The command is run in a shell, so you can write things like `cmd1 && cmd2`. Platform-specific environment variables are also available:
- `CIBW_BEFORE_BUILD_MACOS` | `CIBW_BEFORE_BUILD_WINDOWS` | `CIBW_BEFORE_BUILD_LINUX` | `CIBW_BEFORE_BUILD_IOS` | `CIBW_BEFORE_BUILD_PYODIDE` + `CIBW_BEFORE_BUILD_MACOS` | `CIBW_BEFORE_BUILD_WINDOWS` | `CIBW_BEFORE_BUILD_LINUX` | `CIBW_BEFORE_BUILD_ANDROID` | `CIBW_BEFORE_BUILD_IOS` | `CIBW_BEFORE_BUILD_PYODIDE` #### Examples @@ -1051,9 +1053,7 @@ Default: - on Linux: `'auditwheel repair -w {dest_dir} {wheel}'` - on macOS: `'delocate-wheel --require-archs {delocate_archs} -w {dest_dir} -v {wheel}'` -- on Windows: `''` -- on iOS: `''` -- on Pyodide: `''` +- on other platforms: `''` A shell command to repair a built wheel by copying external library dependencies into the wheel tree and relinking them. The command is run on each built wheel (except for pure Python ones) before testing it. @@ -1067,7 +1067,7 @@ The following placeholders must be used inside the command and will be replaced The command is run in a shell, so you can run multiple commands like `cmd1 && cmd2`. Platform-specific environment variables are also available:
-`CIBW_REPAIR_WHEEL_COMMAND_MACOS` | `CIBW_REPAIR_WHEEL_COMMAND_WINDOWS` | `CIBW_REPAIR_WHEEL_COMMAND_LINUX` | `CIBW_REPAIR_WHEEL_COMMAND_IOS` | `CIBW_REPAIR_WHEEL_COMMAND_PYODIDE` +`CIBW_REPAIR_WHEEL_COMMAND_MACOS` | `CIBW_REPAIR_WHEEL_COMMAND_WINDOWS` | `CIBW_REPAIR_WHEEL_COMMAND_LINUX` | `CIBW_REPAIR_WHEEL_COMMAND_ANDROID` | `CIBW_REPAIR_WHEEL_COMMAND_IOS` | `CIBW_REPAIR_WHEEL_COMMAND_PYODIDE` !!! tip cibuildwheel doesn't yet ship a default repair command for Windows. @@ -1364,7 +1364,7 @@ specifiers inline with the `packages: SPECIFIER...` syntax. `./constraints.txt` if that's not found. Platform-specific environment variables are also available:
-`CIBW_DEPENDENCY_VERSIONS_MACOS` | `CIBW_DEPENDENCY_VERSIONS_WINDOWS` | `CIBW_DEPENDENCY_VERSIONS_IOS` | `CIBW_DEPENDENCY_VERSIONS_PYODIDE` +`CIBW_DEPENDENCY_VERSIONS_MACOS` | `CIBW_DEPENDENCY_VERSIONS_WINDOWS` | `CIBW_DEPENDENCY_VERSIONS_ANDROID` | `CIBW_DEPENDENCY_VERSIONS_IOS` | `CIBW_DEPENDENCY_VERSIONS_PYODIDE` !!! note This option does not affect the tools used on the Linux build - those versions @@ -1440,12 +1440,12 @@ Alternatively, you can use the [`CIBW_TEST_SOURCES`](#test-sources) setting to create a temporary folder populated with a specific subset of project files to run your test suite. -On all platforms other than iOS, the command is run in a shell, so you can write things like `cmd1 && cmd2`. +On all platforms other than Android and iOS, the command is run in a shell, so you can write things like `cmd1 && cmd2`. -On iOS, the value of the `CIBW_TEST_COMMAND` setting is interpreted as the arguments to pass to `python -m` - that is, a Python module name, followed by arguments that will be assigned to `sys.argv`. Shell commands cannot be used. +On Android and iOS, the value of the `CIBW_TEST_COMMAND` setting is interpreted as the arguments to pass to `python -m` - that is, a Python module name, followed by arguments that will be assigned to `sys.argv`. Shell commands cannot be used. Platform-specific environment variables are also available:
-`CIBW_TEST_COMMAND_MACOS` | `CIBW_TEST_COMMAND_WINDOWS` | `CIBW_TEST_COMMAND_LINUX` | `CIBW_TEST_COMMAND_IOS` | `CIBW_TEST_COMMAND_PYODIDE` +`CIBW_TEST_COMMAND_MACOS` | `CIBW_TEST_COMMAND_WINDOWS` | `CIBW_TEST_COMMAND_LINUX` | `CIBW_TEST_COMMAND_ANDROID` | `CIBW_TEST_COMMAND_IOS` | `CIBW_TEST_COMMAND_PYODIDE` #### Examples @@ -1502,7 +1502,7 @@ The active Python binary can be accessed using `python`, and pip with `pip`; cib The command is run in a shell, so you can write things like `cmd1 && cmd2`. Platform-specific environment variables are also available:
- `CIBW_BEFORE_TEST_MACOS` | `CIBW_BEFORE_TEST_WINDOWS` | `CIBW_BEFORE_TEST_LINUX` | `CIBW_BEFORE_TEST_IOS` | `CIBW_BEFORE_TEST_PYODIDE` + `CIBW_BEFORE_TEST_MACOS` | `CIBW_BEFORE_TEST_WINDOWS` | `CIBW_BEFORE_TEST_LINUX` | `CIBW_BEFORE_TEST_ANDROID` | `CIBW_BEFORE_TEST_IOS` | `CIBW_BEFORE_TEST_PYODIDE` #### Examples @@ -1564,13 +1564,13 @@ project, required for running the tests. If specified, these files and folders will be copied into a temporary folder, and that temporary folder will be used as the working directory for running the test suite. -The use of `CIBW_TEST_SOURCES` is *required* for iOS builds. This is because the -simulator does not have access to the project directory, as it is not stored on -the simulator device. On iOS, the files will be copied into the test application, +The use of `CIBW_TEST_SOURCES` is *required* for Android and iOS tests, because +they run in a virtual machine that does not have access to the project directory. +On these platforms, the files will be copied into the test application, rather than a temporary folder. Platform-specific environment variables are also available:
-`CIBW_TEST_SOURCES_MACOS` | `CIBW_TEST_SOURCES_WINDOWS` | `CIBW_TEST_SOURCES_LINUX` | `CIBW_TEST_SOURCES_IOS` | `CIBW_TEST_SOURCES_PYODIDE` +`CIBW_TEST_SOURCES_MACOS` | `CIBW_TEST_SOURCES_WINDOWS` | `CIBW_TEST_SOURCES_LINUX` | `CIBW_TEST_SOURCES_ANDROID` | `CIBW_TEST_SOURCES_IOS` | `CIBW_TEST_SOURCES_PYODIDE` #### Examples @@ -1598,7 +1598,7 @@ Platform-specific environment variables are also available:
Space-separated list of dependencies required for running the tests. Platform-specific environment variables are also available:
-`CIBW_TEST_REQUIRES_MACOS` | `CIBW_TEST_REQUIRES_WINDOWS` | `CIBW_TEST_REQUIRES_LINUX` | `CIBW_TEST_REQUIRES_IOS` | `CIBW_TEST_REQUIRES_PYODIDE` +`CIBW_TEST_REQUIRES_MACOS` | `CIBW_TEST_REQUIRES_WINDOWS` | `CIBW_TEST_REQUIRES_LINUX` | `CIBW_TEST_REQUIRES_ANDROID` | `CIBW_TEST_REQUIRES_IOS` | `CIBW_TEST_REQUIRES_PYODIDE` #### Examples @@ -1638,7 +1638,7 @@ tests. This can be used to avoid having to redefine test dependencies in `setup.cfg` or `setup.py`. Platform-specific environment variables are also available:
-`CIBW_TEST_EXTRAS_MACOS` | `CIBW_TEST_EXTRAS_WINDOWS` | `CIBW_TEST_EXTRAS_LINUX` | `CIBW_TEST_EXTRAS_IOS` | `CIBW_TEST_EXTRAS_PYODIDE` +`CIBW_TEST_EXTRAS_MACOS` | `CIBW_TEST_EXTRAS_WINDOWS` | `CIBW_TEST_EXTRAS_LINUX` | `CIBW_TEST_EXTRAS_ANDROID` | `CIBW_TEST_EXTRAS_IOS` | `CIBW_TEST_EXTRAS_PYODIDE` #### Examples @@ -1765,7 +1765,7 @@ export CIBW_DEBUG_TRACEBACK=TRUE A number from 1 to 3 to increase the level of verbosity (corresponding to invoking pip with `-v`, `-vv`, and `-vvv`), between -1 and -3 (`-q`, `-qq`, and `-qqq`), or just 0 (default verbosity). These flags are useful while debugging a build when the output of the actual build invoked by `pip wheel` is required. Has no effect on the `build` backend, which produces verbose output by default. Platform-specific environment variables are also available:
-`CIBW_BUILD_VERBOSITY_MACOS` | `CIBW_BUILD_VERBOSITY_WINDOWS` | `CIBW_BUILD_VERBOSITY_LINUX` | `CIBW_BUILD_VERBOSITY_IOS` | `CIBW_BUILD_VERBOSITY_PYODIDE` +`CIBW_BUILD_VERBOSITY_MACOS` | `CIBW_BUILD_VERBOSITY_WINDOWS` | `CIBW_BUILD_VERBOSITY_LINUX` | `CIBW_BUILD_VERBOSITY_ANDROID` | `CIBW_BUILD_VERBOSITY_IOS` | `CIBW_BUILD_VERBOSITY_PYODIDE` #### Examples diff --git a/docs/platforms/android.md b/docs/platforms/android.md new file mode 100644 index 000000000..2c9008aa5 --- /dev/null +++ b/docs/platforms/android.md @@ -0,0 +1,73 @@ +# Android builds + +## Prerequisites + +cibuildwheel can build and test Android wheels on any POSIX platform supported by the +Android development tools, which currently means Linux x86_64, macOS ARM64 or macOS +x86_64. + +Building Android wheels requires the build machine to have a working Python executable +of the same version. See the notes on [Linux](linux.md) and [macOS](macos.md) for +details of how this is installed. + +If you already have an Android SDK, export the `ANDROID_HOME` environment variable to +point at its location. Otherwise, here's how to install it: + +* Download the "Command line tools" from . +* Create a directory `android-sdk/cmdline-tools`, and unzip the command line + tools package into it. +* Rename `android-sdk/cmdline-tools/cmdline-tools` to + `android-sdk/cmdline-tools/latest`. +* `export ANDROID_HOME=/path/to/android-sdk` + +cibuildwheel will automatically use the SDK's `sdkmanager` to install any packages it +needs. + +It also requires the following commands to be on the `PATH`: + +* `curl` +* `java` (or set the `JAVA_HOME` environment variable) + +## Android version compatibility + +Android builds will honor the `api_level` environment variable to set the minimum +supported [API level](https://developer.android.com/tools/releases/platforms) for +generated wheels. This will default to the minimum API level of the selected Python +version. + +## Build frontend support + +Android builds only support the `build` frontend. In principle, support for the +`build[uv]` frontend should be possible, but `uv` [doesn't currently have support for +cross-platform builds](https://github.com/astral-sh/uv/issues/7957), and [doesn't have +support for iOS or Android wheel tags](https://github.com/astral-sh/uv/issues/8029). + +## Cross platform builds + +Android builds are *cross platform builds*, as cibuildwheel does not support running +compilers and other build tools "on device". The supported build platforms (listed +above) can be used to build wheels for any supported Android architecture. However, +wheels can only be *tested* on a machine of the same architecture – see the section +below. + +## Tests + +If tests have been configured, the test suite will be executed on a Gradle-managed +emulator matching the architecture of the build machine - for example, if you're +building on an ARM64 machine, an ARM64 wheel can be tested on an ARM64 emulator. +Cross-architecture testing is not supported. + +On Linux, the emulator needs access to the KVM virtualization interface, and a DISPLAY +environment variable pointing at an X server. Xvfb is acceptable. + +The Android test environment can't support running shell scripts, so the +[`CIBW_TEST_COMMAND`](../options.md#test-command) value must be specified as if it were +a command line being passed to `python -m ...`. In addition, the project must use +[`CIBW_TEST_SOURCES`](../options.md#test-sources) to specify the minimum subset of files +that should be copied to the test environment. This is because the test must be run "on +device", and the device will not have access to the local project directory. + +The test process uses the same testbed used by CPython itself to run the CPython test +suite. It is a Gradle project that has been configured to have a single JUnit - the +result of which reports the success or failure of running `python -m +`. diff --git a/docs/setup.md b/docs/setup.md index e82f5a3e8..121b058d5 100644 --- a/docs/setup.md +++ b/docs/setup.md @@ -11,6 +11,7 @@ Each platform that cibuildwheel supports has its own prerequisites and platform- * [Linux](./platforms/linux.md) * [Windows](./platforms/windows.md) * [macOS](./platforms/macos.md) +* [Android](./platforms/android.md) * [iOS](./platforms/ios.md) * [Experimental: Pyodide (WebAssembly)](./platforms/pyodide.md) @@ -119,7 +120,7 @@ Commit this file, and push to GitHub - either to your default branch, or to a PR For more info on this file, check out the [docs](https://help.github.com/en/actions/reference/workflow-syntax-for-github-actions). -[`examples/github-deploy.yml`](https://github.com/pypa/cibuildwheel/blob/main/examples/github-deploy.yml) extends this minimal example to include iOS and Pyodide builds, and a demonstration of how to automatically upload the built wheels to PyPI. +[`examples/github-deploy.yml`](https://github.com/pypa/cibuildwheel/blob/main/examples/github-deploy.yml) extends this minimal example to include Android, iOS and Pyodide builds, and a demonstration of how to automatically upload the built wheels to PyPI. ### Azure Pipelines [linux/mac/windows] {: #azure-pipelines} diff --git a/mkdocs.yml b/mkdocs.yml index 1d0150736..c5f1bcc77 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -31,6 +31,7 @@ nav: - platforms/linux.md - platforms/windows.md - platforms/macos.md + - platforms/android.md - platforms/ios.md - platforms/pyodide.md - About the project: From 8e03e8715733889b85e8209a5ee4ea26b873cefd Mon Sep 17 00:00:00 2001 From: Malcolm Smith Date: Thu, 17 Apr 2025 22:01:34 +0100 Subject: [PATCH 04/57] Docs cleanups --- README.md | 6 +++--- docs/ci-services.md | 2 +- docs/options.md | 2 +- docs/platforms.md | 18 +++++++++--------- examples/github-deploy.yml | 9 ++++++++- 5 files changed, 22 insertions(+), 15 deletions(-) diff --git a/README.md b/README.md index e298589b1..65b35d957 100644 --- a/README.md +++ b/README.md @@ -67,9 +67,8 @@ Usage ¹ [Requires emulation](https://cibuildwheel.pypa.io/en/stable/faq/#emulation), distributed separately. Other services may also support Linux ARM through emulation or third-party build hosts, but these are not tested in our CI.
² [Uses cross-compilation](https://cibuildwheel.pypa.io/en/stable/faq/#windows-arm64). It is not possible to test `arm64` on this CI platform.
-³ Requires a macOS runner; runs tests on the simulator for the runner's architecture. -⁴ Requires runner to be Linux x86_64, macOS ARM64 or macOS x86_64. Runs tests on the emulator -for the runner's architecture. +³ Requires a macOS runner; runs tests on the simulator for the runner's architecture.
+⁴ Requires runner to be Linux x86_64, macOS ARM64 or macOS x86_64. Runs tests on the emulator for the runner's architecture.
Example setup @@ -140,6 +139,7 @@ Options | | [`CIBW_ENVIRONMENT_PASS_LINUX`](https://cibuildwheel.pypa.io/en/stable/options/#environment-pass) | Set environment variables on the host to pass-through to the container during the build. | | | [`CIBW_BEFORE_ALL`](https://cibuildwheel.pypa.io/en/stable/options/#before-all) | Execute a shell command on the build system before any wheels are built. | | | [`CIBW_BEFORE_BUILD`](https://cibuildwheel.pypa.io/en/stable/options/#before-build) | Execute a shell command preparing each wheel's build | +| | [`CIBW_XBUILD_TOOLS`](https://cibuildwheel.pypa.io/en/stable/options/#xbuild-tools) | Binaries on the path that should be included in an isolated cross-build environment. | | | [`CIBW_REPAIR_WHEEL_COMMAND`](https://cibuildwheel.pypa.io/en/stable/options/#repair-wheel-command) | Execute a shell command to repair each built wheel | | | [`CIBW_MANYLINUX_*_IMAGE`
`CIBW_MUSLLINUX_*_IMAGE`](https://cibuildwheel.pypa.io/en/stable/options/#linux-image) | Specify alternative manylinux / musllinux Docker images | | | [`CIBW_CONTAINER_ENGINE`](https://cibuildwheel.pypa.io/en/stable/options/#container-engine) | Specify which container engine to use when building Linux wheels | diff --git a/docs/ci-services.md b/docs/ci-services.md index ba0c53ec8..07fceb07d 100644 --- a/docs/ci-services.md +++ b/docs/ci-services.md @@ -53,7 +53,7 @@ Commit this file, and push to GitHub - either to your default branch, or to a PR For more info on this file, check out the [docs](https://help.github.com/en/actions/reference/workflow-syntax-for-github-actions). -[`examples/github-deploy.yml`](https://github.com/pypa/cibuildwheel/blob/main/examples/github-deploy.yml) extends this minimal example to include iOS and Pyodide builds, and a demonstration of how to automatically upload the built wheels to PyPI. +[`examples/github-deploy.yml`](https://github.com/pypa/cibuildwheel/blob/main/examples/github-deploy.yml) extends this minimal example to include Android, iOS and Pyodide builds, and a demonstration of how to automatically upload the built wheels to PyPI. ### Azure Pipelines [linux/mac/windows] {: #azure-pipelines} diff --git a/docs/options.md b/docs/options.md index 345efe175..7c1cb00b4 100644 --- a/docs/options.md +++ b/docs/options.md @@ -16,7 +16,7 @@ Default: `auto` - For `linux`, you need [Docker or Podman](#container-engine) running, on Linux, macOS, or Windows. - For `macos` and `windows`, you need to be running on the respective system, with a working compiler toolchain installed - Xcode Command Line tools for macOS, and MSVC for Windows. -- For `android` you need to be running on Linux or macOS, with an Android SDK installed. See [here](android.md) for more details. +- For `android` you need to be running on Linux or macOS, with an Android SDK installed. See [here](platforms.md#android) for more details. - For `ios` you need to be running on macOS, with Xcode and the iOS simulator installed. - For `pyodide` you need to be on an x86-64 linux runner and `python3.12` must be available in `PATH`. diff --git a/docs/platforms.md b/docs/platforms.md index ccfb57059..b91ae9ce2 100644 --- a/docs/platforms.md +++ b/docs/platforms.md @@ -177,8 +177,8 @@ Android development tools, which currently means Linux x86_64, macOS ARM64 or ma x86_64. Building Android wheels requires the build machine to have a working Python executable -of the same version. See the notes on [Linux](linux.md) and [macOS](macos.md) for -details of how this is installed. +of the same version. See the [Linux](#linux) and [macOS](#macos) sections for details +of how this is installed. If you already have an Android SDK, export the `ANDROID_HOME` environment variable to point at its location. Otherwise, here's how to install it: @@ -215,7 +215,7 @@ support for iOS or Android wheel tags](https://github.com/astral-sh/uv/issues/80 ### Cross platform builds Android builds are *cross platform builds*, as cibuildwheel does not support running -compilers and other build tools "on device". The supported build platforms (listed +compilers and other build tools "on device". Any supported build platform (listed above) can be used to build wheels for any supported Android architecture. However, wheels can only be *tested* on a machine of the same architecture – see the section below. @@ -223,7 +223,7 @@ below. ### Tests If tests have been configured, the test suite will be executed on a Gradle-managed -emulator matching the architecture of the build machine - for example, if you're +emulator matching the architecture of the build machine – for example, if you're building on an ARM64 machine, an ARM64 wheel can be tested on an ARM64 emulator. Cross-architecture testing is not supported. @@ -231,16 +231,16 @@ On Linux, the emulator needs access to the KVM virtualization interface, and a D environment variable pointing at an X server. Xvfb is acceptable. The Android test environment can't support running shell scripts, so the -[`CIBW_TEST_COMMAND`](../options.md#test-command) value must be specified as if it were +[`CIBW_TEST_COMMAND`](options.md#test-command) value must be specified as if it were a command line being passed to `python -m ...`. In addition, the project must use -[`CIBW_TEST_SOURCES`](../options.md#test-sources) to specify the minimum subset of files +[`CIBW_TEST_SOURCES`](options.md#test-sources) to specify the minimum subset of files that should be copied to the test environment. This is because the test must be run "on device", and the device will not have access to the local project directory. The test process uses the same testbed used by CPython itself to run the CPython test -suite. It is a Gradle project that has been configured to have a single JUnit - the -result of which reports the success or failure of running `python -m -`. +suite. It is a Gradle project that has been configured to have a single JUnit test, +the result of which reports the success or failure of running +`python -m `. ## iOS diff --git a/examples/github-deploy.yml b/examples/github-deploy.yml index eb678b8b7..bd7c782aa 100644 --- a/examples/github-deploy.yml +++ b/examples/github-deploy.yml @@ -16,7 +16,8 @@ jobs: runs-on: ${{ matrix.runs-on }} strategy: matrix: - os: [ linux-intel, linux-arm, windows, macOS-intel, macOS-arm, iOS, pyodide ] + os: [ linux-intel, linux-arm, windows, macos-intel, macos-arm, + android-intel, android-arm, ios, pyodide ] include: - archs: auto platform: auto @@ -33,6 +34,12 @@ jobs: # macos-14+ (including latest) are ARM64 runners runs-on: macos-latest archs: auto,universal2 + - os: android-intel + runs-on: ubuntu-latest + platform: android + - os: android-arm + runs-on: macos-latest + platform: android - os: ios runs-on: macos-latest platform: ios From 442886802a937a2a7a5828a1d039238cf04515fc Mon Sep 17 00:00:00 2001 From: Malcolm Smith Date: Wed, 23 Apr 2025 14:24:28 +0100 Subject: [PATCH 05/57] Add Android platform module; implement top-level structure and target Python installation --- cibuildwheel/platforms/android.py | 153 ++++++++++++++++++++++++++++++ 1 file changed, 153 insertions(+) create mode 100644 cibuildwheel/platforms/android.py diff --git a/cibuildwheel/platforms/android.py b/cibuildwheel/platforms/android.py new file mode 100644 index 000000000..f7a283588 --- /dev/null +++ b/cibuildwheel/platforms/android.py @@ -0,0 +1,153 @@ +import os +import shutil +import subprocess +from dataclasses import dataclass +from pathlib import Path + +from filelock import FileLock + +from .. import errors +from ..architecture import Architecture +from ..logger import log +from ..options import BuildOptions, Options +from ..selector import BuildSelector +from ..util import resources +from ..util.cmd import shell +from ..util.file import CIBW_CACHE_PATH, download, move_file +from ..util.helpers import prepare_command +from ..util.packaging import find_compatible_wheel + + +@dataclass(frozen=True) +class PythonConfiguration: + version: str + identifier: str + url: str + + +def all_python_configurations() -> list[PythonConfiguration]: + return [PythonConfiguration(**item) for item in resources.read_python_configs("android")] + + +def get_python_configurations( + build_selector: BuildSelector, architectures: set[Architecture] +) -> list[PythonConfiguration]: + return [ + c + for c in all_python_configurations() + if any(c.identifier.endswith(f"-android_{arch.value}") for arch in architectures) + and build_selector(c.identifier) + ] + + +def shell_prepared(build_options: BuildOptions, command: str) -> None: + shell( + prepare_command(command, project=".", package=build_options.package_dir), + env=build_options.environment.as_dictionary(os.environ), + ) + + +def before_all(options: Options, python_configurations: list[PythonConfiguration]) -> None: + before_all_options = options.build_options(python_configurations[0].identifier) + if before_all_options.before_all: + log.step("Running before_all...") + shell_prepared(before_all_options, before_all_options.before_all) + + +@dataclass +class Builder: + config: PythonConfiguration + build_options: BuildOptions + tmp_path: Path + built_wheels: list[Path] + + def build(self) -> None: + log.build_start(self.config.identifier) + self.tmp_path.mkdir() + self.setup_python() + + compatible_wheel = find_compatible_wheel(self.built_wheels, self.config.identifier) + if compatible_wheel: + print( + f"\nFound previously built wheel {compatible_wheel.name} that is " + f"compatible with {self.config.identifier}. Skipping build step..." + ) + repaired_wheel = compatible_wheel + else: + self.setup_env() + self.before_build() + repaired_wheel = self.build_wheel() + + self.test_wheel(repaired_wheel) + if compatible_wheel is not None: + self.built_wheels.append( + move_file(repaired_wheel, self.build_options.output_dir / repaired_wheel.name) + ) + + shutil.rmtree(self.tmp_path) + log.build_end() + + def setup_python(self) -> None: + log.step("Installing target Python...") + python_tgz = CIBW_CACHE_PATH / self.config.url.rpartition("/")[-1] + with FileLock(f"{python_tgz}.lock"): + if not python_tgz.exists(): + download(self.config.url, python_tgz) + + self.python_path = self.tmp_path / "python" + self.python_path.mkdir() + shutil.unpack_archive(python_tgz, self.python_path) + + def setup_env(self) -> None: + log.step("Setting up build environment...") + # TODO + + def before_build(self) -> None: + if self.build_options.before_build: + log.step("Running before_build...") + # TODO must run in the build environment + shell_prepared(self.build_options, self.build_options.before_build) + + def build_wheel(self) -> Path: + log.step("Building wheel...") + built_wheel_dir = self.tmp_path / "built_wheel" + + # TODO + + built_wheels = list(built_wheel_dir.glob("*.whl")) + if len(built_wheels) != 1: + msg = f"{built_wheel_dir} contains {len(built_wheels)} wheels; expected 1" + raise errors.FatalError(msg) + return built_wheels[0] + + def test_wheel(self, wheel: Path) -> None: + if self.build_options.test_command and self.build_options.test_selector( + self.config.identifier + ): + log.step("Testing wheel...") + print("FIXME", wheel) + # TODO pass environment from cibuildwheel config? + # TODO require pip 25.1 for Android tag support + + +def build(options: Options, tmp_path: Path) -> None: + configs = get_python_configurations( + options.globals.build_selector, options.globals.architectures + ) + if not configs: + return + + try: + before_all(options, configs) + built_wheels: list[Path] = [] + for config in configs: + Builder( + config, + options.build_options(config.identifier), + tmp_path / config.identifier, + built_wheels, + ).build() + + except subprocess.CalledProcessError as error: + msg = f"Command {error.cmd} failed with code {error.returncode}. {error.stdout or ''}" + raise errors.FatalError(msg) from error From f3084f0fb92263accfff69b02071be8d1de04a2a Mon Sep 17 00:00:00 2001 From: Malcolm Smith Date: Wed, 23 Apr 2025 19:49:44 +0100 Subject: [PATCH 06/57] Implement setup_env and build_wheel --- .pre-commit-config.yaml | 1 + bin/update_pythons.py | 6 +- cibuildwheel/frontend.py | 16 ++++ cibuildwheel/platforms/android.py | 150 ++++++++++++++++++++++++++---- docs/platforms.md | 5 +- pyproject.toml | 1 + 6 files changed, 155 insertions(+), 24 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 11e7e982a..30dabab2f 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -29,6 +29,7 @@ repos: args: ["--python-version=3.11"] additional_dependencies: &mypy-dependencies - bracex + - build - dependency-groups>=1.2 - nox>=2025.2.9 - orjson diff --git a/bin/update_pythons.py b/bin/update_pythons.py index ce6466c8c..ed94ecdc1 100755 --- a/bin/update_pythons.py +++ b/bin/update_pythons.py @@ -20,6 +20,7 @@ from rich.syntax import Syntax from cibuildwheel.extra import dump_python_configurations +from cibuildwheel.platforms.android import android_triplet log = logging.getLogger("cibw") @@ -261,10 +262,7 @@ def update_version_android( # Return a config using the highest version for the given specifier. if sorted_versions: max_version = sorted_versions[0] - triplet = { - "arm64_v8a": "aarch64-linux-android", - "x86_64": "x86_64-linux-android", - }[identifier.split("_", 1)[1]] + triplet = android_triplet(identifier) return ConfigAndroid( identifier=identifier, version=str(version), diff --git a/cibuildwheel/frontend.py b/cibuildwheel/frontend.py index 79796a1be..444f428a9 100644 --- a/cibuildwheel/frontend.py +++ b/cibuildwheel/frontend.py @@ -58,6 +58,22 @@ def _split_config_settings(config_settings: str) -> list[str]: return [f"-C{setting}" for setting in config_settings_list] +# Based on build.__main__.main. +def parse_config_settings(config_settings_str: str) -> dict[str, str | list[str]]: + config_settings: dict[str, str | list[str]] = {} + for arg in shlex.split(config_settings_str): + setting, _, value = arg.partition("=") + existing_value = config_settings.get(setting) + if existing_value is None: + config_settings[setting] = value + elif isinstance(existing_value, str): + config_settings[setting] = [existing_value] + else: + existing_value.append(value) + + return config_settings + + def get_build_frontend_extra_flags( build_frontend: BuildFrontendConfig, verbosity_level: int, config_settings: str ) -> list[str]: diff --git a/cibuildwheel/platforms/android.py b/cibuildwheel/platforms/android.py index f7a283588..cb4d95a4b 100644 --- a/cibuildwheel/platforms/android.py +++ b/cibuildwheel/platforms/android.py @@ -1,21 +1,33 @@ import os import shutil import subprocess +from collections.abc import Generator +from contextlib import contextmanager from dataclasses import dataclass from pathlib import Path +from build import ProjectBuilder from filelock import FileLock from .. import errors from ..architecture import Architecture +from ..frontend import BuildFrontendConfig, get_build_frontend_extra_flags, parse_config_settings from ..logger import log from ..options import BuildOptions, Options from ..selector import BuildSelector from ..util import resources -from ..util.cmd import shell +from ..util.cmd import call, shell from ..util.file import CIBW_CACHE_PATH, download, move_file from ..util.helpers import prepare_command from ..util.packaging import find_compatible_wheel +from ..venv import constraint_flags, virtualenv + + +def android_triplet(identifier: str) -> str: + return { + "arm64_v8a": "aarch64-linux-android", + "x86_64": "x86_64-linux-android", + }[identifier.split("_", 1)[1]] @dataclass(frozen=True) @@ -40,10 +52,10 @@ def get_python_configurations( ] -def shell_prepared(build_options: BuildOptions, command: str) -> None: +def shell_prepared(command: str, build_options: BuildOptions, env: dict[str, str]) -> None: shell( prepare_command(command, project=".", package=build_options.package_dir), - env=build_options.environment.as_dictionary(os.environ), + env=env, ) @@ -51,19 +63,23 @@ def before_all(options: Options, python_configurations: list[PythonConfiguration before_all_options = options.build_options(python_configurations[0].identifier) if before_all_options.before_all: log.step("Running before_all...") - shell_prepared(before_all_options, before_all_options.before_all) + shell_prepared( + before_all_options.before_all, + before_all_options, + before_all_options.environment.as_dictionary(os.environ), + ) @dataclass class Builder: config: PythonConfiguration build_options: BuildOptions - tmp_path: Path + tmp_dir: Path built_wheels: list[Path] def build(self) -> None: log.build_start(self.config.identifier) - self.tmp_path.mkdir() + self.tmp_dir.mkdir() self.setup_python() compatible_wheel = find_compatible_wheel(self.built_wheels, self.config.identifier) @@ -84,7 +100,7 @@ def build(self) -> None: move_file(repaired_wheel, self.build_options.output_dir / repaired_wheel.name) ) - shutil.rmtree(self.tmp_path) + shutil.rmtree(self.tmp_dir) log.build_end() def setup_python(self) -> None: @@ -94,25 +110,127 @@ def setup_python(self) -> None: if not python_tgz.exists(): download(self.config.url, python_tgz) - self.python_path = self.tmp_path / "python" - self.python_path.mkdir() - shutil.unpack_archive(python_tgz, self.python_path) + self.python_dir = self.tmp_dir / "python" + self.python_dir.mkdir() + shutil.unpack_archive(python_tgz, self.python_dir) def setup_env(self) -> None: log.step("Setting up build environment...") - # TODO + + python_exe_name = f"python{self.config.version}" + python_exe = shutil.which(python_exe_name) + if not python_exe: + msg = f"Couldn't find {python_exe_name} on the PATH" + raise errors.FatalError(msg) + + # Create virtual environment + self.venv_dir = self.tmp_dir / "venv" + dependency_constraint = self.build_options.dependency_constraints.get_for_python_version( + version=self.config.version, tmp_dir=self.tmp_dir + ) + self.env = virtualenv( + self.config.version, + Path(python_exe), + self.venv_dir, + dependency_constraint, + use_uv=False, + ) + + # Apply custom environment variables, and check environment is still valid + self.env = self.build_options.environment.as_dictionary(self.env) + self.env["PIP_DISABLE_PIP_VERSION_CHECK"] = "1" + for command in ["python", "pip"]: + which = call("which", command, env=self.env, capture_stdout=True).strip() + if which != f"{self.venv_dir}/bin/{command}": + msg = ( + "{command} available on PATH doesn't match our installed instance. If you " + "have modified PATH, ensure that you don't overwrite cibuildwheel's entry " + "or insert {command} above it." + ) + call(command, "--version", env=self.env) + + # Install build tools + self.build_frontend = self.build_options.build_frontend or BuildFrontendConfig("build") + if self.build_frontend.name != "build": + msg = "Android requires the build frontend to be 'build'" + raise errors.FatalError(msg) + self.pip_install("build", *constraint_flags(dependency_constraint)) + + # Install build-time requirements. These must be installed for the build platform, not for + # Android, which is why we can't allow them to be installed by the `build` subprocess. + pb = ProjectBuilder( + self.build_options.package_dir, python_executable=f"{self.venv_dir}/bin/python" + ) + self.pip_install(*pb.build_system_requires) + + # get_requires_for_build runs the package's build script, so it must be called while + # simulating Android. + with self.simulate_android(): + requires_for_build = pb.get_requires_for_build( + "wheel", parse_config_settings(self.build_options.config_settings) + ) + self.pip_install(*requires_for_build) + + def pip_install(self, *args: str) -> None: + if args: + call("pip", "install", "--upgrade", *args, env=self.env) + + @contextmanager + def simulate_android(self) -> Generator[None]: + site_packages = self.venv_dir / f"lib/python{self.config.version}/site-packages" + (site_packages / "_cross_venv.pth").write_text( + f"import _cross_venv; _cross_venv.initialize('{self.config.identifier}')" + ) + shutil.copy(resources.PATH / "_cross_venv.py", site_packages) + + env = self.env.copy() + for line in call( + self.python_dir / "android.py", + "env", + android_triplet(self.config.identifier), + capture_stdout=True, + ).splitlines(): + key, value = line.split("=", 1) + env[key] = value + + original_env = {key: os.environ.get(key) for key in env} + os.environ.update(env) + + try: + yield + finally: + for name in ["_cross_venv.pth", "_cross_venv.py"]: + (site_packages / name).unlink() + + for key, original_value in original_env.items(): + if original_value is None: + del os.environ[key] + else: + os.environ[key] = original_value def before_build(self) -> None: if self.build_options.before_build: log.step("Running before_build...") - # TODO must run in the build environment - shell_prepared(self.build_options, self.build_options.before_build) + shell_prepared(self.build_options.before_build, self.build_options, self.env) def build_wheel(self) -> Path: log.step("Building wheel...") - built_wheel_dir = self.tmp_path / "built_wheel" - - # TODO + built_wheel_dir = self.tmp_dir / "built_wheel" + with self.simulate_android(): + call( + "python", + "-m", + "build", + self.build_options.package_dir, + "--wheel", + "--no-isolation", + f"--outdir={built_wheel_dir}", + *get_build_frontend_extra_flags( + self.build_frontend, + self.build_options.build_verbosity, + self.build_options.config_settings, + ), + ) built_wheels = list(built_wheel_dir.glob("*.whl")) if len(built_wheels) != 1: diff --git a/docs/platforms.md b/docs/platforms.md index b91ae9ce2..04236c8a4 100644 --- a/docs/platforms.md +++ b/docs/platforms.md @@ -176,10 +176,6 @@ cibuildwheel can build and test Android wheels on any POSIX platform supported b Android development tools, which currently means Linux x86_64, macOS ARM64 or macOS x86_64. -Building Android wheels requires the build machine to have a working Python executable -of the same version. See the [Linux](#linux) and [macOS](#macos) sections for details -of how this is installed. - If you already have an Android SDK, export the `ANDROID_HOME` environment variable to point at its location. Otherwise, here's how to install it: @@ -195,6 +191,7 @@ needs. It also requires the following commands to be on the `PATH`: +* `pythonX.Y`, where `X.Y` is the version of Python you're building for. * `curl` * `java` (or set the `JAVA_HOME` environment variable) diff --git a/pyproject.toml b/pyproject.toml index b3523e2be..6f34a685b 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -39,6 +39,7 @@ classifiers = [ dependencies = [ "bashlex!=0.13", "bracex", + "build >= 1.0.0", "certifi", "dependency-groups>=1.2", "filelock", From e715aa42c029a38b47abd680d7d28cb31dc8e394 Mon Sep 17 00:00:00 2001 From: Malcolm Smith Date: Fri, 25 Apr 2025 19:08:48 +0100 Subject: [PATCH 07/57] lru-dict build working --- cibuildwheel/platforms/android.py | 50 +++++++++++--------- cibuildwheel/resources/_cross_venv.py | 68 +++++++++++++++++++++++++++ docs/platforms.md | 2 +- 3 files changed, 97 insertions(+), 23 deletions(-) create mode 100644 cibuildwheel/resources/_cross_venv.py diff --git a/cibuildwheel/platforms/android.py b/cibuildwheel/platforms/android.py index cb4d95a4b..811376a27 100644 --- a/cibuildwheel/platforms/android.py +++ b/cibuildwheel/platforms/android.py @@ -1,6 +1,8 @@ import os +import shlex import shutil import subprocess +import sys from collections.abc import Generator from contextlib import contextmanager from dataclasses import dataclass @@ -95,7 +97,7 @@ def build(self) -> None: repaired_wheel = self.build_wheel() self.test_wheel(repaired_wheel) - if compatible_wheel is not None: + if compatible_wheel is None: self.built_wheels.append( move_file(repaired_wheel, self.build_options.output_dir / repaired_wheel.name) ) @@ -149,6 +151,14 @@ def setup_env(self) -> None: ) call(command, "--version", env=self.env) + # Add cross-venv files, which will be activated by simulate_android. + self.site_packages = next(self.venv_dir.glob("lib/python*/site-packages")) + for path in [ + resources.PATH / "_cross_venv.py", + next(self.python_dir.glob("prefix/lib/python*/_sysconfigdata_*.py")), + ]: + shutil.copy(path, self.site_packages) + # Install build tools self.build_frontend = self.build_options.build_frontend or BuildFrontendConfig("build") if self.build_frontend.name != "build": @@ -177,31 +187,26 @@ def pip_install(self, *args: str) -> None: @contextmanager def simulate_android(self) -> Generator[None]: - site_packages = self.venv_dir / f"lib/python{self.config.version}/site-packages" - (site_packages / "_cross_venv.pth").write_text( - f"import _cross_venv; _cross_venv.initialize('{self.config.identifier}')" - ) - shutil.copy(resources.PATH / "_cross_venv.py", site_packages) - - env = self.env.copy() - for line in call( - self.python_dir / "android.py", - "env", - android_triplet(self.config.identifier), - capture_stdout=True, - ).splitlines(): - key, value = line.split("=", 1) - env[key] = value - - original_env = {key: os.environ.get(key) for key in env} - os.environ.update(env) + if not hasattr(self, "android_env"): + self.android_env = self.env.copy() + env_output = call(self.python_dir / "android.py", "env", capture_stdout=True) + sys.stdout.write(env_output) + for line in env_output.splitlines(): + key, value = line.removeprefix("export ").split("=", 1) + value_split = shlex.split(value) + assert len(value_split) == 1, value_split + self.android_env[key] = value_split[0] + + original_env = {key: os.environ.get(key) for key in self.android_env} + os.environ.update(self.android_env) + + pth_file = self.site_packages / "_cross_venv.pth" + pth_file.write_text("import _cross_venv; _cross_venv.initialize()") try: yield finally: - for name in ["_cross_venv.pth", "_cross_venv.py"]: - (site_packages / name).unlink() - + pth_file.unlink() for key, original_value in original_env.items(): if original_value is None: del os.environ[key] @@ -224,6 +229,7 @@ def build_wheel(self) -> Path: self.build_options.package_dir, "--wheel", "--no-isolation", + "--skip-dependency-check", f"--outdir={built_wheel_dir}", *get_build_frontend_extra_flags( self.build_frontend, diff --git a/cibuildwheel/resources/_cross_venv.py b/cibuildwheel/resources/_cross_venv.py new file mode 100644 index 000000000..6b907e818 --- /dev/null +++ b/cibuildwheel/resources/_cross_venv.py @@ -0,0 +1,68 @@ +# This module is copied into the site-packages directory of an Android build environment, and +# activated via a .pth file when we want the environment to simulate Android. + +import os +import platform +import re +import sys +import sysconfig +from pathlib import Path + + +def initialize(): + # os ###################################################################### + def cross_os_uname(): + return os.uname_result( + ( + "Linux", + "localhost", + # The Linux kernel version and release are unlikely to be significant, but return + # realistic values anyway (from an API level 24 emulator). + "3.18.91+", + "#1 SMP PREEMPT Tue Jan 9 20:35:43 UTC 2018", + os.environ["HOST"].split("-")[0], + ) + ) + + os.name = "posix" + os.uname = cross_os_uname + + # platform ################################################################ + # + # We can't determine the user-visible Android version number from the API level, so return a + # string which will work fine for display, but will fail to parse as a version number. + def cross_android_ver(*args, **kwargs): + return platform.AndroidVer( + release=f"API level {cross_getandroidapilevel()}", + api_level=cross_getandroidapilevel(), + manufacturer="Google", + model="sdk_gphone64", + device="emu64", + is_emulator=True, + ) + + # platform.uname, platform.system etc. are all implemented in terms of platform.android_ver. + platform.android_ver = cross_android_ver + + # sys ##################################################################### + def cross_getandroidapilevel(): + return sysconfig.get_config_var("ANDROID_API_LEVEL") + + sys.cross_compiling = True # Some packages may recognize this from the crossenv tool. + sys.getandroidapilevel = cross_getandroidapilevel + sys.implementation._multiarch = os.environ["HOST"] + sys.platform = "android" + + # _get_sysconfigdata_name is implemented in terms of sys.abiflags, sys.platform and + # sys.implementation._multiarch. Determine the abiflags from the filename. + sysconfigdata_path = next(Path(__file__).parent.glob("_sysconfigdata_*.py")) + sys.abiflags = re.match(r"_sysconfigdata_(.*?)_", sysconfigdata_path.name)[1] + + # sysconfig ############################################################### + # + sysconfig._init_config_vars() + + # sysconfig.get_platform (which determines the wheel tag) is implemented in terms of + # sys.platform, sysconfig.get_config_var("ANDROID_API_LEVEL"), and os.uname. + if api_level := os.environ.get("ANDROID_API_LEVEL"): + sysconfig.get_config_vars()["ANDROID_API_LEVEL"] = int(api_level) diff --git a/docs/platforms.md b/docs/platforms.md index 04236c8a4..1ce001ed2 100644 --- a/docs/platforms.md +++ b/docs/platforms.md @@ -197,7 +197,7 @@ It also requires the following commands to be on the `PATH`: ### Android version compatibility -Android builds will honor the `api_level` environment variable to set the minimum +Android builds will honor the `ANDROID_API_LEVEL` environment variable to set the minimum supported [API level](https://developer.android.com/tools/releases/platforms) for generated wheels. This will default to the minimum API level of the selected Python version. From 423401385092950f54e15701cf0a3ef5f1deeb3d Mon Sep 17 00:00:00 2001 From: Malcolm Smith Date: Wed, 30 Apr 2025 20:16:44 +0100 Subject: [PATCH 08/57] Alter prefix in sysconfigdata file; fix various issues with FLAGS variables --- cibuildwheel/platforms/android.py | 82 +++++++++++++++++++++++++-- cibuildwheel/resources/_cross_venv.py | 7 +-- 2 files changed, 79 insertions(+), 10 deletions(-) diff --git a/cibuildwheel/platforms/android.py b/cibuildwheel/platforms/android.py index 811376a27..f30eaac13 100644 --- a/cibuildwheel/platforms/android.py +++ b/cibuildwheel/platforms/android.py @@ -1,12 +1,14 @@ +import importlib.util import os import shlex import shutil import subprocess -import sys from collections.abc import Generator from contextlib import contextmanager from dataclasses import dataclass from pathlib import Path +from pprint import pprint +from typing import Any from build import ProjectBuilder from filelock import FileLock @@ -145,10 +147,11 @@ def setup_env(self) -> None: which = call("which", command, env=self.env, capture_stdout=True).strip() if which != f"{self.venv_dir}/bin/{command}": msg = ( - "{command} available on PATH doesn't match our installed instance. If you " - "have modified PATH, ensure that you don't overwrite cibuildwheel's entry " - "or insert {command} above it." + f"{command} available on PATH doesn't match our installed instance. If you " + f"have modified PATH, ensure that you don't overwrite cibuildwheel's entry " + f"or insert {command} above it." ) + raise errors.FatalError(msg) call(command, "--version", env=self.env) # Add cross-venv files, which will be activated by simulate_android. @@ -157,7 +160,9 @@ def setup_env(self) -> None: resources.PATH / "_cross_venv.py", next(self.python_dir.glob("prefix/lib/python*/_sysconfigdata_*.py")), ]: - shutil.copy(path, self.site_packages) + out_path = Path(shutil.copy(path, self.site_packages)) + if "sysconfigdata" in path.name: + self.localize_sysconfigdata(out_path) # Install build tools self.build_frontend = self.build_options.build_frontend or BuildFrontendConfig("build") @@ -181,6 +186,54 @@ def setup_env(self) -> None: ) self.pip_install(*requires_for_build) + def localize_sysconfigdata(self, sysconfigdata_path: Path) -> None: + spec = importlib.util.spec_from_file_location(sysconfigdata_path.stem, sysconfigdata_path) + assert spec is not None + module = importlib.util.module_from_spec(spec) + assert spec.loader is not None + spec.loader.exec_module(module) + + with sysconfigdata_path.open("w") as f: + f.write("# Generated by cibuildwheel\n") + f.write("build_time_vars = ") + self.sysconfigdata = self.localized_vars( + module.build_time_vars, self.python_dir / "prefix" + ) + pprint(self.sysconfigdata, stream=f, compact=True) + + def localized_vars(self, orig_vars: dict[str, Any], prefix: Path) -> dict[str, Any]: + orig_prefix = orig_vars["prefix"] + localized_vars = {} + for key, value in orig_vars.items(): + # The host's sysconfigdata will include references to build-time paths. + # Update these to refer to the current prefix. + final = value + if isinstance(final, str): + final = final.replace(orig_prefix, str(prefix)) + + if key == "ANDROID_API_LEVEL": + if api_level := os.environ.get(key): + final = int(api_level) + + # Build systems vary in whether FLAGS variables are read from sysconfig, and if so, + # whether they're replaced by environment variables or combined with them. Even + # setuptools has changed its behavior here + # (https://github.com/pypa/setuptools/issues/4836). + # + # Ensure consistency by clearing the sysconfig variables and letting the environment + # variables take effect alone. This will also work for any non-Python build systems + # which the build script may call. + elif key in ["CFLAGS", "CXXFLAGS", "LDFLAGS"]: + final = "" + + # These variables contain an embedded copy of LDFLAGS. + elif key in ["LDSHARED", "LDCXXSHARED"]: + final = final.removesuffix(" " + orig_vars["LDFLAGS"]) + + localized_vars[key] = final + + return localized_vars + def pip_install(self, *args: str) -> None: if args: call("pip", "install", "--upgrade", *args, env=self.env) @@ -190,13 +243,30 @@ def simulate_android(self) -> Generator[None]: if not hasattr(self, "android_env"): self.android_env = self.env.copy() env_output = call(self.python_dir / "android.py", "env", capture_stdout=True) - sys.stdout.write(env_output) for line in env_output.splitlines(): key, value = line.removeprefix("export ").split("=", 1) value_split = shlex.split(value) assert len(value_split) == 1, value_split self.android_env[key] = value_split[0] + # localized_vars cleared the CFLAGS and CXXFLAGS in the sysconfigdata, but most + # packages take their optimization flags from these variables. Pass these flags via + # environment variables instead. + # + # We don't enable debug information, because it significantly increases binary size, + # and most Android app developers don't have the NDK installed, so they would have no + # way to strip it. + opt = " ".join( + word for word in self.sysconfigdata["OPT"].split() if not word.startswith("-g") + ) + for key in ["CFLAGS", "CXXFLAGS"]: + self.android_env[key] += " " + opt + + # Format the environment so it can be pasted into a shell. + for key, value in sorted(self.android_env.items()): + if self.env.get(key) != value: + print(f"export {key}={shlex.quote(value)}") + original_env = {key: os.environ.get(key) for key in self.android_env} os.environ.update(self.android_env) diff --git a/cibuildwheel/resources/_cross_venv.py b/cibuildwheel/resources/_cross_venv.py index 6b907e818..e6182288c 100644 --- a/cibuildwheel/resources/_cross_venv.py +++ b/cibuildwheel/resources/_cross_venv.py @@ -62,7 +62,6 @@ def cross_getandroidapilevel(): # sysconfig._init_config_vars() - # sysconfig.get_platform (which determines the wheel tag) is implemented in terms of - # sys.platform, sysconfig.get_config_var("ANDROID_API_LEVEL"), and os.uname. - if api_level := os.environ.get("ANDROID_API_LEVEL"): - sysconfig.get_config_vars()["ANDROID_API_LEVEL"] = int(api_level) + # sysconfig.get_platform, which determines the wheel tag, is implemented in terms of + # sys.platform, sysconfig.get_config_var("ANDROID_API_LEVEL") (see localized_vars in + # android.py), and os.uname. From f2048b79a05a468b307077e2e276a1ec64a837b2 Mon Sep 17 00:00:00 2001 From: Malcolm Smith Date: Sun, 4 May 2025 22:46:14 +0100 Subject: [PATCH 09/57] Implement Android testing --- bin/update_pythons.py | 1 + cibuildwheel/__main__.py | 20 +--- cibuildwheel/architecture.py | 37 ++++-- cibuildwheel/platforms/android.py | 105 +++++++++++++++--- .../resources/constraints-pyodide312.txt | 2 +- .../resources/constraints-python310.txt | 2 +- .../resources/constraints-python311.txt | 2 +- .../resources/constraints-python312.txt | 2 +- .../resources/constraints-python313.txt | 2 +- .../resources/constraints-python38.txt | 2 +- .../resources/constraints-python39.txt | 2 +- cibuildwheel/resources/constraints.txt | 2 +- docs/options.md | 9 +- docs/platforms.md | 7 +- pyproject.toml | 2 +- 15 files changed, 143 insertions(+), 54 deletions(-) diff --git a/bin/update_pythons.py b/bin/update_pythons.py index ed94ecdc1..10ad83a65 100755 --- a/bin/update_pythons.py +++ b/bin/update_pythons.py @@ -241,6 +241,7 @@ def update_version_macos( class AndroidVersions: + # This should be replaced with official python.org downloads once they're available. MAVEN_URL = "https://repo.maven.apache.org/maven2/com/chaquo/python/python" def __init__(self) -> None: diff --git a/cibuildwheel/__main__.py b/cibuildwheel/__main__.py index 088286ac1..09fec6939 100644 --- a/cibuildwheel/__main__.py +++ b/cibuildwheel/__main__.py @@ -17,7 +17,7 @@ import cibuildwheel import cibuildwheel.util from cibuildwheel import errors -from cibuildwheel.architecture import Architecture, allowed_architectures_check +from cibuildwheel.architecture import Architecture, allowed_architectures_check, native_platform from cibuildwheel.ci import CIProvider, detect_ci_provider, fix_ansi_codes_for_github_actions from cibuildwheel.logger import log from cibuildwheel.options import CommandLineArguments, Options, compute_options @@ -246,22 +246,6 @@ def _compute_platform_only(only: str) -> PlatformName: raise errors.ConfigurationError(msg) -def _compute_platform_auto() -> PlatformName: - if sys.platform.startswith("linux"): - return "linux" - elif sys.platform == "darwin": - return "macos" - elif sys.platform == "win32": - return "windows" - else: - msg = ( - 'Unable to detect platform from "sys.platform". cibuildwheel doesn\'t ' - "support building wheels for this platform. You might be able to build for a different " - "platform using the --platform argument. Check --help output for more information." - ) - raise errors.ConfigurationError(msg) - - def _compute_platform(args: CommandLineArguments) -> PlatformName: platform_option_value = args.platform or os.environ.get("CIBW_PLATFORM", "") or "auto" @@ -281,7 +265,7 @@ def _compute_platform(args: CommandLineArguments) -> PlatformName: elif platform_option_value != "auto": return typing.cast(PlatformName, platform_option_value) - return _compute_platform_auto() + return native_platform() @contextlib.contextmanager diff --git a/cibuildwheel/architecture.py b/cibuildwheel/architecture.py index 6d33e53f5..b7d9e0681 100644 --- a/cibuildwheel/architecture.py +++ b/cibuildwheel/architecture.py @@ -28,6 +28,30 @@ ] +def arch_synonym(arch: str, from_platform: PlatformName, to_platform: PlatformName) -> str | None: + for arch_synonym in ARCH_SYNONYMS: + if arch == arch_synonym.get(from_platform): + return arch_synonym.get(to_platform, arch) + + return arch + + +def native_platform() -> PlatformName: + if sys.platform.startswith("linux"): + return "linux" + elif sys.platform == "darwin": + return "macos" + elif sys.platform == "win32": + return "windows" + else: + msg = ( + 'Unable to detect platform from "sys.platform". cibuildwheel doesn\'t ' + "support building wheels for this platform. You might be able to build for a different " + "platform using the --platform argument. Check --help output for more information." + ) + raise errors.ConfigurationError(msg) + + def _check_aarch32_el0() -> bool: """Check if running armv7l natively on aarch64 is supported""" if not sys.platform.startswith("linux"): @@ -126,15 +150,12 @@ def native_arch(platform: PlatformName) -> "Architecture | None": # we might need to rename the native arch to the machine we're running # on, as the same arch can have different names on different platforms if host_platform != platform: - for arch_synonym in ARCH_SYNONYMS: - if native_machine == arch_synonym.get(host_platform): - synonym = arch_synonym[platform] - - if synonym is None: - # can't build anything on this platform - return None + synonym = arch_synonym(native_machine, host_platform, platform) + if synonym is None: + # can't build anything on this platform + return None - native_architecture = Architecture(synonym) + native_architecture = Architecture(synonym) return native_architecture diff --git a/cibuildwheel/platforms/android.py b/cibuildwheel/platforms/android.py index f30eaac13..0acabc3cc 100644 --- a/cibuildwheel/platforms/android.py +++ b/cibuildwheel/platforms/android.py @@ -1,5 +1,7 @@ import importlib.util import os +import platform +import re import shlex import shutil import subprocess @@ -14,14 +16,15 @@ from filelock import FileLock from .. import errors -from ..architecture import Architecture +from ..architecture import Architecture, arch_synonym, native_platform from ..frontend import BuildFrontendConfig, get_build_frontend_extra_flags, parse_config_settings from ..logger import log from ..options import BuildOptions, Options from ..selector import BuildSelector +from ..typing import PathOrStr from ..util import resources from ..util.cmd import call, shell -from ..util.file import CIBW_CACHE_PATH, download, move_file +from ..util.file import CIBW_CACHE_PATH, copy_test_sources, download, move_file from ..util.helpers import prepare_command from ..util.packaging import find_compatible_wheel from ..venv import constraint_flags, virtualenv @@ -31,7 +34,16 @@ def android_triplet(identifier: str) -> str: return { "arm64_v8a": "aarch64-linux-android", "x86_64": "x86_64-linux-android", - }[identifier.split("_", 1)[1]] + }[parse_identifier(identifier)[1]] + + +def parse_identifier(identifier: str) -> tuple[str, str]: + match = re.fullmatch(r"cp(\d)(\d+)-android_(.+)", identifier) + if not match: + msg = f"invalid Android identifier: '{identifier}'" + raise ValueError(msg) + major, minor, arch = match.groups() + return (f"{major}.{minor}", arch) @dataclass(frozen=True) @@ -40,6 +52,10 @@ class PythonConfiguration: identifier: str url: str + @property + def arch(self) -> str: + return parse_identifier(self.identifier)[1] + def all_python_configurations() -> list[PythonConfiguration]: return [PythonConfiguration(**item) for item in resources.read_python_configs("android")] @@ -51,8 +67,7 @@ def get_python_configurations( return [ c for c in all_python_configurations() - if any(c.identifier.endswith(f"-android_{arch.value}") for arch in architectures) - and build_selector(c.identifier) + if c.arch in architectures and build_selector(c.identifier) ] @@ -234,7 +249,7 @@ def localized_vars(self, orig_vars: dict[str, Any], prefix: Path) -> dict[str, A return localized_vars - def pip_install(self, *args: str) -> None: + def pip_install(self, *args: PathOrStr) -> None: if args: call("pip", "install", "--upgrade", *args, env=self.env) @@ -242,7 +257,9 @@ def pip_install(self, *args: str) -> None: def simulate_android(self) -> Generator[None]: if not hasattr(self, "android_env"): self.android_env = self.env.copy() - env_output = call(self.python_dir / "android.py", "env", capture_stdout=True) + env_output = call( + self.python_dir / "android.py", "env", env=self.env, capture_stdout=True + ) for line in env_output.splitlines(): key, value = line.removeprefix("export ").split("=", 1) value_split = shlex.split(value) @@ -312,16 +329,76 @@ def build_wheel(self) -> Path: if len(built_wheels) != 1: msg = f"{built_wheel_dir} contains {len(built_wheels)} wheels; expected 1" raise errors.FatalError(msg) - return built_wheels[0] + built_wheel = built_wheels[0] + + if built_wheel.name.endswith("none-any.whl"): + raise errors.NonPlatformWheelError() + return built_wheel def test_wheel(self, wheel: Path) -> None: - if self.build_options.test_command and self.build_options.test_selector( - self.config.identifier + if not ( + self.build_options.test_command + and self.build_options.test_selector(self.config.identifier) ): - log.step("Testing wheel...") - print("FIXME", wheel) - # TODO pass environment from cibuildwheel config? - # TODO require pip 25.1 for Android tag support + return + + log.step("Testing wheel...") + if self.config.arch != arch_synonym(platform.machine(), native_platform(), "android"): + log.warning("Skipping tests on non-native architecture") + return + + if self.build_options.before_test: + shell_prepared(self.build_options.before_test, self.build_options, self.env) + + # Install the wheel and test-requires. + site_packages_dir = self.tmp_dir / "site-packages" + site_packages_dir.mkdir() + self.pip_install( + "--only-binary=:all:", + "--platform", + f"android_{self.sysconfigdata['ANDROID_API_LEVEL']}_{self.config.arch}", + "--extra-index-url", + "https://chaquo.com/pypi-13.1/", + "--target", + site_packages_dir, + f"{wheel}{self.build_options.test_extras}", + *self.build_options.test_requires, + ) + + # Copy test-sources. + # + # TODO: decide what to do if this isn't specified + # (https://github.com/pypa/cibuildwheel/pull/2363#issuecomment-2849413429) + cwd_dir = self.tmp_dir / "cwd" + cwd_dir.mkdir() + copy_test_sources(self.build_options.test_sources, self.build_options.package_dir, cwd_dir) + + # Parse test-command. + test_args = shlex.split(self.build_options.test_command) + if test_args[:2] in [["python", "-c"], ["python", "-m"]]: + test_args[:3] = [test_args[1], test_args[2], "--"] + elif test_args[0] in ["pytest"]: + test_args[:1] = ["-m", test_args[0], "--"] + else: + msg = ( + f"Test command '{self.build_options.test_command}' is not supported on this " + f"platform. Supported commands are 'python -m', 'python -c' and 'pytest'." + ) + raise errors.FatalError(msg) + + # Run the test app. + call( + self.python_dir / "android.py", + "test", + "--managed", + "maxVersion", + "--site-packages", + site_packages_dir, + "--cwd", + cwd_dir, + *test_args, + env=self.env, + ) def build(options: Options, tmp_path: Path) -> None: diff --git a/cibuildwheel/resources/constraints-pyodide312.txt b/cibuildwheel/resources/constraints-pyodide312.txt index 2a0bfbce7..68c6359bb 100644 --- a/cibuildwheel/resources/constraints-pyodide312.txt +++ b/cibuildwheel/resources/constraints-pyodide312.txt @@ -48,7 +48,7 @@ packaging==24.2 # build # pyodide-build # unearth -pip==25.0.1 +pip==25.1.1 # via -r .nox/update_constraints/tmp/constraints-pyodide.in platformdirs==4.3.7 # via virtualenv diff --git a/cibuildwheel/resources/constraints-python310.txt b/cibuildwheel/resources/constraints-python310.txt index da66890e6..e58cd57db 100644 --- a/cibuildwheel/resources/constraints-python310.txt +++ b/cibuildwheel/resources/constraints-python310.txt @@ -18,7 +18,7 @@ packaging==24.2 # via # build # delocate -pip==25.0.1 +pip==25.1.1 # via -r cibuildwheel/resources/constraints.in platformdirs==4.3.7 # via virtualenv diff --git a/cibuildwheel/resources/constraints-python311.txt b/cibuildwheel/resources/constraints-python311.txt index ec5abaa74..b5d144675 100644 --- a/cibuildwheel/resources/constraints-python311.txt +++ b/cibuildwheel/resources/constraints-python311.txt @@ -16,7 +16,7 @@ packaging==24.2 # via # build # delocate -pip==25.0.1 +pip==25.1.1 # via -r cibuildwheel/resources/constraints.in platformdirs==4.3.7 # via virtualenv diff --git a/cibuildwheel/resources/constraints-python312.txt b/cibuildwheel/resources/constraints-python312.txt index ec5abaa74..b5d144675 100644 --- a/cibuildwheel/resources/constraints-python312.txt +++ b/cibuildwheel/resources/constraints-python312.txt @@ -16,7 +16,7 @@ packaging==24.2 # via # build # delocate -pip==25.0.1 +pip==25.1.1 # via -r cibuildwheel/resources/constraints.in platformdirs==4.3.7 # via virtualenv diff --git a/cibuildwheel/resources/constraints-python313.txt b/cibuildwheel/resources/constraints-python313.txt index ec5abaa74..b5d144675 100644 --- a/cibuildwheel/resources/constraints-python313.txt +++ b/cibuildwheel/resources/constraints-python313.txt @@ -16,7 +16,7 @@ packaging==24.2 # via # build # delocate -pip==25.0.1 +pip==25.1.1 # via -r cibuildwheel/resources/constraints.in platformdirs==4.3.7 # via virtualenv diff --git a/cibuildwheel/resources/constraints-python38.txt b/cibuildwheel/resources/constraints-python38.txt index f5d8ee8af..ef5dbbbd7 100644 --- a/cibuildwheel/resources/constraints-python38.txt +++ b/cibuildwheel/resources/constraints-python38.txt @@ -18,7 +18,7 @@ packaging==24.2 # via # build # delocate -pip==25.0.1 +pip==25.1.1 # via -r cibuildwheel/resources/constraints.in platformdirs==4.3.6 # via virtualenv diff --git a/cibuildwheel/resources/constraints-python39.txt b/cibuildwheel/resources/constraints-python39.txt index da66890e6..e58cd57db 100644 --- a/cibuildwheel/resources/constraints-python39.txt +++ b/cibuildwheel/resources/constraints-python39.txt @@ -18,7 +18,7 @@ packaging==24.2 # via # build # delocate -pip==25.0.1 +pip==25.1.1 # via -r cibuildwheel/resources/constraints.in platformdirs==4.3.7 # via virtualenv diff --git a/cibuildwheel/resources/constraints.txt b/cibuildwheel/resources/constraints.txt index ec5abaa74..b5d144675 100644 --- a/cibuildwheel/resources/constraints.txt +++ b/cibuildwheel/resources/constraints.txt @@ -16,7 +16,7 @@ packaging==24.2 # via # build # delocate -pip==25.0.1 +pip==25.1.1 # via -r cibuildwheel/resources/constraints.in platformdirs==4.3.7 # via virtualenv diff --git a/docs/options.md b/docs/options.md index 7c1cb00b4..4f506056b 100644 --- a/docs/options.md +++ b/docs/options.md @@ -1242,7 +1242,14 @@ run your test suite. On all platforms other than Android and iOS, the command is run in a shell, so you can write things like `cmd1 && cmd2`. -On Android and iOS, the value of the `CIBW_TEST_COMMAND` setting is interpreted as the arguments to pass to `python -m` - that is, a Python module name, followed by arguments that will be assigned to `sys.argv`. Shell commands cannot be used. +On Android, the command is parsed by `shlex.split`, and is required to be in one of the following +forms: + +* `python -c command ...` +* `python -m module_name ...` +* `pytest ...` (converted to `python -m pytest ...`) + +On iOS, the command is parsed by `shlex.split`, and is interpreted as the arguments to pass to `python -m` - that is, a Python module name, followed by arguments that will be assigned to `sys.argv`. Platform-specific environment variables are also available:
`CIBW_TEST_COMMAND_MACOS` | `CIBW_TEST_COMMAND_WINDOWS` | `CIBW_TEST_COMMAND_LINUX` | `CIBW_TEST_COMMAND_ANDROID` | `CIBW_TEST_COMMAND_IOS` | `CIBW_TEST_COMMAND_PYODIDE` diff --git a/docs/platforms.md b/docs/platforms.md index 1ce001ed2..13b328622 100644 --- a/docs/platforms.md +++ b/docs/platforms.md @@ -228,16 +228,15 @@ On Linux, the emulator needs access to the KVM virtualization interface, and a D environment variable pointing at an X server. Xvfb is acceptable. The Android test environment can't support running shell scripts, so the -[`CIBW_TEST_COMMAND`](options.md#test-command) value must be specified as if it were -a command line being passed to `python -m ...`. In addition, the project must use +[`CIBW_TEST_COMMAND`](options.md#test-command) value must be a Python command – see its +documentation for details. In addition, the project should use [`CIBW_TEST_SOURCES`](options.md#test-sources) to specify the minimum subset of files that should be copied to the test environment. This is because the test must be run "on device", and the device will not have access to the local project directory. The test process uses the same testbed used by CPython itself to run the CPython test suite. It is a Gradle project that has been configured to have a single JUnit test, -the result of which reports the success or failure of running -`python -m `. +the result of which reports the success or failure of running the test command. ## iOS diff --git a/pyproject.toml b/pyproject.toml index 6f34a685b..1ee413c80 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -39,7 +39,7 @@ classifiers = [ dependencies = [ "bashlex!=0.13", "bracex", - "build >= 1.0.0", + "build>=1.0.0", "certifi", "dependency-groups>=1.2", "filelock", From be664be43ecae440290a77f402edc997a3ace97f Mon Sep 17 00:00:00 2001 From: Malcolm Smith Date: Sun, 4 May 2025 23:48:42 +0100 Subject: [PATCH 10/57] Add type annotations to _cross_venv --- .pre-commit-config.yaml | 2 +- cibuildwheel/resources/_cross_venv.py | 26 ++++++++++++++++---------- 2 files changed, 17 insertions(+), 11 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 3ef6e8bd1..75664952d 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -26,6 +26,7 @@ repos: - id: mypy name: mypy 3.11 on cibuildwheel/ args: ["--python-version=3.11"] + exclude: ^cibuildwheel/resources/_cross_venv.py$ # Requires Python 3.13 or later additional_dependencies: &mypy-dependencies - bracex - build @@ -48,7 +49,6 @@ repos: - validate-pyproject - id: mypy name: mypy 3.13 - exclude: ^cibuildwheel/resources/.*py$ args: ["--python-version=3.13"] additional_dependencies: *mypy-dependencies diff --git a/cibuildwheel/resources/_cross_venv.py b/cibuildwheel/resources/_cross_venv.py index e6182288c..83e4779a3 100644 --- a/cibuildwheel/resources/_cross_venv.py +++ b/cibuildwheel/resources/_cross_venv.py @@ -7,11 +7,12 @@ import sys import sysconfig from pathlib import Path +from typing import Any -def initialize(): +def initialize() -> None: # os ###################################################################### - def cross_os_uname(): + def cross_os_uname() -> os.uname_result: return os.uname_result( ( "Linux", @@ -31,7 +32,7 @@ def cross_os_uname(): # # We can't determine the user-visible Android version number from the API level, so return a # string which will work fine for display, but will fail to parse as a version number. - def cross_android_ver(*args, **kwargs): + def cross_android_ver(*args: Any, **kwargs: Any) -> platform.AndroidVer: return platform.AndroidVer( release=f"API level {cross_getandroidapilevel()}", api_level=cross_getandroidapilevel(), @@ -45,22 +46,27 @@ def cross_android_ver(*args, **kwargs): platform.android_ver = cross_android_ver # sys ##################################################################### - def cross_getandroidapilevel(): - return sysconfig.get_config_var("ANDROID_API_LEVEL") + def cross_getandroidapilevel() -> int: + api_level = sysconfig.get_config_var("ANDROID_API_LEVEL") + assert isinstance(api_level, int) + return api_level - sys.cross_compiling = True # Some packages may recognize this from the crossenv tool. - sys.getandroidapilevel = cross_getandroidapilevel - sys.implementation._multiarch = os.environ["HOST"] + # Some packages may recognize sys.cross_compiling from the crossenv tool. + sys.cross_compiling = True # type: ignore[attr-defined] + sys.getandroidapilevel = cross_getandroidapilevel # type: ignore[attr-defined] + sys.implementation._multiarch = os.environ["HOST"] # type: ignore[attr-defined] sys.platform = "android" # _get_sysconfigdata_name is implemented in terms of sys.abiflags, sys.platform and # sys.implementation._multiarch. Determine the abiflags from the filename. sysconfigdata_path = next(Path(__file__).parent.glob("_sysconfigdata_*.py")) - sys.abiflags = re.match(r"_sysconfigdata_(.*?)_", sysconfigdata_path.name)[1] + abiflags_match = re.match(r"_sysconfigdata_(.*?)_", sysconfigdata_path.name) + assert abiflags_match is not None + sys.abiflags = abiflags_match[1] # sysconfig ############################################################### # - sysconfig._init_config_vars() + sysconfig._init_config_vars() # type: ignore[attr-defined] # sysconfig.get_platform, which determines the wheel tag, is implemented in terms of # sys.platform, sysconfig.get_config_var("ANDROID_API_LEVEL") (see localized_vars in From d56114b2cae3b3127f897f5fb50b129b7944393e Mon Sep 17 00:00:00 2001 From: Malcolm Smith Date: Mon, 5 May 2025 19:43:01 +0100 Subject: [PATCH 11/57] Revert Python 3.8 to pip 25.0.1 --- cibuildwheel/resources/constraints-python38.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cibuildwheel/resources/constraints-python38.txt b/cibuildwheel/resources/constraints-python38.txt index ffe7ce5ed..490e373a9 100644 --- a/cibuildwheel/resources/constraints-python38.txt +++ b/cibuildwheel/resources/constraints-python38.txt @@ -18,7 +18,7 @@ packaging==25.0 # via # build # delocate -pip==25.1.1 +pip==25.0.1 # via -r cibuildwheel/resources/constraints.in platformdirs==4.3.6 # via virtualenv From 71ba70dcb8287a4761b4ee4d93a3b9c81cde381e Mon Sep 17 00:00:00 2001 From: Malcolm Smith Date: Mon, 5 May 2025 22:15:02 +0100 Subject: [PATCH 12/57] Make test-sources required on Android --- cibuildwheel/platforms/android.py | 9 +++++---- docs/options.md | 15 +++++++++------ docs/platforms.md | 6 +++--- 3 files changed, 17 insertions(+), 13 deletions(-) diff --git a/cibuildwheel/platforms/android.py b/cibuildwheel/platforms/android.py index 0acabc3cc..9384ce99f 100644 --- a/cibuildwheel/platforms/android.py +++ b/cibuildwheel/platforms/android.py @@ -365,10 +365,11 @@ def test_wheel(self, wheel: Path) -> None: *self.build_options.test_requires, ) - # Copy test-sources. - # - # TODO: decide what to do if this isn't specified - # (https://github.com/pypa/cibuildwheel/pull/2363#issuecomment-2849413429) + # Copy test-sources. This option is required, as the project directory isn't visible on the + # emulator. + if not self.build_options.test_sources: + msg = "Testing on this platform requires a definition of test-sources." + raise errors.FatalError(msg) cwd_dir = self.tmp_dir / "cwd" cwd_dir.mkdir() copy_test_sources(self.build_options.test_sources, self.build_options.package_dir, cwd_dir) diff --git a/docs/options.md b/docs/options.md index 10040a4ed..ab7ad67ae 100644 --- a/docs/options.md +++ b/docs/options.md @@ -1251,8 +1251,8 @@ run your test suite. On all platforms other than Android and iOS, the command is run in a shell, so you can write things like `cmd1 && cmd2`. -On Android, the command is parsed by `shlex.split`, and is required to be in one of the following -forms: +On Android, the command is parsed by `shlex.split`, and is required to be in +one of the following forms: * `python -c command ...` * `python -m module_name ...` @@ -1372,10 +1372,13 @@ project, required for running the tests. If specified, these files and folders will be copied into a temporary folder, and that temporary folder will be used as the working directory for running the test suite. -The use of `CIBW_TEST_SOURCES` is *required* for Android and iOS tests, because -they run in a virtual machine that does not have access to the project directory. -On these platforms, the files will be copied into the test application, -rather than a temporary folder. +The default is to copy nothing, and run the tests from your project directory. +This is not possible on Android and iOS, because they run tests in a virtual +machine that does not have access to the project directory. On these platforms, +the `CIBW_TEST_SOURCES` option is required. + +If your tests do not need any extra files, you can run them from an almost +empty directory by setting this option to a dummy file such as your README. Platform-specific environment variables are also available:
`CIBW_TEST_SOURCES_MACOS` | `CIBW_TEST_SOURCES_WINDOWS` | `CIBW_TEST_SOURCES_LINUX` | `CIBW_TEST_SOURCES_ANDROID` | `CIBW_TEST_SOURCES_IOS` | `CIBW_TEST_SOURCES_PYODIDE` diff --git a/docs/platforms.md b/docs/platforms.md index 13b328622..935d8c837 100644 --- a/docs/platforms.md +++ b/docs/platforms.md @@ -197,9 +197,9 @@ It also requires the following commands to be on the `PATH`: ### Android version compatibility -Android builds will honor the `ANDROID_API_LEVEL` environment variable to set the minimum -supported [API level](https://developer.android.com/tools/releases/platforms) for -generated wheels. This will default to the minimum API level of the selected Python +Android builds will honor the `ANDROID_API_LEVEL` environment variable to set the +minimum supported [API level](https://developer.android.com/tools/releases/platforms) +for generated wheels. This will default to the minimum API level of the selected Python version. ### Build frontend support From e1ef58f8336f7fef06c6e194a2f66d9bbf6d607b Mon Sep 17 00:00:00 2001 From: Malcolm Smith Date: Tue, 6 May 2025 19:42:12 +0100 Subject: [PATCH 13/57] Add Android integration tests --- cibuildwheel/platforms/android.py | 14 +- cibuildwheel/resources/build-platforms.toml | 4 +- docs/platforms.md | 2 +- test/test_android.py | 218 ++++++++++++++++++++ test/utils.py | 2 +- 5 files changed, 231 insertions(+), 9 deletions(-) create mode 100644 test/test_android.py diff --git a/cibuildwheel/platforms/android.py b/cibuildwheel/platforms/android.py index 9384ce99f..9b5403c15 100644 --- a/cibuildwheel/platforms/android.py +++ b/cibuildwheel/platforms/android.py @@ -343,8 +343,12 @@ def test_wheel(self, wheel: Path) -> None: return log.step("Testing wheel...") - if self.config.arch != arch_synonym(platform.machine(), native_platform(), "android"): - log.warning("Skipping tests on non-native architecture") + native_arch = arch_synonym(platform.machine(), native_platform(), "android") + if self.config.arch != native_arch: + log.warning( + f"Skipping tests for {self.config.arch}, as the build machine only " + f"supports {native_arch}" + ) return if self.build_options.before_test: @@ -368,7 +372,7 @@ def test_wheel(self, wheel: Path) -> None: # Copy test-sources. This option is required, as the project directory isn't visible on the # emulator. if not self.build_options.test_sources: - msg = "Testing on this platform requires a definition of test-sources." + msg = "Testing on Android requires a definition of test-sources." raise errors.FatalError(msg) cwd_dir = self.tmp_dir / "cwd" cwd_dir.mkdir() @@ -382,8 +386,8 @@ def test_wheel(self, wheel: Path) -> None: test_args[:1] = ["-m", test_args[0], "--"] else: msg = ( - f"Test command '{self.build_options.test_command}' is not supported on this " - f"platform. Supported commands are 'python -m', 'python -c' and 'pytest'." + f"Test command '{self.build_options.test_command}' is not supported on " + f"Android. Supported commands are 'python -m', 'python -c' and 'pytest'." ) raise errors.FatalError(msg) diff --git a/cibuildwheel/resources/build-platforms.toml b/cibuildwheel/resources/build-platforms.toml index 77da99898..0dd32a64d 100644 --- a/cibuildwheel/resources/build-platforms.toml +++ b/cibuildwheel/resources/build-platforms.toml @@ -185,8 +185,8 @@ python_configurations = [ [android] python_configurations = [ - { identifier = "cp313-android_arm64_v8a", version = "3.13", url = "https://repo.maven.apache.org/maven2/com/chaquo/python/python/3.13.2/python-3.13.2-aarch64-linux-android.tar.gz" }, - { identifier = "cp313-android_x86_64", version = "3.13", url = "https://repo.maven.apache.org/maven2/com/chaquo/python/python/3.13.2/python-3.13.2-x86_64-linux-android.tar.gz" }, + { identifier = "cp313-android_arm64_v8a", version = "3.13", url = "https://chaquo.com/python-3.13.3+20250503.174536-aarch64-linux-android.tar.gz" }, + { identifier = "cp313-android_x86_64", version = "3.13", url = "https://chaquo.com/python-3.13.3+20250503.185233-x86_64-linux-android.tar.gz" }, ] [ios] diff --git a/docs/platforms.md b/docs/platforms.md index 935d8c837..966ba9dcd 100644 --- a/docs/platforms.md +++ b/docs/platforms.md @@ -229,7 +229,7 @@ environment variable pointing at an X server. Xvfb is acceptable. The Android test environment can't support running shell scripts, so the [`CIBW_TEST_COMMAND`](options.md#test-command) value must be a Python command – see its -documentation for details. In addition, the project should use +documentation for details. In addition, the project must use [`CIBW_TEST_SOURCES`](options.md#test-sources) to specify the minimum subset of files that should be copied to the test environment. This is because the test must be run "on device", and the device will not have access to the local project directory. diff --git a/test/test_android.py b/test/test_android.py new file mode 100644 index 000000000..71525f46c --- /dev/null +++ b/test/test_android.py @@ -0,0 +1,218 @@ +import platform +from dataclasses import dataclass +from subprocess import CalledProcessError +from textwrap import dedent + +import pytest + +from .test_projects import new_c_project +from .utils import cibuildwheel_run + +system_machine = (platform.system(), platform.machine()) +if system_machine not in [("Linux", "x86_64"), ("Darwin", "arm64"), ("Darwin", "x86_64")]: + pytest.skip( + f"Android development tools are not available for {system_machine}", + allow_module_level=True, + ) + + +@dataclass +class Architecture: + linux_machine: str + macos_machine: str + android_abi: str + + +archs = [ + Architecture("aarch64", "arm64", "arm64_v8a"), + Architecture("x86_64", "x86_64", "x86_64"), +] +native_arch = next( + arch for arch in archs if platform.machine() in [arch.linux_machine, arch.macos_machine] +) +other_arch = next(arch for arch in archs if arch != native_arch) + + +cp313_env = { + "CIBW_PLATFORM": "android", + "CIBW_BUILD": "cp313-*", + "CIBW_TEST_SOURCES": "setup.cfg", # Dummy file to ensure the variable is non-empty. +} + + +@pytest.mark.parametrize( + ("frontend", "expected_success"), + [("build", True), ("build[uv]", False), ("pip", False)], +) +def test_frontend(frontend, expected_success, tmp_path, capfd): + new_c_project().generate(tmp_path) + try: + wheels = cibuildwheel_run( + tmp_path, + add_env={**cp313_env, "CIBW_BUILD_FRONTEND": frontend}, + ) + except CalledProcessError: + if expected_success: + pytest.fail("unexpected failure") + assert "Android requires the build frontend to be 'build'" in capfd.readouterr().err + else: + if not expected_success: + pytest.fail("unexpected success") + assert wheels == [f"spam-0.1.0-cp313-cp313-android_21_{native_arch.android_abi}.whl"] + + +# Any tests which involve the testbed app must be run serially, because all copies of the testbed +# app run on the same emulator with the same application ID. +@pytest.mark.serial +def test_archs(tmp_path, capfd): + new_c_project().generate(tmp_path) + wheels = cibuildwheel_run( + tmp_path, + add_env={ + **cp313_env, + "CIBW_ARCHS": "all", + "CIBW_TEST_COMMAND": ( + 'python -c \'import platform; print("machine" + "=" + platform.machine())\'' + ), + }, + ) + assert wheels == [f"spam-0.1.0-cp313-cp313-android_21_{arch.android_abi}.whl" for arch in archs] + + # The native architecture should run tests. + stdout, stderr = capfd.readouterr() + machine_lines = [line for line in stdout.splitlines() if "machine=" in line] + assert len(machine_lines) == 1 + assert machine_lines[0] == f"machine={native_arch.linux_machine}" + + # The non-native architecture should give a warning that it can't run tests. + assert ( + f"warning: Skipping tests for {other_arch.android_abi}, as the build machine " + f"only supports {native_arch.android_abi}" + ) in stderr + + +def test_build_requires(tmp_path, capfd): + # Build-time requirements should be installed for the build platform, not for Android. Prove + # this by installing some non-pure-Python requirements and using them in setup.py. + # + # setup_requires is installed via ProjectBuilder.get_requires_for_build. + project = new_c_project( + setup_py_setup_args_add="setup_requires=['cmake==3.31.4']", + setup_py_add=dedent( + """\ + if "egg_info" not in sys.argv: + import subprocess + subprocess.run(["cmake", "--version"], check=True) + + from bitarray import bitarray + print(f"{bitarray('10110').count()=}") + """ + ), + ) + + # [build_system] requires is installed via ProjectBuilder.build_system_requires. + project.files["pyproject.toml"] = dedent( + """\ + [build-system] + requires = ["setuptools", "wheel", "bitarray==3.3.2"] + """ + ) + + project.generate(tmp_path) + cibuildwheel_run(tmp_path, add_env={**cp313_env}) + + # Test for a specific version to minimize the chance that we ran a system cmake. + stdout = capfd.readouterr().out + assert "cmake version 3.31.4" in stdout + assert "bitarray('10110').count()=3" in stdout + + +@pytest.mark.serial +@pytest.mark.parametrize( + ("command", "expected_success", "expected_output"), + [ + # Success + ("python -c 'import test_spam; test_spam.test_spam()'", True, "Spam test passed"), + ("python -m pytest test_spam.py", True, "=== 1 passed in "), + ("pytest test_spam.py", True, "=== 1 passed in "), + # Build-time failure + ( + "./test_spam.py", + False, + ( + "Test command './test_spam.py' is not supported on Android. " + "Supported commands are 'python -m', 'python -c' and 'pytest'." + ), + ), + # Runtime failure + ("pytest test_ham.py", False, "not found: test_ham.py"), + ], +) +def test_test_command(command, expected_success, expected_output, tmp_path, capfd): + project = new_c_project() + project.files["test_spam.py"] = dedent( + """\ + import spam + + def test_spam(): + assert spam.filter("ham") + assert not spam.filter("spam") + print("Spam test passed") + """ + ) + + project.generate(tmp_path) + try: + cibuildwheel_run( + tmp_path, + add_env={ + **cp313_env, + "CIBW_TEST_SOURCES": "test_spam.py", + "CIBW_TEST_REQUIRES": "pytest==8.3.5", + "CIBW_TEST_COMMAND": command, + }, + ) + except CalledProcessError: + if expected_success: + pytest.fail("unexpected failure") + assert expected_output in capfd.readouterr().err + else: + if not expected_success: + pytest.fail("unexpected success") + assert expected_output in capfd.readouterr().out + + +@pytest.mark.serial +def test_no_test_sources(tmp_path, capfd): + new_c_project().generate(tmp_path) + with pytest.raises(CalledProcessError): + cibuildwheel_run( + tmp_path, + add_env={ + **cp313_env, + "CIBW_TEST_SOURCES": "", + "CIBW_TEST_COMMAND": "python -c 'import sys'", + }, + ) + assert "Testing on Android requires a definition of test-sources." in capfd.readouterr().err + + +@pytest.mark.serial +def test_api_level(tmp_path, capfd): + new_c_project().generate(tmp_path) + wheels = cibuildwheel_run( + tmp_path, + add_env={ + **cp313_env, + "ANDROID_API_LEVEL": "33", + # Verify that Android dependencies can be installed from the Chaquopy repository, and + # that wheels tagged with an older version of Android (in this case 24) are still + # accepted. + "CIBW_TEST_REQUIRES": "bitarray==3.0.0", + "CIBW_TEST_COMMAND": ( + "python -c 'from bitarray import bitarray; print(~bitarray(\"01100\"))'" + ), + }, + ) + assert wheels == [f"spam-0.1.0-cp313-cp313-android_33_{native_arch.android_abi}.whl"] + assert "bitarray('10011')" in capfd.readouterr().out diff --git a/test/utils.py b/test/utils.py index 957ee7295..55f7b7c7a 100644 --- a/test/utils.py +++ b/test/utils.py @@ -141,7 +141,7 @@ def cibuildwheel_run( cwd=project_path, check=True, ) - wheels = [p.name for p in (output_dir or Path(tmp_output_dir)).iterdir()] + wheels = sorted(p.name for p in (output_dir or Path(tmp_output_dir)).iterdir()) return wheels From 40ac49a2e3467b54c5a853e8b0d0b08ceef0174c Mon Sep 17 00:00:00 2001 From: Malcolm Smith Date: Wed, 28 May 2025 18:36:49 +0100 Subject: [PATCH 14/57] Test cleanups --- test/test_android.py | 132 +++++++++++++++++++++++-------------------- 1 file changed, 70 insertions(+), 62 deletions(-) diff --git a/test/test_android.py b/test/test_android.py index 71525f46c..8fc2fe8de 100644 --- a/test/test_android.py +++ b/test/test_android.py @@ -1,3 +1,4 @@ +import os import platform from dataclasses import dataclass from subprocess import CalledProcessError @@ -15,6 +16,12 @@ allow_module_level=True, ) +if "ANDROID_HOME" not in os.environ: + pytest.skip( + "ANDROID_HOME environment variable is not set", + allow_module_level=True, + ) + @dataclass class Architecture: @@ -30,7 +37,6 @@ class Architecture: native_arch = next( arch for arch in archs if platform.machine() in [arch.linux_machine, arch.macos_machine] ) -other_arch = next(arch for arch in archs if arch != native_arch) cp313_env = { @@ -40,25 +46,24 @@ class Architecture: } -@pytest.mark.parametrize( - ("frontend", "expected_success"), - [("build", True), ("build[uv]", False), ("pip", False)], -) -def test_frontend(frontend, expected_success, tmp_path, capfd): +def test_frontend_good(tmp_path): + new_c_project().generate(tmp_path) + wheels = cibuildwheel_run( + tmp_path, + add_env={**cp313_env, "CIBW_BUILD_FRONTEND": "build"}, + ) + assert wheels == [f"spam-0.1.0-cp313-cp313-android_21_{native_arch.android_abi}.whl"] + + +@pytest.mark.parametrize("frontend", ["build[uv]", "pip"]) +def test_frontend_bad(frontend, tmp_path, capfd): new_c_project().generate(tmp_path) - try: - wheels = cibuildwheel_run( + with pytest.raises(CalledProcessError): + cibuildwheel_run( tmp_path, add_env={**cp313_env, "CIBW_BUILD_FRONTEND": frontend}, ) - except CalledProcessError: - if expected_success: - pytest.fail("unexpected failure") - assert "Android requires the build frontend to be 'build'" in capfd.readouterr().err - else: - if not expected_success: - pytest.fail("unexpected success") - assert wheels == [f"spam-0.1.0-cp313-cp313-android_21_{native_arch.android_abi}.whl"] + assert "Android requires the build frontend to be 'build'" in capfd.readouterr().err # Any tests which involve the testbed app must be run serially, because all copies of the testbed @@ -84,11 +89,13 @@ def test_archs(tmp_path, capfd): assert len(machine_lines) == 1 assert machine_lines[0] == f"machine={native_arch.linux_machine}" - # The non-native architecture should give a warning that it can't run tests. - assert ( - f"warning: Skipping tests for {other_arch.android_abi}, as the build machine " - f"only supports {native_arch.android_abi}" - ) in stderr + # The non-native architectures should give a warning that they can't run tests. + for arch in archs: + if arch != native_arch: + assert ( + f"warning: Skipping tests for {arch.android_abi}, as the build machine " + f"only supports {native_arch.android_abi}" + ) in stderr def test_build_requires(tmp_path, capfd): @@ -127,28 +134,8 @@ def test_build_requires(tmp_path, capfd): assert "bitarray('10110').count()=3" in stdout -@pytest.mark.serial -@pytest.mark.parametrize( - ("command", "expected_success", "expected_output"), - [ - # Success - ("python -c 'import test_spam; test_spam.test_spam()'", True, "Spam test passed"), - ("python -m pytest test_spam.py", True, "=== 1 passed in "), - ("pytest test_spam.py", True, "=== 1 passed in "), - # Build-time failure - ( - "./test_spam.py", - False, - ( - "Test command './test_spam.py' is not supported on Android. " - "Supported commands are 'python -m', 'python -c' and 'pytest'." - ), - ), - # Runtime failure - ("pytest test_ham.py", False, "not found: test_ham.py"), - ], -) -def test_test_command(command, expected_success, expected_output, tmp_path, capfd): +@pytest.fixture +def spam_env(tmp_path): project = new_c_project() project.files["test_spam.py"] = dedent( """\ @@ -160,26 +147,47 @@ def test_spam(): print("Spam test passed") """ ) - project.generate(tmp_path) - try: - cibuildwheel_run( - tmp_path, - add_env={ - **cp313_env, - "CIBW_TEST_SOURCES": "test_spam.py", - "CIBW_TEST_REQUIRES": "pytest==8.3.5", - "CIBW_TEST_COMMAND": command, - }, - ) - except CalledProcessError: - if expected_success: - pytest.fail("unexpected failure") - assert expected_output in capfd.readouterr().err - else: - if not expected_success: - pytest.fail("unexpected success") - assert expected_output in capfd.readouterr().out + + return { + **cp313_env, + "CIBW_TEST_SOURCES": "test_spam.py", + "CIBW_TEST_REQUIRES": "pytest==8.3.5", + } + + +@pytest.mark.serial +@pytest.mark.parametrize( + ("command", "expected_output"), + [ + ("python -c 'import test_spam; test_spam.test_spam()'", "Spam test passed"), + ("python -m pytest test_spam.py", "=== 1 passed in "), + ("pytest test_spam.py", "=== 1 passed in "), + ], +) +def test_test_command_good(command, expected_output, tmp_path, spam_env, capfd): + cibuildwheel_run(tmp_path, add_env={**spam_env, "CIBW_TEST_COMMAND": command}) + assert expected_output in capfd.readouterr().out + + +@pytest.mark.serial +@pytest.mark.parametrize( + ("command", "expected_output"), + [ + # Build-time failure + ( + "./test_spam.py", + "Test command './test_spam.py' is not supported on Android. " + "Supported commands are 'python -m', 'python -c' and 'pytest'.", + ), + # Runtime failure + ("pytest test_ham.py", "not found: test_ham.py"), + ], +) +def test_test_command_bad(command, expected_output, tmp_path, spam_env, capfd): + with pytest.raises(CalledProcessError): + cibuildwheel_run(tmp_path, add_env={**spam_env, "CIBW_TEST_COMMAND": command}) + assert expected_output in capfd.readouterr().err @pytest.mark.serial From 535bb7498b2b28ebff36594c4c79f044cf402543 Mon Sep 17 00:00:00 2001 From: Malcolm Smith Date: Wed, 28 May 2025 19:23:17 +0100 Subject: [PATCH 15/57] Add test of all available Python versions --- test/test_android.py | 10 +++++- test/utils.py | 73 +++++++++++++++++++++++++++----------------- 2 files changed, 54 insertions(+), 29 deletions(-) diff --git a/test/test_android.py b/test/test_android.py index 8fc2fe8de..d2f392162 100644 --- a/test/test_android.py +++ b/test/test_android.py @@ -7,7 +7,7 @@ import pytest from .test_projects import new_c_project -from .utils import cibuildwheel_run +from .utils import cibuildwheel_run, expected_wheels system_machine = (platform.system(), platform.machine()) if system_machine not in [("Linux", "x86_64"), ("Darwin", "arm64"), ("Darwin", "x86_64")]: @@ -66,6 +66,14 @@ def test_frontend_bad(frontend, tmp_path, capfd): assert "Android requires the build frontend to be 'build'" in capfd.readouterr().err +def test_python_versions(tmp_path): + new_c_project().generate(tmp_path) + wheels = cibuildwheel_run(tmp_path, add_env={"CIBW_PLATFORM": "android"}) + assert wheels == expected_wheels( + "spam", "0.1.0", target_platform="android", machine_arch=native_arch.android_abi + ) + + # Any tests which involve the testbed app must be run serially, because all copies of the testbed # app run on the same emulator with the same application ID. @pytest.mark.serial diff --git a/test/utils.py b/test/utils.py index 55f7b7c7a..7c2ecb49e 100644 --- a/test/utils.py +++ b/test/utils.py @@ -158,6 +158,7 @@ def expected_wheels( manylinux_versions: list[str] | None = None, musllinux_versions: list[str] | None = None, macosx_deployment_target: str = "10.9", + target_platform: str | None = None, machine_arch: str | None = None, python_abi_tags: list[str] | None = None, include_universal2: bool = False, @@ -167,14 +168,17 @@ def expected_wheels( """ Returns the expected wheels from a run of cibuildwheel. """ + if target_platform is None: + target_platform = platform + if machine_arch is None: machine_arch = pm.machine() - if platform == "linux": + if target_platform == "linux": machine_arch = arch_name_for_linux(machine_arch) architectures = [machine_arch] if not single_arch: - if platform == "linux": + if target_platform == "linux": if machine_arch == "x86_64": architectures.append("i686") elif ( @@ -183,7 +187,7 @@ def expected_wheels( and _AARCH64_CAN_RUN_ARMV7 ): architectures.append("armv7l") - elif platform == "windows" and machine_arch == "AMD64": + elif target_platform == "windows" and machine_arch == "AMD64": architectures.append("x86") return [ @@ -192,6 +196,7 @@ def expected_wheels( for wheel in _expected_wheels( package_name, package_version, + target_platform, architecture, manylinux_versions, musllinux_versions, @@ -206,6 +211,7 @@ def expected_wheels( def _expected_wheels( package_name: str, package_version: str, + platform: str, machine_arch: str, manylinux_versions: list[str] | None, musllinux_versions: list[str] | None, @@ -233,34 +239,37 @@ def _expected_wheels( if musllinux_versions is None: musllinux_versions = ["musllinux_1_2"] - if platform == "pyodide" and python_abi_tags is None: - python_abi_tags = ["cp312-cp312"] if python_abi_tags is None: - python_abi_tags = [ - "cp38-cp38", - "cp39-cp39", - "cp310-cp310", - "cp311-cp311", - "cp312-cp312", - "cp313-cp313", - "cp313-cp313t", - ] + if platform == "android": + python_abi_tags = ["cp313-cp313"] + elif platform == "pyodide": + python_abi_tags = ["cp312-cp312"] + else: + python_abi_tags = [ + "cp38-cp38", + "cp39-cp39", + "cp310-cp310", + "cp311-cp311", + "cp312-cp312", + "cp313-cp313", + "cp313-cp313t", + ] - if machine_arch == "ARM64": - # no CPython 3.8 on Windows ARM64 - python_abi_tags.pop(0) + if machine_arch == "ARM64": + # no CPython 3.8 on Windows ARM64 + python_abi_tags.pop(0) - if machine_arch in ["x86_64", "i686", "AMD64", "aarch64", "arm64"]: - python_abi_tags += [ - "pp38-pypy38_pp73", - "pp39-pypy39_pp73", - "pp310-pypy310_pp73", - "pp311-pypy311_pp73", - ] - if machine_arch in ["x86_64", "AMD64", "aarch64", "arm64"]: - python_abi_tags += [ - "graalpy311-graalpy242_311_native", - ] + if machine_arch in ["x86_64", "i686", "AMD64", "aarch64", "arm64"]: + python_abi_tags += [ + "pp38-pypy38_pp73", + "pp39-pypy39_pp73", + "pp310-pypy310_pp73", + "pp311-pypy311_pp73", + ] + if machine_arch in ["x86_64", "AMD64", "aarch64", "arm64"]: + python_abi_tags += [ + "graalpy311-graalpy242_311_native", + ] if single_python: python_tag = "cp{}{}-".format(*SINGLE_PYTHON_VERSION) @@ -327,6 +336,14 @@ def _expected_wheels( if include_universal2: platform_tags.append(f"macosx_{min_macosx.replace('.', '_')}_universal2") + + elif platform == "android": + api_level = { + "cp313-cp313": 21, + "cp314-cp314": 24, + }[python_abi_tag] + platform_tags = [f"android_{api_level}_{machine_arch}"] + else: msg = f"Unsupported platform {platform!r}" raise Exception(msg) From 00b7cc1d2c9ae377d8557afb2dd4be50ace46e6f Mon Sep 17 00:00:00 2001 From: Malcolm Smith Date: Wed, 28 May 2025 22:45:29 +0100 Subject: [PATCH 16/57] Update test-sources and test-command behavior to match iOS --- cibuildwheel/platforms/android.py | 27 ++++++++++++++----- .../resources/testing_temp_dir_file.py | 4 +-- docs/options.md | 12 +++------ test/test_android.py | 22 ++++++++++----- 4 files changed, 42 insertions(+), 23 deletions(-) diff --git a/cibuildwheel/platforms/android.py b/cibuildwheel/platforms/android.py index 9b5403c15..d0e01e105 100644 --- a/cibuildwheel/platforms/android.py +++ b/cibuildwheel/platforms/android.py @@ -369,14 +369,29 @@ def test_wheel(self, wheel: Path) -> None: *self.build_options.test_requires, ) - # Copy test-sources. This option is required, as the project directory isn't visible on the - # emulator. - if not self.build_options.test_sources: - msg = "Testing on Android requires a definition of test-sources." - raise errors.FatalError(msg) + # Copy test-sources. cwd_dir = self.tmp_dir / "cwd" cwd_dir.mkdir() - copy_test_sources(self.build_options.test_sources, self.build_options.package_dir, cwd_dir) + if self.build_options.test_sources: + copy_test_sources( + self.build_options.test_sources, self.build_options.package_dir, cwd_dir + ) + else: + (cwd_dir / "test_fail.py").write_text( + resources.TEST_FAIL_CWD_FILE.read_text(), + ) + + # Android doesn't support placeholders in the test command. + if any( + ("{" + placeholder + "}") in self.build_options.test_command + for placeholder in ["project", "package"] + ): + msg = ( + f"Test command '{self.build_options.test_command}' with a " + "'{project}' or '{package}' placeholder is not supported on Android, " + "because the source directory is not visible on the emulator." + ) + raise errors.FatalError(msg) # Parse test-command. test_args = shlex.split(self.build_options.test_command) diff --git a/cibuildwheel/resources/testing_temp_dir_file.py b/cibuildwheel/resources/testing_temp_dir_file.py index 094f2f6fb..cc3e15a2b 100644 --- a/cibuildwheel/resources/testing_temp_dir_file.py +++ b/cibuildwheel/resources/testing_temp_dir_file.py @@ -8,11 +8,11 @@ class TestStringMethods(unittest.TestCase): def test_fail(self) -> NoReturn: - if sys.platform == "ios": + if sys.platform in ["android", "ios"]: msg = ( "You tried to run tests from the testbed app's working " "directory, without specifying `test-sources`. " - "On iOS, you must copy your test files to the testbed app by " + "On this platform, you must copy your test files to the testbed app by " "setting the `test-sources` option in your cibuildwheel " "configuration." ) diff --git a/docs/options.md b/docs/options.md index d53a244ef..159786583 100644 --- a/docs/options.md +++ b/docs/options.md @@ -1302,6 +1302,10 @@ tree. To access your test code, you have a couple of options: - `{project}` is an absolute path to the project root - the working directory where cibuildwheel was called. + These placeholders are not available on Android and iOS, because those + platforms run tests in a virtual machine that does not have access to + the build machine's filesystem. + On all platforms other than Android and iOS, the command is run in a shell, so you can write things like `cmd1 && cmd2`. On Android and iOS, the command is parsed by `shlex.split`, and is required to @@ -1432,14 +1436,6 @@ project, required for running the tests. If specified, these files and folders will be copied into a temporary folder, and that temporary folder will be used as the working directory for running the test suite. -The default is to copy nothing, and run the tests from your project directory. -This is not possible on Android and iOS, because they run tests in a virtual -machine that does not have access to the project directory. On these platforms, -the `CIBW_TEST_SOURCES` option is required. - -If your tests do not need any extra files, you can run them from an almost -empty directory by setting this option to a dummy file such as your README. - Platform-specific environment variables are also available:
`CIBW_TEST_SOURCES_MACOS` | `CIBW_TEST_SOURCES_WINDOWS` | `CIBW_TEST_SOURCES_LINUX` | `CIBW_TEST_SOURCES_ANDROID` | `CIBW_TEST_SOURCES_IOS` | `CIBW_TEST_SOURCES_PYODIDE` diff --git a/test/test_android.py b/test/test_android.py index cf9390273..6d531a9a6 100644 --- a/test/test_android.py +++ b/test/test_android.py @@ -42,7 +42,6 @@ class Architecture: cp313_env = { "CIBW_PLATFORM": "android", "CIBW_BUILD": "cp313-*", - "CIBW_TEST_SOURCES": "setup.cfg", # Dummy file to ensure the variable is non-empty. } @@ -190,6 +189,16 @@ def test_test_command_good(command, expected_output, tmp_path, spam_env, capfd): ), # Runtime failure ("pytest test_ham.py", "not found: test_ham.py"), + ( + "pytest {project}", + "Test command 'pytest {project}' with a '{project}' or '{package}' " + "placeholder is not supported on Android", + ), + ( + "pytest {package}", + "Test command 'pytest {package}' with a '{project}' or '{package}' " + "placeholder is not supported on Android", + ), ], ) def test_test_command_bad(command, expected_output, tmp_path, spam_env, capfd): @@ -204,13 +213,12 @@ def test_no_test_sources(tmp_path, capfd): with pytest.raises(CalledProcessError): cibuildwheel_run( tmp_path, - add_env={ - **cp313_env, - "CIBW_TEST_SOURCES": "", - "CIBW_TEST_COMMAND": "python -c 'import sys'", - }, + add_env={**cp313_env, "CIBW_TEST_COMMAND": "python -m unittest discover"}, ) - assert "Testing on Android requires a definition of test-sources." in capfd.readouterr().err + assert ( + "On this platform, you must copy your test files to the testbed app by " + "setting the `test-sources` option" + ) in capfd.readouterr().err @pytest.mark.serial From 83f6073bdb0f0ebbc75b0be91a373d62f1c22a04 Mon Sep 17 00:00:00 2001 From: Malcolm Smith Date: Wed, 28 May 2025 23:39:09 +0100 Subject: [PATCH 17/57] Documentation cleanups --- README.md | 16 ++++++++-------- docs/options.md | 10 +++++----- docs/platforms.md | 9 +++------ 3 files changed, 16 insertions(+), 19 deletions(-) diff --git a/README.md b/README.md index a6f37090e..3df0a21b0 100644 --- a/README.md +++ b/README.md @@ -56,14 +56,14 @@ Usage `cibuildwheel` runs inside a CI service. Supported platforms depend on which service you're using: -| | Linux | macOS | Windows | Linux ARM | macOS ARM | Windows ARM | iOS | -|-----------------|-------|-------|---------|-----------|-----------|-------------|-----| -| GitHub Actions | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅³ | -| Azure Pipelines | ✅ | ✅ | ✅ | | ✅ | ✅² | ✅³ | -| Travis CI | ✅ | | ✅ | ✅ | | | | -| CircleCI | ✅ | ✅ | | ✅ | ✅ | | ✅³ | -| Gitlab CI | ✅ | ✅ | ✅ | ✅¹ | ✅ | | ✅³ | -| Cirrus CI | ✅ | ✅ | ✅ | ✅ | ✅ | | ✅³ | +| | Linux | macOS | Windows | Linux ARM | macOS ARM | Windows ARM | Android | iOS | +|-----------------|-------|-------|---------|-----------|-----------|-------------|---------|-----| +| GitHub Actions | ✅ | ✅ | ✅ | ✅ | ✅ | ✅² | ✅⁴ | ✅³ | +| Azure Pipelines | ✅ | ✅ | ✅ | | ✅ | ✅² | ✅⁴ | ✅³ | +| Travis CI | ✅ | | ✅ | ✅ | | | ✅⁴ | | +| CircleCI | ✅ | ✅ | | ✅ | ✅ | | ✅⁴ | ✅³ | +| Gitlab CI | ✅ | ✅ | ✅ | ✅¹ | ✅ | | ✅⁴ | ✅³ | +| Cirrus CI | ✅ | ✅ | ✅ | ✅ | ✅ | | ✅⁴ | ✅³ | ¹ [Requires emulation](https://cibuildwheel.pypa.io/en/stable/faq/#emulation), distributed separately. Other services may also support Linux ARM through emulation or third-party build hosts, but these are not tested in our CI.
² [Uses cross-compilation](https://cibuildwheel.pypa.io/en/stable/faq/#windows-arm64). It is not possible to test `arm64` on this CI platform.
diff --git a/docs/options.md b/docs/options.md index 159786583..f3192ae8a 100644 --- a/docs/options.md +++ b/docs/options.md @@ -447,8 +447,8 @@ possible, both through `--installer=uv` passed to build, as well as when making all build and test environments. This will generally speed up cibuildwheel. Make sure you have an external uv on Windows and macOS, either by pre-installing it, or installing cibuildwheel with the uv extra, -`cibuildwheel[uv]`. uv currently does not support Windows ARM, -musllinux s390x, Android, or iOS. Legacy dependencies like +`cibuildwheel[uv]`. uv currently does not support Windows on ARM, +musllinux on s390x, Android, or iOS. Legacy dependencies like setuptools on Python < 3.12 and pip are not installed if using uv. On Android and Pyodide, only "build" is supported. @@ -1302,9 +1302,9 @@ tree. To access your test code, you have a couple of options: - `{project}` is an absolute path to the project root - the working directory where cibuildwheel was called. - These placeholders are not available on Android and iOS, because those - platforms run tests in a virtual machine that does not have access to - the build machine's filesystem. + These placeholders are not available on Android and iOS, because those + platforms run tests in a virtual machine that does not have access to + the build machine's filesystem. On all platforms other than Android and iOS, the command is run in a shell, so you can write things like `cmd1 && cmd2`. diff --git a/docs/platforms.md b/docs/platforms.md index 6e07d2fcf..7b23d378b 100644 --- a/docs/platforms.md +++ b/docs/platforms.md @@ -237,11 +237,8 @@ On Linux, the emulator needs access to the KVM virtualization interface, and a D environment variable pointing at an X server. Xvfb is acceptable. The Android test environment can't support running shell scripts, so the -[`CIBW_TEST_COMMAND`](options.md#test-command) value must be a Python command – see its -documentation for details. In addition, the project must use -[`CIBW_TEST_SOURCES`](options.md#test-sources) to specify the minimum subset of files -that should be copied to the test environment. This is because the test must be run "on -device", and the device will not have access to the local project directory. +[`test-command`](options.md#test-command) must be a Python command – see its +documentation for details. The test process uses the same testbed used by CPython itself to run the CPython test suite. It is a Gradle project that has been configured to have a single JUnit test, @@ -311,6 +308,6 @@ If your project requires additional tools to build (such as `cmake`, `ninja`, or If tests have been configured, the test suite will be executed on the simulator matching the architecture of the build machine - that is, if you're building on an ARM64 macOS machine, the ARM64 wheel will be tested on an ARM64 simulator. It is not possible to use cibuildwheel to test wheels on other simulators, or on physical devices. -The iOS test environment can't support running shell scripts, so the [`CIBW_TEST_COMMAND`](options.md#test-command) value must be specified as if it were a command line being passed to `python -m ...`. In addition, the project must use [`CIBW_TEST_SOURCES`](options.md#test-sources) to specify the minimum subset of files that should be copied to the test environment. This is because the test must be run "on device", and the simulator device will not have access to the local project directory. +The iOS test environment can't support running shell scripts, so the [`CIBW_TEST_COMMAND`](options.md#test-command) value must be specified as if it were a command line being passed to `python -m ...`. The test process uses the same testbed used by CPython itself to run the CPython test suite. It is an Xcode project that has been configured to have a single Xcode "XCUnit" test - the result of which reports the success or failure of running `python -m `. From 6e90e0188d273f4724f815ba06a490b37fbb9c24 Mon Sep 17 00:00:00 2001 From: Malcolm Smith Date: Fri, 30 May 2025 14:25:18 +0100 Subject: [PATCH 18/57] Replace Builder class with a set of global functions --- cibuildwheel/options.py | 5 +- cibuildwheel/platforms/android.py | 676 +++++++++--------- cibuildwheel/platforms/ios.py | 3 +- cibuildwheel/platforms/linux.py | 4 +- cibuildwheel/platforms/macos.py | 4 +- cibuildwheel/platforms/pyodide.py | 4 +- cibuildwheel/platforms/windows.py | 4 +- cibuildwheel/resources/_cross_venv.pth | 1 + cibuildwheel/resources/_cross_venv.py | 3 + .../resources/cibuildwheel.schema.json | 2 +- unit_test/main_tests/main_options_test.py | 3 +- unit_test/options_test.py | 4 +- 12 files changed, 364 insertions(+), 349 deletions(-) create mode 100644 cibuildwheel/resources/_cross_venv.pth diff --git a/cibuildwheel/options.py b/cibuildwheel/options.py index 7c98f1e21..84b0e8d7a 100644 --- a/cibuildwheel/options.py +++ b/cibuildwheel/options.py @@ -109,7 +109,7 @@ class BuildOptions: test_groups: list[str] test_environment: ParsedEnvironment build_verbosity: int - build_frontend: BuildFrontendConfig | None + build_frontend: BuildFrontendConfig config_settings: str container_engine: OCIContainerEngineConfig pyodide_version: str | None @@ -766,9 +766,8 @@ def _compute_build_options(self, identifier: str | None) -> BuildOptions: env_plat=False, option_format=ShlexTableFormat(sep="; ", pair_sep=":", allow_merge=False), ) - build_frontend: BuildFrontendConfig | None if not build_frontend_str or build_frontend_str == "default": - build_frontend = None + build_frontend = BuildFrontendConfig("build") else: try: build_frontend = BuildFrontendConfig.from_config_string(build_frontend_str) diff --git a/cibuildwheel/platforms/android.py b/cibuildwheel/platforms/android.py index d0e01e105..825812167 100644 --- a/cibuildwheel/platforms/android.py +++ b/cibuildwheel/platforms/android.py @@ -5,23 +5,21 @@ import shlex import shutil import subprocess -from collections.abc import Generator -from contextlib import contextmanager from dataclasses import dataclass from pathlib import Path from pprint import pprint from typing import Any from build import ProjectBuilder +from build.env import IsolatedEnv from filelock import FileLock from .. import errors from ..architecture import Architecture, arch_synonym, native_platform -from ..frontend import BuildFrontendConfig, get_build_frontend_extra_flags, parse_config_settings +from ..frontend import get_build_frontend_extra_flags, parse_config_settings from ..logger import log from ..options import BuildOptions, Options from ..selector import BuildSelector -from ..typing import PathOrStr from ..util import resources from ..util.cmd import call, shell from ..util.file import CIBW_CACHE_PATH, copy_test_sources, download, move_file @@ -89,356 +87,370 @@ def before_all(options: Options, python_configurations: list[PythonConfiguration ) -@dataclass -class Builder: - config: PythonConfiguration - build_options: BuildOptions - tmp_dir: Path - built_wheels: list[Path] +def build(options: Options, tmp_path: Path) -> None: + configs = get_python_configurations( + options.globals.build_selector, options.globals.architectures + ) + if not configs: + return - def build(self) -> None: - log.build_start(self.config.identifier) - self.tmp_dir.mkdir() - self.setup_python() + try: + before_all(options, configs) - compatible_wheel = find_compatible_wheel(self.built_wheels, self.config.identifier) - if compatible_wheel: - print( - f"\nFound previously built wheel {compatible_wheel.name} that is " - f"compatible with {self.config.identifier}. Skipping build step..." - ) - repaired_wheel = compatible_wheel - else: - self.setup_env() - self.before_build() - repaired_wheel = self.build_wheel() - - self.test_wheel(repaired_wheel) - if compatible_wheel is None: - self.built_wheels.append( - move_file(repaired_wheel, self.build_options.output_dir / repaired_wheel.name) + built_wheels: list[Path] = [] + for config in configs: + log.build_start(config.identifier) + build_options = options.build_options(config.identifier) + build_path = tmp_path / config.identifier + build_path.mkdir() + + python_dir = setup_target_python(config, build_path) + + compatible_wheel = find_compatible_wheel(built_wheels, config.identifier) + if compatible_wheel: + print( + f"\nFound previously built wheel {compatible_wheel.name} that is " + f"compatible with {config.identifier}. Skipping build step..." + ) + repaired_wheel = compatible_wheel + else: + env, android_env = setup_env(config, build_options, build_path, python_dir) + before_build(build_options, env) + repaired_wheel = build_wheel(build_options, build_path, android_env) + + test_wheel( + config, build_options, build_path, python_dir, env, android_env, repaired_wheel ) - shutil.rmtree(self.tmp_dir) - log.build_end() - - def setup_python(self) -> None: - log.step("Installing target Python...") - python_tgz = CIBW_CACHE_PATH / self.config.url.rpartition("/")[-1] - with FileLock(f"{python_tgz}.lock"): - if not python_tgz.exists(): - download(self.config.url, python_tgz) + if compatible_wheel is None: + built_wheels.append( + move_file(repaired_wheel, build_options.output_dir / repaired_wheel.name) + ) - self.python_dir = self.tmp_dir / "python" - self.python_dir.mkdir() - shutil.unpack_archive(python_tgz, self.python_dir) + shutil.rmtree(build_path) + log.build_end() - def setup_env(self) -> None: - log.step("Setting up build environment...") + except subprocess.CalledProcessError as error: + msg = f"Command {error.cmd} failed with code {error.returncode}. {error.stdout or ''}" + raise errors.FatalError(msg) from error - python_exe_name = f"python{self.config.version}" - python_exe = shutil.which(python_exe_name) - if not python_exe: - msg = f"Couldn't find {python_exe_name} on the PATH" - raise errors.FatalError(msg) - # Create virtual environment - self.venv_dir = self.tmp_dir / "venv" - dependency_constraint = self.build_options.dependency_constraints.get_for_python_version( - version=self.config.version, tmp_dir=self.tmp_dir - ) - self.env = virtualenv( - self.config.version, - Path(python_exe), - self.venv_dir, - dependency_constraint, - use_uv=False, - ) +def setup_target_python(config: PythonConfiguration, build_path: Path) -> Path: + log.step("Installing target Python...") + python_tgz = CIBW_CACHE_PATH / config.url.rpartition("/")[-1] + with FileLock(f"{python_tgz}.lock"): + if not python_tgz.exists(): + download(config.url, python_tgz) - # Apply custom environment variables, and check environment is still valid - self.env = self.build_options.environment.as_dictionary(self.env) - self.env["PIP_DISABLE_PIP_VERSION_CHECK"] = "1" - for command in ["python", "pip"]: - which = call("which", command, env=self.env, capture_stdout=True).strip() - if which != f"{self.venv_dir}/bin/{command}": - msg = ( - f"{command} available on PATH doesn't match our installed instance. If you " - f"have modified PATH, ensure that you don't overwrite cibuildwheel's entry " - f"or insert {command} above it." - ) - raise errors.FatalError(msg) - call(command, "--version", env=self.env) - - # Add cross-venv files, which will be activated by simulate_android. - self.site_packages = next(self.venv_dir.glob("lib/python*/site-packages")) - for path in [ - resources.PATH / "_cross_venv.py", - next(self.python_dir.glob("prefix/lib/python*/_sysconfigdata_*.py")), - ]: - out_path = Path(shutil.copy(path, self.site_packages)) - if "sysconfigdata" in path.name: - self.localize_sysconfigdata(out_path) - - # Install build tools - self.build_frontend = self.build_options.build_frontend or BuildFrontendConfig("build") - if self.build_frontend.name != "build": - msg = "Android requires the build frontend to be 'build'" - raise errors.FatalError(msg) - self.pip_install("build", *constraint_flags(dependency_constraint)) + python_dir = build_path / "python" + python_dir.mkdir() + shutil.unpack_archive(python_tgz, python_dir) + return python_dir - # Install build-time requirements. These must be installed for the build platform, not for - # Android, which is why we can't allow them to be installed by the `build` subprocess. - pb = ProjectBuilder( - self.build_options.package_dir, python_executable=f"{self.venv_dir}/bin/python" - ) - self.pip_install(*pb.build_system_requires) - # get_requires_for_build runs the package's build script, so it must be called while - # simulating Android. - with self.simulate_android(): - requires_for_build = pb.get_requires_for_build( - "wheel", parse_config_settings(self.build_options.config_settings) - ) - self.pip_install(*requires_for_build) - - def localize_sysconfigdata(self, sysconfigdata_path: Path) -> None: - spec = importlib.util.spec_from_file_location(sysconfigdata_path.stem, sysconfigdata_path) - assert spec is not None - module = importlib.util.module_from_spec(spec) - assert spec.loader is not None - spec.loader.exec_module(module) - - with sysconfigdata_path.open("w") as f: - f.write("# Generated by cibuildwheel\n") - f.write("build_time_vars = ") - self.sysconfigdata = self.localized_vars( - module.build_time_vars, self.python_dir / "prefix" - ) - pprint(self.sysconfigdata, stream=f, compact=True) - - def localized_vars(self, orig_vars: dict[str, Any], prefix: Path) -> dict[str, Any]: - orig_prefix = orig_vars["prefix"] - localized_vars = {} - for key, value in orig_vars.items(): - # The host's sysconfigdata will include references to build-time paths. - # Update these to refer to the current prefix. - final = value - if isinstance(final, str): - final = final.replace(orig_prefix, str(prefix)) - - if key == "ANDROID_API_LEVEL": - if api_level := os.environ.get(key): - final = int(api_level) - - # Build systems vary in whether FLAGS variables are read from sysconfig, and if so, - # whether they're replaced by environment variables or combined with them. Even - # setuptools has changed its behavior here - # (https://github.com/pypa/setuptools/issues/4836). - # - # Ensure consistency by clearing the sysconfig variables and letting the environment - # variables take effect alone. This will also work for any non-Python build systems - # which the build script may call. - elif key in ["CFLAGS", "CXXFLAGS", "LDFLAGS"]: - final = "" - - # These variables contain an embedded copy of LDFLAGS. - elif key in ["LDSHARED", "LDCXXSHARED"]: - final = final.removesuffix(" " + orig_vars["LDFLAGS"]) - - localized_vars[key] = final - - return localized_vars - - def pip_install(self, *args: PathOrStr) -> None: - if args: - call("pip", "install", "--upgrade", *args, env=self.env) - - @contextmanager - def simulate_android(self) -> Generator[None]: - if not hasattr(self, "android_env"): - self.android_env = self.env.copy() - env_output = call( - self.python_dir / "android.py", "env", env=self.env, capture_stdout=True - ) - for line in env_output.splitlines(): - key, value = line.removeprefix("export ").split("=", 1) - value_split = shlex.split(value) - assert len(value_split) == 1, value_split - self.android_env[key] = value_split[0] - - # localized_vars cleared the CFLAGS and CXXFLAGS in the sysconfigdata, but most - # packages take their optimization flags from these variables. Pass these flags via - # environment variables instead. - # - # We don't enable debug information, because it significantly increases binary size, - # and most Android app developers don't have the NDK installed, so they would have no - # way to strip it. - opt = " ".join( - word for word in self.sysconfigdata["OPT"].split() if not word.startswith("-g") - ) - for key in ["CFLAGS", "CXXFLAGS"]: - self.android_env[key] += " " + opt - - # Format the environment so it can be pasted into a shell. - for key, value in sorted(self.android_env.items()): - if self.env.get(key) != value: - print(f"export {key}={shlex.quote(value)}") - - original_env = {key: os.environ.get(key) for key in self.android_env} - os.environ.update(self.android_env) - - pth_file = self.site_packages / "_cross_venv.pth" - pth_file.write_text("import _cross_venv; _cross_venv.initialize()") - - try: - yield - finally: - pth_file.unlink() - for key, original_value in original_env.items(): - if original_value is None: - del os.environ[key] - else: - os.environ[key] = original_value - - def before_build(self) -> None: - if self.build_options.before_build: - log.step("Running before_build...") - shell_prepared(self.build_options.before_build, self.build_options, self.env) - - def build_wheel(self) -> Path: - log.step("Building wheel...") - built_wheel_dir = self.tmp_dir / "built_wheel" - with self.simulate_android(): - call( - "python", - "-m", - "build", - self.build_options.package_dir, - "--wheel", - "--no-isolation", - "--skip-dependency-check", - f"--outdir={built_wheel_dir}", - *get_build_frontend_extra_flags( - self.build_frontend, - self.build_options.build_verbosity, - self.build_options.config_settings, - ), - ) +def setup_env( + config: PythonConfiguration, build_options: BuildOptions, build_path: Path, python_dir: Path +) -> tuple[dict[str, str], dict[str, str]]: + log.step("Setting up build environment...") - built_wheels = list(built_wheel_dir.glob("*.whl")) - if len(built_wheels) != 1: - msg = f"{built_wheel_dir} contains {len(built_wheels)} wheels; expected 1" - raise errors.FatalError(msg) - built_wheel = built_wheels[0] - - if built_wheel.name.endswith("none-any.whl"): - raise errors.NonPlatformWheelError() - return built_wheel - - def test_wheel(self, wheel: Path) -> None: - if not ( - self.build_options.test_command - and self.build_options.test_selector(self.config.identifier) - ): - return - - log.step("Testing wheel...") - native_arch = arch_synonym(platform.machine(), native_platform(), "android") - if self.config.arch != native_arch: - log.warning( - f"Skipping tests for {self.config.arch}, as the build machine only " - f"supports {native_arch}" - ) - return - - if self.build_options.before_test: - shell_prepared(self.build_options.before_test, self.build_options, self.env) - - # Install the wheel and test-requires. - site_packages_dir = self.tmp_dir / "site-packages" - site_packages_dir.mkdir() - self.pip_install( - "--only-binary=:all:", - "--platform", - f"android_{self.sysconfigdata['ANDROID_API_LEVEL']}_{self.config.arch}", - "--extra-index-url", - "https://chaquo.com/pypi-13.1/", - "--target", - site_packages_dir, - f"{wheel}{self.build_options.test_extras}", - *self.build_options.test_requires, - ) + python_exe_name = f"python{config.version}" + python_exe = shutil.which(python_exe_name) + if not python_exe: + msg = f"Couldn't find {python_exe_name} on the PATH" + raise errors.FatalError(msg) - # Copy test-sources. - cwd_dir = self.tmp_dir / "cwd" - cwd_dir.mkdir() - if self.build_options.test_sources: - copy_test_sources( - self.build_options.test_sources, self.build_options.package_dir, cwd_dir - ) - else: - (cwd_dir / "test_fail.py").write_text( - resources.TEST_FAIL_CWD_FILE.read_text(), - ) + # Create virtual environment + venv_dir = build_path / "venv" + dependency_constraint = build_options.dependency_constraints.get_for_python_version( + version=config.version, tmp_dir=build_path + ) + env = virtualenv( + config.version, + Path(python_exe), + venv_dir, + dependency_constraint, + use_uv=False, + ) - # Android doesn't support placeholders in the test command. - if any( - ("{" + placeholder + "}") in self.build_options.test_command - for placeholder in ["project", "package"] - ): + # Apply custom environment variables, and check environment is still valid + env = build_options.environment.as_dictionary(env) + env["PIP_DISABLE_PIP_VERSION_CHECK"] = "1" + for command in ["python", "pip"]: + which = call("which", command, env=env, capture_stdout=True).strip() + if which != f"{venv_dir}/bin/{command}": msg = ( - f"Test command '{self.build_options.test_command}' with a " - "'{project}' or '{package}' placeholder is not supported on Android, " - "because the source directory is not visible on the emulator." + f"{command} available on PATH doesn't match our installed instance. If you " + f"have modified PATH, ensure that you don't overwrite cibuildwheel's entry " + f"or insert {command} above it." ) raise errors.FatalError(msg) + call(command, "--version", env=env) + + # Construct an alternative environment which simulates running on Android. + android_env = setup_android_env(python_dir, venv_dir, env) + + # Install build tools + build_frontend = build_options.build_frontend + if build_frontend.name != "build": + msg = "Android requires the build frontend to be 'build'" + raise errors.FatalError(msg) + call("pip", "install", "build", *constraint_flags(dependency_constraint), env=env) + + # Install build-time requirements. These must be installed for the build platform, + # but queried while simulating Android. The `build` CLI doesn't support this + # combination, so we use its API to query the requirements, but install them + # ourselves. We'll later run `build` in the same environment, passing the + # `--no-isolation` option. + class AndroidEnv(IsolatedEnv): + @property + def python_executable(self) -> str: + return f"{venv_dir}/bin/python" + + def make_extra_environ(self) -> dict[str, str]: + return android_env + + pb = ProjectBuilder.from_isolated_env(AndroidEnv(), build_options.package_dir) + if pb.build_system_requires: + call("pip", "install", *pb.build_system_requires, env=env) + + requires_for_build = pb.get_requires_for_build( + "wheel", parse_config_settings(build_options.config_settings) + ) + if requires_for_build: + call("pip", "install", *requires_for_build, env=env) + + return env, android_env + + +def localize_sysconfigdata(sysconfigdata_path: Path, python_dir: Path) -> dict[str, Any]: + spec = importlib.util.spec_from_file_location(sysconfigdata_path.stem, sysconfigdata_path) + assert spec is not None + module = importlib.util.module_from_spec(spec) + assert spec.loader is not None + spec.loader.exec_module(module) + + with sysconfigdata_path.open("w") as f: + f.write("# Generated by cibuildwheel\n") + f.write("build_time_vars = ") + sysconfigdata = localized_vars(module.build_time_vars, python_dir / "prefix") + pprint(sysconfigdata, stream=f, compact=True) + return sysconfigdata + + +def localized_vars(orig_vars: dict[str, Any], prefix: Path) -> dict[str, Any]: + orig_prefix = orig_vars["prefix"] + localized_vars = {} + for key, value in orig_vars.items(): + # The host's sysconfigdata will include references to build-time paths. + # Update these to refer to the current prefix. + final = value + if isinstance(final, str): + final = final.replace(orig_prefix, str(prefix)) + + if key == "ANDROID_API_LEVEL": + if api_level := os.environ.get(key): + final = int(api_level) + + # Build systems vary in whether FLAGS variables are read from sysconfig, and if so, + # whether they're replaced by environment variables or combined with them. Even + # setuptools has changed its behavior here + # (https://github.com/pypa/setuptools/issues/4836). + # + # Ensure consistency by clearing the sysconfig variables and letting the environment + # variables take effect alone. This will also work for any non-Python build systems + # which the build script may call. + elif key in ["CFLAGS", "CXXFLAGS", "LDFLAGS"]: + final = "" + + # These variables contain an embedded copy of LDFLAGS. + elif key in ["LDSHARED", "LDCXXSHARED"]: + final = final.removesuffix(" " + orig_vars["LDFLAGS"]) + + localized_vars[key] = final + + return localized_vars + + +def setup_android_env(python_dir: Path, venv_dir: Path, env: dict[str, str]) -> dict[str, str]: + site_packages = next(venv_dir.glob("lib/python*/site-packages")) + for suffix in ["pth", "py"]: + shutil.copy(resources.PATH / f"_cross_venv.{suffix}", site_packages) + + sysconfigdata_path = Path( + shutil.copy( + next(python_dir.glob("prefix/lib/python*/_sysconfigdata_*.py")), + site_packages, + ) + ) + sysconfigdata = localize_sysconfigdata(sysconfigdata_path, python_dir) + + android_env = env.copy() + android_env["CIBW_CROSS_VENV"] = "1" # Activates the code in _cross_venv.py. + + env_output = call(python_dir / "android.py", "env", env=env, capture_stdout=True) + for line in env_output.splitlines(): + key, value = line.removeprefix("export ").split("=", 1) + value_split = shlex.split(value) + assert len(value_split) == 1, value_split + android_env[key] = value_split[0] + + # localized_vars cleared the CFLAGS and CXXFLAGS in the sysconfigdata, but most + # packages take their optimization flags from these variables. Pass these flags via + # environment variables instead. + # + # We don't enable debug information, because it significantly increases binary size, + # and most Android app developers don't have the NDK installed, so they would have no + # way to strip it. + opt = " ".join(word for word in sysconfigdata["OPT"].split() if not word.startswith("-g")) + for key in ["CFLAGS", "CXXFLAGS"]: + android_env[key] += " " + opt + + # Format the environment so it can be pasted into a shell. + for key, value in sorted(android_env.items()): + if env.get(key) != value: + print(f"export {key}={shlex.quote(value)}") + + return android_env + + +def before_build(build_options: BuildOptions, env: dict[str, str]) -> None: + if build_options.before_build: + log.step("Running before_build...") + shell_prepared(build_options.before_build, build_options, env) + + +def build_wheel(build_options: BuildOptions, build_path: Path, android_env: dict[str, str]) -> Path: + log.step("Building wheel...") + built_wheel_dir = build_path / "built_wheel" + call( + "python", + "-m", + "build", + build_options.package_dir, + "--wheel", + "--no-isolation", + "--skip-dependency-check", + f"--outdir={built_wheel_dir}", + *get_build_frontend_extra_flags( + build_options.build_frontend, + build_options.build_verbosity, + build_options.config_settings, + ), + env=android_env, + ) - # Parse test-command. - test_args = shlex.split(self.build_options.test_command) - if test_args[:2] in [["python", "-c"], ["python", "-m"]]: - test_args[:3] = [test_args[1], test_args[2], "--"] - elif test_args[0] in ["pytest"]: - test_args[:1] = ["-m", test_args[0], "--"] - else: - msg = ( - f"Test command '{self.build_options.test_command}' is not supported on " - f"Android. Supported commands are 'python -m', 'python -c' and 'pytest'." - ) - raise errors.FatalError(msg) + built_wheels = list(built_wheel_dir.glob("*.whl")) + if len(built_wheels) != 1: + msg = f"{built_wheel_dir} contains {len(built_wheels)} wheels; expected 1" + raise errors.FatalError(msg) + built_wheel = built_wheels[0] + + if built_wheel.name.endswith("none-any.whl"): + raise errors.NonPlatformWheelError() + return built_wheel + + +def test_wheel( + config: PythonConfiguration, + build_options: BuildOptions, + build_path: Path, + python_dir: Path, + env: dict[str, str], + android_env: dict[str, str], + wheel: Path, +) -> None: + if not (build_options.test_command and build_options.test_selector(config.identifier)): + return - # Run the test app. - call( - self.python_dir / "android.py", - "test", - "--managed", - "maxVersion", - "--site-packages", - site_packages_dir, - "--cwd", - cwd_dir, - *test_args, - env=self.env, + log.step("Testing wheel...") + native_arch = arch_synonym(platform.machine(), native_platform(), "android") + if config.arch != native_arch: + log.warning( + f"Skipping tests for {config.arch}, as the build machine only supports {native_arch}" ) + return + if build_options.before_test: + shell_prepared(build_options.before_test, build_options, env) -def build(options: Options, tmp_path: Path) -> None: - configs = get_python_configurations( - options.globals.build_selector, options.globals.architectures + # Install the wheel and test-requires. + platform_tag = ( + call( + "python", + "-c", + "import sysconfig; print(sysconfig.get_platform())", + env=android_env, + capture_stdout=True, + ) + .strip() + .replace("-", "_") ) - if not configs: - return - try: - before_all(options, configs) - built_wheels: list[Path] = [] - for config in configs: - Builder( - config, - options.build_options(config.identifier), - tmp_path / config.identifier, - built_wheels, - ).build() + site_packages_dir = build_path / "site-packages" + site_packages_dir.mkdir() + call( + "pip", + "install", + "--only-binary=:all:", + "--platform", + platform_tag, + # This index contains Android wheels for many of the most common packages + # which the wheel under test might depend on. Once enough of those packages + # have been released for Android on PyPI, this argument can be removed. + "--extra-index-url", + "https://chaquo.com/pypi-13.1/", + "--target", + site_packages_dir, + f"{wheel}{build_options.test_extras}", + *build_options.test_requires, + env=env, + ) - except subprocess.CalledProcessError as error: - msg = f"Command {error.cmd} failed with code {error.returncode}. {error.stdout or ''}" - raise errors.FatalError(msg) from error + # Copy test-sources. + cwd_dir = build_path / "cwd" + cwd_dir.mkdir() + if build_options.test_sources: + copy_test_sources(build_options.test_sources, build_options.package_dir, cwd_dir) + else: + (cwd_dir / "test_fail.py").write_text( + resources.TEST_FAIL_CWD_FILE.read_text(), + ) + + # Android doesn't support placeholders in the test command. + if any( + ("{" + placeholder + "}") in build_options.test_command + for placeholder in ["project", "package"] + ): + msg = ( + f"Test command '{build_options.test_command}' with a " + "'{project}' or '{package}' placeholder is not supported on Android, " + "because the source directory is not visible on the emulator." + ) + raise errors.FatalError(msg) + + # Parse test-command. + test_args = shlex.split(build_options.test_command) + if test_args[:2] in [["python", "-c"], ["python", "-m"]]: + test_args[:3] = [test_args[1], test_args[2], "--"] + elif test_args[0] in ["pytest"]: + test_args[:1] = ["-m", test_args[0], "--"] + else: + msg = ( + f"Test command '{build_options.test_command}' is not supported on " + f"Android. Supported commands are 'python -m', 'python -c' and 'pytest'." + ) + raise errors.FatalError(msg) + + # Run the test app. + call( + python_dir / "android.py", + "test", + "--managed", + "maxVersion", + "--site-packages", + site_packages_dir, + "--cwd", + cwd_dir, + *test_args, + env=env, + ) diff --git a/cibuildwheel/platforms/ios.py b/cibuildwheel/platforms/ios.py index 660af67c1..ab6735569 100644 --- a/cibuildwheel/platforms/ios.py +++ b/cibuildwheel/platforms/ios.py @@ -17,7 +17,6 @@ from ..architecture import Architecture from ..environment import ParsedEnvironment from ..frontend import ( - BuildFrontendConfig, BuildFrontendName, get_build_frontend_extra_flags, ) @@ -437,7 +436,7 @@ def build(options: Options, tmp_path: Path) -> None: for config in python_configurations: build_options = options.build_options(config.identifier) - build_frontend = build_options.build_frontend or BuildFrontendConfig("build") + build_frontend = build_options.build_frontend # uv doesn't support iOS if build_frontend.name == "build[uv]": msg = "uv doesn't support iOS" diff --git a/cibuildwheel/platforms/linux.py b/cibuildwheel/platforms/linux.py index 00cd85b2f..7914a31ef 100644 --- a/cibuildwheel/platforms/linux.py +++ b/cibuildwheel/platforms/linux.py @@ -10,7 +10,7 @@ from .. import errors from ..architecture import Architecture -from ..frontend import BuildFrontendConfig, get_build_frontend_extra_flags +from ..frontend import get_build_frontend_extra_flags from ..logger import log from ..oci_container import OCIContainer, OCIContainerEngineConfig, OCIPlatform from ..options import BuildOptions, Options @@ -204,7 +204,7 @@ def build_in_container( log.build_start(config.identifier) local_identifier_tmp_dir = local_tmp_dir / config.identifier build_options = options.build_options(config.identifier) - build_frontend = build_options.build_frontend or BuildFrontendConfig("build") + build_frontend = build_options.build_frontend use_uv = build_frontend.name == "build[uv]" pip = ["uv", "pip"] if use_uv else ["pip"] diff --git a/cibuildwheel/platforms/macos.py b/cibuildwheel/platforms/macos.py index 3b4534d2a..1e7aeb6b7 100644 --- a/cibuildwheel/platforms/macos.py +++ b/cibuildwheel/platforms/macos.py @@ -19,7 +19,7 @@ from ..architecture import Architecture from ..ci import detect_ci_provider from ..environment import ParsedEnvironment -from ..frontend import BuildFrontendConfig, BuildFrontendName, get_build_frontend_extra_flags +from ..frontend import BuildFrontendName, get_build_frontend_extra_flags from ..logger import log from ..options import Options from ..selector import BuildSelector @@ -407,7 +407,7 @@ def build(options: Options, tmp_path: Path) -> None: for config in python_configurations: build_options = options.build_options(config.identifier) - build_frontend = build_options.build_frontend or BuildFrontendConfig("build") + build_frontend = build_options.build_frontend use_uv = build_frontend.name == "build[uv]" uv_path = find_uv() if use_uv and uv_path is None: diff --git a/cibuildwheel/platforms/pyodide.py b/cibuildwheel/platforms/pyodide.py index 920920923..b53ef2280 100644 --- a/cibuildwheel/platforms/pyodide.py +++ b/cibuildwheel/platforms/pyodide.py @@ -16,7 +16,7 @@ from .. import errors from ..architecture import Architecture from ..environment import ParsedEnvironment -from ..frontend import BuildFrontendConfig, get_build_frontend_extra_flags +from ..frontend import get_build_frontend_extra_flags from ..logger import log from ..options import Options from ..selector import BuildSelector @@ -352,7 +352,7 @@ def build(options: Options, tmp_path: Path) -> None: for config in python_configurations: build_options = options.build_options(config.identifier) - build_frontend = build_options.build_frontend or BuildFrontendConfig("build") + build_frontend = build_options.build_frontend if build_frontend.name == "pip": msg = "The pyodide platform doesn't support pip frontend" diff --git a/cibuildwheel/platforms/windows.py b/cibuildwheel/platforms/windows.py index 521695b60..69e3efb43 100644 --- a/cibuildwheel/platforms/windows.py +++ b/cibuildwheel/platforms/windows.py @@ -15,7 +15,7 @@ from .. import errors from ..architecture import Architecture from ..environment import ParsedEnvironment -from ..frontend import BuildFrontendConfig, BuildFrontendName, get_build_frontend_extra_flags +from ..frontend import BuildFrontendName, get_build_frontend_extra_flags from ..logger import log from ..options import Options from ..selector import BuildSelector @@ -394,7 +394,7 @@ def build(options: Options, tmp_path: Path) -> None: for config in python_configurations: build_options = options.build_options(config.identifier) - build_frontend = build_options.build_frontend or BuildFrontendConfig("build") + build_frontend = build_options.build_frontend use_uv = build_frontend.name == "build[uv]" and can_use_uv(config) log.build_start(config.identifier) diff --git a/cibuildwheel/resources/_cross_venv.pth b/cibuildwheel/resources/_cross_venv.pth new file mode 100644 index 000000000..c8a04bc64 --- /dev/null +++ b/cibuildwheel/resources/_cross_venv.pth @@ -0,0 +1 @@ +import _cross_venv; _cross_venv.initialize() diff --git a/cibuildwheel/resources/_cross_venv.py b/cibuildwheel/resources/_cross_venv.py index 83e4779a3..162245538 100644 --- a/cibuildwheel/resources/_cross_venv.py +++ b/cibuildwheel/resources/_cross_venv.py @@ -11,6 +11,9 @@ def initialize() -> None: + if os.environ.get("CIBW_CROSS_VENV", "0") != "1": + return + # os ###################################################################### def cross_os_uname() -> os.uname_result: return os.uname_result( diff --git a/cibuildwheel/resources/cibuildwheel.schema.json b/cibuildwheel/resources/cibuildwheel.schema.json index fd9fcfe1c..73f719950 100644 --- a/cibuildwheel/resources/cibuildwheel.schema.json +++ b/cibuildwheel/resources/cibuildwheel.schema.json @@ -106,7 +106,7 @@ }, "build-frontend": { "default": "default", - "description": "Set the tool to use to build, either \"pip\" (default for now), \"build\", or \"build[uv]\"", + "description": "Set the tool to use to build, either \"build\" (default), \"build[uv]\", or \"pip\"", "oneOf": [ { "enum": [ diff --git a/unit_test/main_tests/main_options_test.py b/unit_test/main_tests/main_options_test.py index 731d10adc..53fbe8c30 100644 --- a/unit_test/main_tests/main_options_test.py +++ b/unit_test/main_tests/main_options_test.py @@ -493,7 +493,8 @@ def test_defaults(platform, intercepted_build_args): if isinstance(repair_wheel_default, list): repair_wheel_default = " && ".join(repair_wheel_default) assert build_options.repair_command == repair_wheel_default - assert build_options.build_frontend is None + assert build_options.build_frontend.name == "build" + assert build_options.build_frontend.args == () if platform == "linux": assert build_options.manylinux_images diff --git a/unit_test/options_test.py b/unit_test/options_test.py index 9e5a1f089..cc7768b8e 100644 --- a/unit_test/options_test.py +++ b/unit_test/options_test.py @@ -362,11 +362,11 @@ def test_build_frontend_option( parsed_build_frontend = options.build_options(identifier=None).build_frontend if toml_assignment: - assert parsed_build_frontend is not None assert parsed_build_frontend.name == result_name assert parsed_build_frontend.args == result_args else: - assert parsed_build_frontend is None + assert parsed_build_frontend.name == "build" + assert parsed_build_frontend.args == () def test_override_inherit_environment(tmp_path: Path) -> None: From f1d86d0083950b84960f94543d83758322334c4f Mon Sep 17 00:00:00 2001 From: Malcolm Smith Date: Fri, 30 May 2025 14:42:51 +0100 Subject: [PATCH 19/57] Rename "env" to "build_env" --- cibuildwheel/platforms/android.py | 58 +++++++++++++++++++------------ 1 file changed, 36 insertions(+), 22 deletions(-) diff --git a/cibuildwheel/platforms/android.py b/cibuildwheel/platforms/android.py index 825812167..0a1f60879 100644 --- a/cibuildwheel/platforms/android.py +++ b/cibuildwheel/platforms/android.py @@ -114,12 +114,18 @@ def build(options: Options, tmp_path: Path) -> None: ) repaired_wheel = compatible_wheel else: - env, android_env = setup_env(config, build_options, build_path, python_dir) - before_build(build_options, env) + build_env, android_env = setup_env(config, build_options, build_path, python_dir) + before_build(build_options, build_env) repaired_wheel = build_wheel(build_options, build_path, android_env) test_wheel( - config, build_options, build_path, python_dir, env, android_env, repaired_wheel + config, + build_options, + build_path, + python_dir, + build_env, + android_env, + repaired_wheel, ) if compatible_wheel is None: @@ -151,6 +157,12 @@ def setup_target_python(config: PythonConfiguration, build_path: Path) -> Path: def setup_env( config: PythonConfiguration, build_options: BuildOptions, build_path: Path, python_dir: Path ) -> tuple[dict[str, str], dict[str, str]]: + """ + Returns two environment dicts, both pointing at the same virtual environment: + + * build_env, which uses the environment normally. + * android_env, which uses the environment while simulating running on Android. + """ log.step("Setting up build environment...") python_exe_name = f"python{config.version}" @@ -164,7 +176,7 @@ def setup_env( dependency_constraint = build_options.dependency_constraints.get_for_python_version( version=config.version, tmp_dir=build_path ) - env = virtualenv( + build_env = virtualenv( config.version, Path(python_exe), venv_dir, @@ -173,10 +185,10 @@ def setup_env( ) # Apply custom environment variables, and check environment is still valid - env = build_options.environment.as_dictionary(env) - env["PIP_DISABLE_PIP_VERSION_CHECK"] = "1" + build_env = build_options.environment.as_dictionary(build_env) + build_env["PIP_DISABLE_PIP_VERSION_CHECK"] = "1" for command in ["python", "pip"]: - which = call("which", command, env=env, capture_stdout=True).strip() + which = call("which", command, env=build_env, capture_stdout=True).strip() if which != f"{venv_dir}/bin/{command}": msg = ( f"{command} available on PATH doesn't match our installed instance. If you " @@ -184,17 +196,17 @@ def setup_env( f"or insert {command} above it." ) raise errors.FatalError(msg) - call(command, "--version", env=env) + call(command, "--version", env=build_env) - # Construct an alternative environment which simulates running on Android. - android_env = setup_android_env(python_dir, venv_dir, env) + # Construct an altered environment which simulates running on Android. + android_env = setup_android_env(python_dir, venv_dir, build_env) # Install build tools build_frontend = build_options.build_frontend if build_frontend.name != "build": msg = "Android requires the build frontend to be 'build'" raise errors.FatalError(msg) - call("pip", "install", "build", *constraint_flags(dependency_constraint), env=env) + call("pip", "install", "build", *constraint_flags(dependency_constraint), env=build_env) # Install build-time requirements. These must be installed for the build platform, # but queried while simulating Android. The `build` CLI doesn't support this @@ -211,15 +223,15 @@ def make_extra_environ(self) -> dict[str, str]: pb = ProjectBuilder.from_isolated_env(AndroidEnv(), build_options.package_dir) if pb.build_system_requires: - call("pip", "install", *pb.build_system_requires, env=env) + call("pip", "install", *pb.build_system_requires, env=build_env) requires_for_build = pb.get_requires_for_build( "wheel", parse_config_settings(build_options.config_settings) ) if requires_for_build: - call("pip", "install", *requires_for_build, env=env) + call("pip", "install", *requires_for_build, env=build_env) - return env, android_env + return build_env, android_env def localize_sysconfigdata(sysconfigdata_path: Path, python_dir: Path) -> dict[str, Any]: @@ -271,7 +283,9 @@ def localized_vars(orig_vars: dict[str, Any], prefix: Path) -> dict[str, Any]: return localized_vars -def setup_android_env(python_dir: Path, venv_dir: Path, env: dict[str, str]) -> dict[str, str]: +def setup_android_env( + python_dir: Path, venv_dir: Path, build_env: dict[str, str] +) -> dict[str, str]: site_packages = next(venv_dir.glob("lib/python*/site-packages")) for suffix in ["pth", "py"]: shutil.copy(resources.PATH / f"_cross_venv.{suffix}", site_packages) @@ -284,10 +298,10 @@ def setup_android_env(python_dir: Path, venv_dir: Path, env: dict[str, str]) -> ) sysconfigdata = localize_sysconfigdata(sysconfigdata_path, python_dir) - android_env = env.copy() + android_env = build_env.copy() android_env["CIBW_CROSS_VENV"] = "1" # Activates the code in _cross_venv.py. - env_output = call(python_dir / "android.py", "env", env=env, capture_stdout=True) + env_output = call(python_dir / "android.py", "env", env=build_env, capture_stdout=True) for line in env_output.splitlines(): key, value = line.removeprefix("export ").split("=", 1) value_split = shlex.split(value) @@ -307,7 +321,7 @@ def setup_android_env(python_dir: Path, venv_dir: Path, env: dict[str, str]) -> # Format the environment so it can be pasted into a shell. for key, value in sorted(android_env.items()): - if env.get(key) != value: + if build_env.get(key) != value: print(f"export {key}={shlex.quote(value)}") return android_env @@ -355,7 +369,7 @@ def test_wheel( build_options: BuildOptions, build_path: Path, python_dir: Path, - env: dict[str, str], + build_env: dict[str, str], android_env: dict[str, str], wheel: Path, ) -> None: @@ -371,7 +385,7 @@ def test_wheel( return if build_options.before_test: - shell_prepared(build_options.before_test, build_options, env) + shell_prepared(build_options.before_test, build_options, build_env) # Install the wheel and test-requires. platform_tag = ( @@ -403,7 +417,7 @@ def test_wheel( site_packages_dir, f"{wheel}{build_options.test_extras}", *build_options.test_requires, - env=env, + env=build_env, ) # Copy test-sources. @@ -452,5 +466,5 @@ def test_wheel( "--cwd", cwd_dir, *test_args, - env=env, + env=build_env, ) From 6ab5c3fce569d1a73a87e955cdd49086ab7d2eba Mon Sep 17 00:00:00 2001 From: Malcolm Smith Date: Fri, 30 May 2025 17:20:36 +0100 Subject: [PATCH 20/57] Remove Chaquopy repository from default pip command line --- cibuildwheel/platforms/android.py | 5 ----- docs/platforms.md | 13 ++++++++++--- test/test_android.py | 1 + 3 files changed, 11 insertions(+), 8 deletions(-) diff --git a/cibuildwheel/platforms/android.py b/cibuildwheel/platforms/android.py index 0a1f60879..b6c90863f 100644 --- a/cibuildwheel/platforms/android.py +++ b/cibuildwheel/platforms/android.py @@ -408,11 +408,6 @@ def test_wheel( "--only-binary=:all:", "--platform", platform_tag, - # This index contains Android wheels for many of the most common packages - # which the wheel under test might depend on. Once enough of those packages - # have been released for Android on PyPI, this argument can be removed. - "--extra-index-url", - "https://chaquo.com/pypi-13.1/", "--target", site_packages_dir, f"{wheel}{build_options.test_extras}", diff --git a/docs/platforms.md b/docs/platforms.md index 0b67967b7..779c7d6c2 100644 --- a/docs/platforms.md +++ b/docs/platforms.md @@ -236,13 +236,20 @@ Cross-architecture testing is not supported. On Linux, the emulator needs access to the KVM virtualization interface, and a DISPLAY environment variable pointing at an X server. Xvfb is acceptable. +The test process uses the same testbed used by CPython itself to run the CPython test +suite. It is a Gradle project that has been configured to have a single JUnit test, +the result of which reports the success or failure of running the test command. + The Android test environment can't support running shell scripts, so the [`test-command`](options.md#test-command) must be a Python command – see its documentation for details. -The test process uses the same testbed used by CPython itself to run the CPython test -suite. It is a Gradle project that has been configured to have a single JUnit test, -the result of which reports the success or failure of running the test command. +If your package has dependencies which haven't been released on PyPI yet, you may want +to use the [`environment`](options.md#environment) option to set `PIP_EXTRA_INDEX_URL` +to one of the following URLs: + +* https://chaquo.com/pypi-13.1 +* https://pypi.anaconda.org/scientific-python-nightly-wheels/simple ## iOS {: #ios} diff --git a/test/test_android.py b/test/test_android.py index 6d531a9a6..942446fea 100644 --- a/test/test_android.py +++ b/test/test_android.py @@ -232,6 +232,7 @@ def test_api_level(tmp_path, capfd): # Verify that Android dependencies can be installed from the Chaquopy repository, and # that wheels tagged with an older version of Android (in this case 24) are still # accepted. + "CIBW_ENVIRONMENT": "PIP_EXTRA_INDEX_URL=https://chaquo.com/pypi-13.1", "CIBW_TEST_REQUIRES": "bitarray==3.0.0", "CIBW_TEST_COMMAND": ( "python -c 'from bitarray import bitarray; print(~bitarray(\"01100\"))'" From 2bbab5770f613289bcf934080da3084f157eac3f Mon Sep 17 00:00:00 2001 From: Malcolm Smith Date: Fri, 30 May 2025 21:23:38 +0100 Subject: [PATCH 21/57] Move native_platform to platforms module --- cibuildwheel/__main__.py | 4 ++-- cibuildwheel/architecture.py | 16 ---------------- cibuildwheel/platforms/__init__.py | 18 ++++++++++++++++++ cibuildwheel/platforms/android.py | 6 +++--- 4 files changed, 23 insertions(+), 21 deletions(-) diff --git a/cibuildwheel/__main__.py b/cibuildwheel/__main__.py index c04c5c492..7927ee7f1 100644 --- a/cibuildwheel/__main__.py +++ b/cibuildwheel/__main__.py @@ -18,11 +18,11 @@ import cibuildwheel import cibuildwheel.util from cibuildwheel import errors -from cibuildwheel.architecture import Architecture, allowed_architectures_check, native_platform +from cibuildwheel.architecture import Architecture, allowed_architectures_check from cibuildwheel.ci import CIProvider, detect_ci_provider, fix_ansi_codes_for_github_actions from cibuildwheel.logger import log from cibuildwheel.options import CommandLineArguments, Options, compute_options -from cibuildwheel.platforms import ALL_PLATFORM_MODULES, get_build_identifiers +from cibuildwheel.platforms import ALL_PLATFORM_MODULES, get_build_identifiers, native_platform from cibuildwheel.selector import BuildSelector, EnableGroup, selector_matches from cibuildwheel.typing import PLATFORMS, PlatformName from cibuildwheel.util.file import CIBW_CACHE_PATH diff --git a/cibuildwheel/architecture.py b/cibuildwheel/architecture.py index b7bf169e2..ac364a1d0 100644 --- a/cibuildwheel/architecture.py +++ b/cibuildwheel/architecture.py @@ -36,22 +36,6 @@ def arch_synonym(arch: str, from_platform: PlatformName, to_platform: PlatformNa return arch -def native_platform() -> PlatformName: - if sys.platform.startswith("linux"): - return "linux" - elif sys.platform == "darwin": - return "macos" - elif sys.platform == "win32": - return "windows" - else: - msg = ( - 'Unable to detect platform from "sys.platform". cibuildwheel doesn\'t ' - "support building wheels for this platform. You might be able to build for a different " - "platform using the --platform argument. Check --help output for more information." - ) - raise errors.ConfigurationError(msg) - - def _check_aarch32_el0() -> bool: """Check if running armv7l natively on aarch64 is supported""" if not sys.platform.startswith("linux"): diff --git a/cibuildwheel/platforms/__init__.py b/cibuildwheel/platforms/__init__.py index ef78f1407..63c4c37d2 100644 --- a/cibuildwheel/platforms/__init__.py +++ b/cibuildwheel/platforms/__init__.py @@ -1,9 +1,11 @@ from __future__ import annotations +import sys from collections.abc import Sequence from pathlib import Path from typing import Final, Protocol +from cibuildwheel import errors from cibuildwheel.architecture import Architecture from cibuildwheel.options import Options from cibuildwheel.platforms import android, ios, linux, macos, pyodide, windows @@ -33,6 +35,22 @@ def build(self, options: Options, tmp_path: Path) -> None: ... } +def native_platform() -> PlatformName: + if sys.platform.startswith("linux"): + return "linux" + elif sys.platform == "darwin": + return "macos" + elif sys.platform == "win32": + return "windows" + else: + msg = ( + 'Unable to detect platform from "sys.platform". cibuildwheel doesn\'t ' + "support building wheels for this platform. You might be able to build for a different " + "platform using the --platform argument. Check --help output for more information." + ) + raise errors.ConfigurationError(msg) + + def get_build_identifiers( platform_module: PlatformModule, build_selector: BuildSelector, diff --git a/cibuildwheel/platforms/android.py b/cibuildwheel/platforms/android.py index b6c90863f..26cf14063 100644 --- a/cibuildwheel/platforms/android.py +++ b/cibuildwheel/platforms/android.py @@ -14,8 +14,8 @@ from build.env import IsolatedEnv from filelock import FileLock -from .. import errors -from ..architecture import Architecture, arch_synonym, native_platform +from .. import errors, platforms +from ..architecture import Architecture, arch_synonym from ..frontend import get_build_frontend_extra_flags, parse_config_settings from ..logger import log from ..options import BuildOptions, Options @@ -377,7 +377,7 @@ def test_wheel( return log.step("Testing wheel...") - native_arch = arch_synonym(platform.machine(), native_platform(), "android") + native_arch = arch_synonym(platform.machine(), platforms.native_platform(), "android") if config.arch != native_arch: log.warning( f"Skipping tests for {config.arch}, as the build machine only supports {native_arch}" From b57b9e60f430177d877648130b1ee00f4f913116 Mon Sep 17 00:00:00 2001 From: Malcolm Smith Date: Fri, 30 May 2025 21:34:57 +0100 Subject: [PATCH 22/57] Fix parse_config_settings Co-authored-by: Joe Rickerby --- cibuildwheel/frontend.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cibuildwheel/frontend.py b/cibuildwheel/frontend.py index 22300a7e9..7973354fe 100644 --- a/cibuildwheel/frontend.py +++ b/cibuildwheel/frontend.py @@ -67,7 +67,7 @@ def parse_config_settings(config_settings_str: str) -> dict[str, str | list[str] if existing_value is None: config_settings[setting] = value elif isinstance(existing_value, str): - config_settings[setting] = [existing_value] + config_settings[setting] = [existing_value, value] else: existing_value.append(value) From 95a631a51824e71b71a3c9824c781601914422d2 Mon Sep 17 00:00:00 2001 From: Malcolm Smith Date: Fri, 30 May 2025 22:00:39 +0100 Subject: [PATCH 23/57] Add unit tests for parse_config_settings and arch_synonym --- unit_test/architecture_test.py | 19 ++++++++++++++++++- unit_test/main_tests/main_options_test.py | 14 ++++++++++++-- 2 files changed, 30 insertions(+), 3 deletions(-) diff --git a/unit_test/architecture_test.py b/unit_test/architecture_test.py index 934b2655d..bbe94c609 100644 --- a/unit_test/architecture_test.py +++ b/unit_test/architecture_test.py @@ -5,7 +5,7 @@ import pytest import cibuildwheel.architecture -from cibuildwheel.architecture import Architecture +from cibuildwheel.architecture import Architecture, arch_synonym @pytest.fixture( @@ -96,3 +96,20 @@ def test_arch_auto_no_aarch32(monkeypatch): arch_set = Architecture.parse_config("auto32", "linux") assert len(arch_set) == 0 + + +@pytest.mark.parametrize( + ("arch", "from_platform", "to_platform", "expected"), + [ + ("x86_64", "linux", "macos", "x86_64"), + ("x86_64", "macos", "linux", "x86_64"), + ("x86_64", "linux", "windows", "AMD64"), + ("AMD64", "windows", "linux", "x86_64"), + ("x86_64", "linux", "nonexistent", "x86_64"), + ("x86_64", "nonexistent", "linux", "x86_64"), + ("nonexistent", "linux", "windows", "nonexistent"), + ("x86", "windows", "macos", None), + ], +) +def test_arch_synonym(arch, from_platform, to_platform, expected): + assert arch_synonym(arch, from_platform, to_platform) == expected diff --git a/unit_test/main_tests/main_options_test.py b/unit_test/main_tests/main_options_test.py index 53fbe8c30..cccc2b975 100644 --- a/unit_test/main_tests/main_options_test.py +++ b/unit_test/main_tests/main_options_test.py @@ -7,7 +7,7 @@ from cibuildwheel.__main__ import main from cibuildwheel.environment import ParsedEnvironment -from cibuildwheel.frontend import _split_config_settings +from cibuildwheel.frontend import _split_config_settings, parse_config_settings from cibuildwheel.options import BuildOptions, _get_pinned_container_images from cibuildwheel.selector import BuildSelector, EnableGroup from cibuildwheel.util import resources @@ -288,7 +288,9 @@ def test_build_verbosity( @pytest.mark.parametrize("platform_specific", [False, True]) def test_config_settings(platform_specific, platform, intercepted_build_args, monkeypatch): - config_settings = 'setting=value setting=value2 other="something else"' + config_settings = ( + 'setting=value setting=value2 triplet=1 triplet=2 triplet=3 other="something else"' + ) if platform_specific: monkeypatch.setenv("CIBW_CONFIG_SETTINGS_" + platform.upper(), config_settings) monkeypatch.setenv("CIBW_CONFIG_SETTINGS", "a=b") @@ -303,8 +305,16 @@ def test_config_settings(platform_specific, platform, intercepted_build_args, mo assert _split_config_settings(config_settings) == [ "-Csetting=value", "-Csetting=value2", + "-Ctriplet=1", + "-Ctriplet=2", + "-Ctriplet=3", "-Cother=something else", ] + assert parse_config_settings(config_settings) == { + "setting": ["value", "value2"], + "triplet": ["1", "2", "3"], + "other": "something else", + } @pytest.mark.parametrize( From 70ba326d5a598d279f52a9799e19f88ca6b8f100 Mon Sep 17 00:00:00 2001 From: Malcolm Smith Date: Sat, 31 May 2025 14:33:02 +0100 Subject: [PATCH 24/57] Make `shell_prepared` arguments keyword-only, and add tests for the commands that use it --- cibuildwheel/platforms/android.py | 10 ++++----- test/test_android.py | 35 ++++++++++++++++++++++++------- 2 files changed, 33 insertions(+), 12 deletions(-) diff --git a/cibuildwheel/platforms/android.py b/cibuildwheel/platforms/android.py index 26cf14063..ddef20508 100644 --- a/cibuildwheel/platforms/android.py +++ b/cibuildwheel/platforms/android.py @@ -69,7 +69,7 @@ def get_python_configurations( ] -def shell_prepared(command: str, build_options: BuildOptions, env: dict[str, str]) -> None: +def shell_prepared(command: str, *, build_options: BuildOptions, env: dict[str, str]) -> None: shell( prepare_command(command, project=".", package=build_options.package_dir), env=env, @@ -82,8 +82,8 @@ def before_all(options: Options, python_configurations: list[PythonConfiguration log.step("Running before_all...") shell_prepared( before_all_options.before_all, - before_all_options, - before_all_options.environment.as_dictionary(os.environ), + build_options=before_all_options, + env=before_all_options.environment.as_dictionary(os.environ), ) @@ -330,7 +330,7 @@ def setup_android_env( def before_build(build_options: BuildOptions, env: dict[str, str]) -> None: if build_options.before_build: log.step("Running before_build...") - shell_prepared(build_options.before_build, build_options, env) + shell_prepared(build_options.before_build, build_options=build_options, env=env) def build_wheel(build_options: BuildOptions, build_path: Path, android_env: dict[str, str]) -> Path: @@ -385,7 +385,7 @@ def test_wheel( return if build_options.before_test: - shell_prepared(build_options.before_test, build_options, build_env) + shell_prepared(build_options.before_test, build_options=build_options, env=build_env) # Install the wheel and test-requires. platform_tag = ( diff --git a/test/test_android.py b/test/test_android.py index 942446fea..fa3424f43 100644 --- a/test/test_android.py +++ b/test/test_android.py @@ -1,5 +1,6 @@ import os import platform +import re from dataclasses import dataclass from subprocess import CalledProcessError from textwrap import dedent @@ -78,32 +79,52 @@ def test_expected_wheels(tmp_path): @pytest.mark.serial def test_archs(tmp_path, capfd): new_c_project().generate(tmp_path) + + # Build all architectures while checking the handling of the `before` commands. + command_pattern = 'echo "Hello from {0}, package={{package}}, python=$(which python)"' + output_pattern = ( + f"Hello from {{0}}, package={tmp_path}, python=/.+/cp313-android_{{1}}/venv/bin/python" + ) + wheels = cibuildwheel_run( tmp_path, add_env={ **cp313_env, "CIBW_ARCHS": "all", + "CIBW_BEFORE_ALL": "echo 'Hello from before_all'", + "CIBW_BEFORE_BUILD": command_pattern.format("before_build"), + "CIBW_BEFORE_TEST": command_pattern.format("before_test"), "CIBW_TEST_COMMAND": ( - 'python -c \'import platform; print("machine" + "=" + platform.machine())\'' + "python -c 'import platform; print(f\"Hello from {platform.machine()}\")'" ), }, ) assert wheels == [f"spam-0.1.0-cp313-cp313-android_21_{arch.android_abi}.whl" for arch in archs] - # The native architecture should run tests. stdout, stderr = capfd.readouterr() - machine_lines = [line for line in stdout.splitlines() if "machine=" in line] - assert len(machine_lines) == 1 - assert machine_lines[0] == f"machine={native_arch.linux_machine}" + lines = (line for line in stdout.splitlines() if line.startswith("Hello from")) + assert next(lines) == "Hello from before_all" - # The non-native architectures should give a warning that they can't run tests. + # All architectures should be built, but only the native architecture should run tests. for arch in archs: - if arch != native_arch: + abi = arch.android_abi + assert re.fullmatch(output_pattern.format("before_build", abi), next(lines)) + if arch == native_arch: + assert re.fullmatch(output_pattern.format("before_test", abi), next(lines)) + assert next(lines) == f"Hello from {arch.linux_machine}" + else: assert ( f"warning: Skipping tests for {arch.android_abi}, as the build machine " f"only supports {native_arch.android_abi}" ) in stderr + try: + line = next(lines) + except StopIteration: + pass + else: + pytest.fail(f"Unexpected line: {line!r}") + def test_build_requires(tmp_path, capfd): # Build-time requirements should be installed for the build platform, not for Android. Prove From b191c0a97bd929c3d5b68b1e3ebe1d4a4c4da261 Mon Sep 17 00:00:00 2001 From: Malcolm Smith Date: Sat, 31 May 2025 14:44:32 +0100 Subject: [PATCH 25/57] Replace `importlib.util.spec_from_file_location` with `runpy.run_path` --- cibuildwheel/platforms/android.py | 11 +++-------- 1 file changed, 3 insertions(+), 8 deletions(-) diff --git a/cibuildwheel/platforms/android.py b/cibuildwheel/platforms/android.py index ddef20508..668761893 100644 --- a/cibuildwheel/platforms/android.py +++ b/cibuildwheel/platforms/android.py @@ -1,4 +1,3 @@ -import importlib.util import os import platform import re @@ -8,6 +7,7 @@ from dataclasses import dataclass from pathlib import Path from pprint import pprint +from runpy import run_path from typing import Any from build import ProjectBuilder @@ -235,16 +235,11 @@ def make_extra_environ(self) -> dict[str, str]: def localize_sysconfigdata(sysconfigdata_path: Path, python_dir: Path) -> dict[str, Any]: - spec = importlib.util.spec_from_file_location(sysconfigdata_path.stem, sysconfigdata_path) - assert spec is not None - module = importlib.util.module_from_spec(spec) - assert spec.loader is not None - spec.loader.exec_module(module) - + sysconfigdata: dict[str, Any] = run_path(str(sysconfigdata_path))["build_time_vars"] with sysconfigdata_path.open("w") as f: f.write("# Generated by cibuildwheel\n") f.write("build_time_vars = ") - sysconfigdata = localized_vars(module.build_time_vars, python_dir / "prefix") + sysconfigdata = localized_vars(sysconfigdata, python_dir / "prefix") pprint(sysconfigdata, stream=f, compact=True) return sysconfigdata From 7c0a95a1ea1beaafaaaca1c633c6ed3163497602 Mon Sep 17 00:00:00 2001 From: Malcolm Smith Date: Sat, 31 May 2025 18:55:37 +0100 Subject: [PATCH 26/57] Use python-build-standalone --- cibuildwheel/platforms/android.py | 16 +++++----------- docs/platforms.md | 1 - 2 files changed, 5 insertions(+), 12 deletions(-) diff --git a/cibuildwheel/platforms/android.py b/cibuildwheel/platforms/android.py index 668761893..4ab4b8a81 100644 --- a/cibuildwheel/platforms/android.py +++ b/cibuildwheel/platforms/android.py @@ -25,6 +25,7 @@ from ..util.file import CIBW_CACHE_PATH, copy_test_sources, download, move_file from ..util.helpers import prepare_command from ..util.packaging import find_compatible_wheel +from ..util.python_build_standalone import create_python_build_standalone_environment from ..venv import constraint_flags, virtualenv @@ -165,23 +166,16 @@ def setup_env( """ log.step("Setting up build environment...") - python_exe_name = f"python{config.version}" - python_exe = shutil.which(python_exe_name) - if not python_exe: - msg = f"Couldn't find {python_exe_name} on the PATH" - raise errors.FatalError(msg) - # Create virtual environment + python_exe = create_python_build_standalone_environment( + config.version, build_path, CIBW_CACHE_PATH + ) venv_dir = build_path / "venv" dependency_constraint = build_options.dependency_constraints.get_for_python_version( version=config.version, tmp_dir=build_path ) build_env = virtualenv( - config.version, - Path(python_exe), - venv_dir, - dependency_constraint, - use_uv=False, + config.version, python_exe, venv_dir, dependency_constraint, use_uv=False ) # Apply custom environment variables, and check environment is still valid diff --git a/docs/platforms.md b/docs/platforms.md index 779c7d6c2..c66ea78d5 100644 --- a/docs/platforms.md +++ b/docs/platforms.md @@ -200,7 +200,6 @@ needs. It also requires the following commands to be on the `PATH`: -* `pythonX.Y`, where `X.Y` is the version of Python you're building for. * `curl` * `java` (or set the `JAVA_HOME` environment variable) From 5ebd0159ef67d1e26b429c9d2f752db8c0fc971a Mon Sep 17 00:00:00 2001 From: Malcolm Smith Date: Sun, 1 Jun 2025 00:02:40 +0100 Subject: [PATCH 27/57] Update Android Python --- cibuildwheel/resources/build-platforms.toml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cibuildwheel/resources/build-platforms.toml b/cibuildwheel/resources/build-platforms.toml index f83cd6e5f..df4b62fa4 100644 --- a/cibuildwheel/resources/build-platforms.toml +++ b/cibuildwheel/resources/build-platforms.toml @@ -225,8 +225,8 @@ python_configurations = [ [android] python_configurations = [ - { identifier = "cp313-android_arm64_v8a", version = "3.13", url = "https://chaquo.com/python-3.13.3+20250503.174536-aarch64-linux-android.tar.gz" }, - { identifier = "cp313-android_x86_64", version = "3.13", url = "https://chaquo.com/python-3.13.3+20250503.185233-x86_64-linux-android.tar.gz" }, + { identifier = "cp313-android_arm64_v8a", version = "3.13", url = "https://repo.maven.apache.org/maven2/com/chaquo/python/python/3.13.3/python-3.13.3-aarch64-linux-android.tar.gz" }, + { identifier = "cp313-android_x86_64", version = "3.13", url = "https://repo.maven.apache.org/maven2/com/chaquo/python/python/3.13.3/python-3.13.3-x86_64-linux-android.tar.gz" }, ] [ios] From 4e9b82b7ff337474a6655049f85ee5c2b020e119 Mon Sep 17 00:00:00 2001 From: Malcolm Smith Date: Mon, 2 Jun 2025 22:35:25 +0100 Subject: [PATCH 28/57] Enable KVM in Linux CI --- bin/run_tests.py | 11 +++++++++++ cibuildwheel/resources/build-platforms.toml | 4 ++-- docs/platforms.md | 5 +++-- test/test_android.py | 10 +++++++--- 4 files changed, 23 insertions(+), 7 deletions(-) diff --git a/bin/run_tests.py b/bin/run_tests.py index 1c0923948..80d02eeee 100755 --- a/bin/run_tests.py +++ b/bin/run_tests.py @@ -42,6 +42,17 @@ if args.run_podman: unit_test_args += ["--run-podman"] + if "CI" in os.environ: + # Enable hardware virtualization for the Android emulator + # (https://stackoverflow.com/a/61984745). + for command in [ + 'echo \'KERNEL=="kvm", GROUP="kvm", MODE="0666", OPTIONS+="static_node=kvm"\' ' + "| sudo tee /etc/udev/rules.d/99-kvm4all.rules", + "sudo udevadm control --reload-rules", + "sudo udevadm trigger --name-match=kvm", + ]: + subprocess.run(command, shell=True, check=True) + print( "\n\n================================== UNIT TESTS ==================================", flush=True, diff --git a/cibuildwheel/resources/build-platforms.toml b/cibuildwheel/resources/build-platforms.toml index df4b62fa4..11ec5b52c 100644 --- a/cibuildwheel/resources/build-platforms.toml +++ b/cibuildwheel/resources/build-platforms.toml @@ -225,8 +225,8 @@ python_configurations = [ [android] python_configurations = [ - { identifier = "cp313-android_arm64_v8a", version = "3.13", url = "https://repo.maven.apache.org/maven2/com/chaquo/python/python/3.13.3/python-3.13.3-aarch64-linux-android.tar.gz" }, - { identifier = "cp313-android_x86_64", version = "3.13", url = "https://repo.maven.apache.org/maven2/com/chaquo/python/python/3.13.3/python-3.13.3-x86_64-linux-android.tar.gz" }, + { identifier = "cp313-android_arm64_v8a", version = "3.13", url = "https://chaquo.com/python-3.13.3+20250602.200342-aarch64-linux-android.tar.gz" }, + { identifier = "cp313-android_x86_64", version = "3.13", url = "https://chaquo.com/python-3.13.3+20250602.201111-x86_64-linux-android.tar.gz" }, ] [ios] diff --git a/docs/platforms.md b/docs/platforms.md index c66ea78d5..56a06e285 100644 --- a/docs/platforms.md +++ b/docs/platforms.md @@ -232,8 +232,9 @@ emulator matching the architecture of the build machine – for example, if you' building on an ARM64 machine, an ARM64 wheel can be tested on an ARM64 emulator. Cross-architecture testing is not supported. -On Linux, the emulator needs access to the KVM virtualization interface, and a DISPLAY -environment variable pointing at an X server. Xvfb is acceptable. +On Linux, the emulator needs access to the KVM virtualization interface. If the +emulator fails to start, try running `$ANDROID_HOME/emulator/emulator -accel-check` +for advice. The test process uses the same testbed used by CPython itself to run the CPython test suite. It is a Gradle project that has been configured to have a single JUnit test, diff --git a/test/test_android.py b/test/test_android.py index fa3424f43..6966ab4c3 100644 --- a/test/test_android.py +++ b/test/test_android.py @@ -10,10 +10,14 @@ from .test_projects import new_c_project from .utils import cibuildwheel_run, expected_wheels -system_machine = (platform.system(), platform.machine()) -if system_machine not in [("Linux", "x86_64"), ("Darwin", "arm64"), ("Darwin", "x86_64")]: +if (platform.system(), platform.machine()) not in [ + ("Linux", "x86_64"), + ("Darwin", "arm64"), + ("Darwin", "x86_64"), +]: pytest.skip( - f"Android development tools are not available for {system_machine}", + f"cibuildwheel does not support building Android wheels on " + f"{platform.system()} {platform.machine()}", allow_module_level=True, ) From e775acaaebe28231bcef137bfc7aab6641fe4a98 Mon Sep 17 00:00:00 2001 From: Malcolm Smith Date: Mon, 2 Jun 2025 23:07:56 +0100 Subject: [PATCH 29/57] Move KVM code to test_android.py --- bin/run_tests.py | 11 ----------- test/test_android.py | 25 +++++++++++++++++++++++-- 2 files changed, 23 insertions(+), 13 deletions(-) diff --git a/bin/run_tests.py b/bin/run_tests.py index 80d02eeee..1c0923948 100755 --- a/bin/run_tests.py +++ b/bin/run_tests.py @@ -42,17 +42,6 @@ if args.run_podman: unit_test_args += ["--run-podman"] - if "CI" in os.environ: - # Enable hardware virtualization for the Android emulator - # (https://stackoverflow.com/a/61984745). - for command in [ - 'echo \'KERNEL=="kvm", GROUP="kvm", MODE="0666", OPTIONS+="static_node=kvm"\' ' - "| sudo tee /etc/udev/rules.d/99-kvm4all.rules", - "sudo udevadm control --reload-rules", - "sudo udevadm trigger --name-match=kvm", - ]: - subprocess.run(command, shell=True, check=True) - print( "\n\n================================== UNIT TESTS ==================================", flush=True, diff --git a/test/test_android.py b/test/test_android.py index 6966ab4c3..c0ed7cc55 100644 --- a/test/test_android.py +++ b/test/test_android.py @@ -2,7 +2,7 @@ import platform import re from dataclasses import dataclass -from subprocess import CalledProcessError +from subprocess import CalledProcessError, run from textwrap import dedent import pytest @@ -10,6 +10,10 @@ from .test_projects import new_c_project from .utils import cibuildwheel_run, expected_wheels +CIBW_PLATFORM = os.environ.get("CIBW_PLATFORM", "android") +if CIBW_PLATFORM != "android": + pytest.skip(f"{CIBW_PLATFORM=}", allow_module_level=True) + if (platform.system(), platform.machine()) not in [ ("Linux", "x86_64"), ("Darwin", "arm64"), @@ -21,12 +25,29 @@ allow_module_level=True, ) -if "ANDROID_HOME" not in os.environ: +ANDROID_HOME = os.environ.get("ANDROID_HOME") +if ANDROID_HOME is None: pytest.skip( "ANDROID_HOME environment variable is not set", allow_module_level=True, ) +# Ensure hardware virtualization is enabled for the emulator +# (https://stackoverflow.com/a/61984745). +try: + run([f"{ANDROID_HOME}/emulator/emulator", "-accel-check"], check=True) +except CalledProcessError: + if "CI" not in os.environ: + raise + else: + for command in [ + 'echo \'KERNEL=="kvm", GROUP="kvm", MODE="0666", OPTIONS+="static_node=kvm"\' ' + "| sudo tee /etc/udev/rules.d/99-kvm4all.rules", + "sudo udevadm control --reload-rules", + "sudo udevadm trigger --name-match=kvm", + ]: + run(command, shell=True, check=True) + @dataclass class Architecture: From 4e94ceb3ca32cc63740c93d412b3c0ab322360c2 Mon Sep 17 00:00:00 2001 From: Malcolm Smith Date: Mon, 2 Jun 2025 23:17:19 +0100 Subject: [PATCH 30/57] Use Java 17 on Azure --- azure-pipelines.yml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index ddc9f18fa..7e06a4f54 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -12,6 +12,11 @@ jobs: - task: UsePythonVersion@0 inputs: versionSpec: '3.11' + - task: JavaToolInstaller@0 + inputs: + versionSpec: '17' + jdkArchitectureOption: 'x64' + jdkSourceOption: 'PreInstalled' - bash: | docker run --rm --privileged docker.io/tonistiigi/binfmt:latest --install all python -m pip install dependency-groups From 3dde388d2c82f9de53eb1242bbf86c714eb6f8aa Mon Sep 17 00:00:00 2001 From: Malcolm Smith Date: Mon, 2 Jun 2025 23:35:56 +0100 Subject: [PATCH 31/57] Install emulator if necessary before running -accel-check --- test/test_android.py | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/test/test_android.py b/test/test_android.py index c0ed7cc55..e53096507 100644 --- a/test/test_android.py +++ b/test/test_android.py @@ -1,7 +1,9 @@ import os import platform import re +import sys from dataclasses import dataclass +from pathlib import Path from subprocess import CalledProcessError, run from textwrap import dedent @@ -34,12 +36,13 @@ # Ensure hardware virtualization is enabled for the emulator # (https://stackoverflow.com/a/61984745). +emulator = Path(f"{ANDROID_HOME}/emulator/emulator") +if not emulator.exists(): + run([f"{ANDROID_HOME}/cmdline-tools/latest/bin/sdkmanager", "emulator"], check=True) try: - run([f"{ANDROID_HOME}/emulator/emulator", "-accel-check"], check=True) + run([emulator, "-accel-check"], check=True) except CalledProcessError: - if "CI" not in os.environ: - raise - else: + if "CI" in os.environ and sys.platform == "linux": for command in [ 'echo \'KERNEL=="kvm", GROUP="kvm", MODE="0666", OPTIONS+="static_node=kvm"\' ' "| sudo tee /etc/udev/rules.d/99-kvm4all.rules", @@ -47,6 +50,8 @@ "sudo udevadm trigger --name-match=kvm", ]: run(command, shell=True, check=True) + else: + raise @dataclass From 16d2a45d0a2038d7067d596a2bd5842d39db2afc Mon Sep 17 00:00:00 2001 From: Malcolm Smith Date: Tue, 3 Jun 2025 16:58:40 +0100 Subject: [PATCH 32/57] Free up additional disk space on Linux runners --- .github/workflows/test.yml | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 684c76cf4..a04cf4fe5 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -40,6 +40,7 @@ jobs: needs: lint runs-on: ${{ matrix.os }} strategy: + fail-fast: false # FIXME remove once working matrix: os: [ubuntu-latest, ubuntu-24.04-arm, windows-latest, windows-11-arm, macos-13, macos-15] python_version: ['3.13'] @@ -59,11 +60,13 @@ jobs: - uses: astral-sh/setup-uv@v6 - # free some space to prevent reaching GHA disk space limits - - name: Clean docker images + - name: Free up disk space if: runner.os == 'Linux' run: | docker system prune -a -f + rm -rf $ANDROID_HOME/ndk/26.* /opt/hostedtoolcache/CodeQL \ + /usr/local/lib/node_modules /usr/local/share/chromium \ + /usr/local/share/powershell df -h # for oci_container unit tests From f072818311024e8e816c658037cfc85271a8bb53 Mon Sep 17 00:00:00 2001 From: Malcolm Smith Date: Tue, 3 Jun 2025 17:22:46 +0100 Subject: [PATCH 33/57] Add sudo --- .github/workflows/test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index a04cf4fe5..1ec7983bb 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -64,7 +64,7 @@ jobs: if: runner.os == 'Linux' run: | docker system prune -a -f - rm -rf $ANDROID_HOME/ndk/26.* /opt/hostedtoolcache/CodeQL \ + sudo rm -rf $ANDROID_HOME/ndk/26.* /opt/hostedtoolcache/CodeQL \ /usr/local/lib/node_modules /usr/local/share/chromium \ /usr/local/share/powershell df -h From d6890a80655609a4843c31c080701714c4098b0a Mon Sep 17 00:00:00 2001 From: Malcolm Smith Date: Tue, 3 Jun 2025 23:21:45 +0100 Subject: [PATCH 34/57] Skip emulator tests on CI platforms that don't support it --- .github/workflows/test.yml | 8 +++ cibuildwheel/resources/build-platforms.toml | 4 +- test/test_android.py | 71 +++++++++++---------- 3 files changed, 47 insertions(+), 36 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 1ec7983bb..16510b24b 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -69,6 +69,14 @@ jobs: /usr/local/share/powershell df -h + # https://github.blog/changelog/2024-04-02-github-actions-hardware-accelerated-android-virtualization-now-available/ + - name: Enable KVM for Android emulator + if: runner.os == 'Linux' && runner.arch == 'X64' + run: | + echo 'KERNEL=="kvm", GROUP="kvm", MODE="0666", OPTIONS+="static_node=kvm"' | sudo tee /etc/udev/rules.d/99-kvm4all.rules + sudo udevadm control --reload-rules + sudo udevadm trigger --name-match=kvm + # for oci_container unit tests - name: Set up QEMU if: runner.os == 'Linux' diff --git a/cibuildwheel/resources/build-platforms.toml b/cibuildwheel/resources/build-platforms.toml index b53bdd072..cbc172c55 100644 --- a/cibuildwheel/resources/build-platforms.toml +++ b/cibuildwheel/resources/build-platforms.toml @@ -226,8 +226,8 @@ python_configurations = [ [android] python_configurations = [ - { identifier = "cp313-android_arm64_v8a", version = "3.13", url = "https://chaquo.com/python-3.13.3+20250602.200342-aarch64-linux-android.tar.gz" }, - { identifier = "cp313-android_x86_64", version = "3.13", url = "https://chaquo.com/python-3.13.3+20250602.201111-x86_64-linux-android.tar.gz" }, + { identifier = "cp313-android_arm64_v8a", version = "3.13", url = "https://chaquo.com/python-3.13.3+20250603.183345-aarch64-linux-android.tar.gz" }, + { identifier = "cp313-android_x86_64", version = "3.13", url = "https://chaquo.com/python-3.13.3+20250603.194341-x86_64-linux-android.tar.gz" }, ] [ios] diff --git a/test/test_android.py b/test/test_android.py index e53096507..a704c06b4 100644 --- a/test/test_android.py +++ b/test/test_android.py @@ -1,10 +1,8 @@ import os import platform import re -import sys from dataclasses import dataclass -from pathlib import Path -from subprocess import CalledProcessError, run +from subprocess import CalledProcessError from textwrap import dedent import pytest @@ -27,31 +25,38 @@ allow_module_level=True, ) -ANDROID_HOME = os.environ.get("ANDROID_HOME") -if ANDROID_HOME is None: - pytest.skip( - "ANDROID_HOME environment variable is not set", - allow_module_level=True, - ) +ci_supports_build = any( + key in os.environ + for key in [ + "GITHUB_ACTIONS", + "TF_BUILD", # Azure Pipelines + ] +) -# Ensure hardware virtualization is enabled for the emulator -# (https://stackoverflow.com/a/61984745). -emulator = Path(f"{ANDROID_HOME}/emulator/emulator") -if not emulator.exists(): - run([f"{ANDROID_HOME}/cmdline-tools/latest/bin/sdkmanager", "emulator"], check=True) -try: - run([emulator, "-accel-check"], check=True) -except CalledProcessError: - if "CI" in os.environ and sys.platform == "linux": - for command in [ - 'echo \'KERNEL=="kvm", GROUP="kvm", MODE="0666", OPTIONS+="static_node=kvm"\' ' - "| sudo tee /etc/udev/rules.d/99-kvm4all.rules", - "sudo udevadm control --reload-rules", - "sudo udevadm trigger --name-match=kvm", - ]: - run(command, shell=True, check=True) +if "ANDROID_HOME" not in os.environ: + msg = "ANDROID_HOME environment variable is not set" + if ci_supports_build: + pytest.fail(msg) else: - raise + pytest.skip(msg, allow_module_level=True) + +# Running an emulator requires the build machine to either be bare-metal or support +# nested virtualization. Many CI services don't support this. GitHub Actions only +# supports it on Linux +# (https://docs.github.com/en/actions/using-github-hosted-runners/using-github-hosted-runners/about-github-hosted-runners#supported-runners-and-hardware-resources), +# but this may extend to macOS once M3 runners are available +# (https://github.com/ReactiveCircus/android-emulator-runner/issues/350). +ci_supports_emulator = "GITHUB_ACTIONS" in os.environ and platform.system() == "Linux" + + +def needs_emulator(test): + # All copies of the testbed app run on the same emulator with the same + # application ID, so these tests must be run serially. + test = pytest.mark.serial(test) + + if ci_supports_build and not ci_supports_emulator: + test = pytest.mark.skip("This CI platform doesn't support the emulator")(test) + return test @dataclass @@ -104,9 +109,7 @@ def test_expected_wheels(tmp_path): ) -# Any tests which involve the testbed app must be run serially, because all copies of the testbed -# app run on the same emulator with the same application ID. -@pytest.mark.serial +@needs_emulator def test_archs(tmp_path, capfd): new_c_project().generate(tmp_path) @@ -144,7 +147,7 @@ def test_archs(tmp_path, capfd): assert next(lines) == f"Hello from {arch.linux_machine}" else: assert ( - f"warning: Skipping tests for {arch.android_abi}, as the build machine " + f"Skipping tests for {arch.android_abi}, as the build machine " f"only supports {native_arch.android_abi}" ) in stderr @@ -214,7 +217,7 @@ def test_spam(): } -@pytest.mark.serial +@needs_emulator @pytest.mark.parametrize( ("command", "expected_output"), [ @@ -228,7 +231,7 @@ def test_test_command_good(command, expected_output, tmp_path, spam_env, capfd): assert expected_output in capfd.readouterr().out -@pytest.mark.serial +@needs_emulator @pytest.mark.parametrize( ("command", "expected_output"), [ @@ -258,7 +261,7 @@ def test_test_command_bad(command, expected_output, tmp_path, spam_env, capfd): assert expected_output in capfd.readouterr().err -@pytest.mark.serial +@needs_emulator def test_no_test_sources(tmp_path, capfd): new_c_project().generate(tmp_path) with pytest.raises(CalledProcessError): @@ -272,7 +275,7 @@ def test_no_test_sources(tmp_path, capfd): ) in capfd.readouterr().err -@pytest.mark.serial +@needs_emulator def test_api_level(tmp_path, capfd): new_c_project().generate(tmp_path) wheels = cibuildwheel_run( From 62817d8852192e59efbe15dc412a5fb7e369177e Mon Sep 17 00:00:00 2001 From: Malcolm Smith Date: Wed, 4 Jun 2025 00:28:03 +0100 Subject: [PATCH 35/57] Download Android Python from Maven Central --- README.md | 2 +- cibuildwheel/resources/build-platforms.toml | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 90f8f4bc8..047255b5c 100644 --- a/README.md +++ b/README.md @@ -68,7 +68,7 @@ Usage ¹ [Requires emulation](https://cibuildwheel.pypa.io/en/stable/faq/#emulation), distributed separately. Other services may also support Linux ARM through emulation or third-party build hosts, but these are not tested in our CI.
² [Uses cross-compilation](https://cibuildwheel.pypa.io/en/stable/faq/#windows-arm64). It is not possible to test `arm64` on this CI platform.
³ Requires a macOS runner; runs tests on the simulator for the runner's architecture.
-⁴ Requires runner to be Linux x86_64, macOS ARM64 or macOS x86_64. Runs tests on the emulator for the runner's architecture.
+⁴ Building for Android requires the runner to be Linux x86_64, macOS ARM64 or macOS x86_64. Testing is supported on the same platforms, but also requires the runner to either be bare-metal, or support nested virtualization. CI platforms known to meet this requirement are: GitHub Actions Linux x86_64.
Example setup diff --git a/cibuildwheel/resources/build-platforms.toml b/cibuildwheel/resources/build-platforms.toml index cbc172c55..b46c4b479 100644 --- a/cibuildwheel/resources/build-platforms.toml +++ b/cibuildwheel/resources/build-platforms.toml @@ -226,8 +226,8 @@ python_configurations = [ [android] python_configurations = [ - { identifier = "cp313-android_arm64_v8a", version = "3.13", url = "https://chaquo.com/python-3.13.3+20250603.183345-aarch64-linux-android.tar.gz" }, - { identifier = "cp313-android_x86_64", version = "3.13", url = "https://chaquo.com/python-3.13.3+20250603.194341-x86_64-linux-android.tar.gz" }, + { identifier = "cp313-android_arm64_v8a", version = "3.13", url = "https://repo.maven.apache.org/maven2/com/chaquo/python/python/3.13.3.20250603.183345/python-3.13.3.20250603.183345-aarch64-linux-android.tar.gz" }, + { identifier = "cp313-android_x86_64", version = "3.13", url = "https://repo.maven.apache.org/maven2/com/chaquo/python/python/3.13.3.20250603.183345/python-3.13.3.20250603.183345-x86_64-linux-android.tar.gz" }, ] [ios] From 01e787fe1e72b0f6e28cca3f27534e2a60def4ff Mon Sep 17 00:00:00 2001 From: Malcolm Smith Date: Wed, 4 Jun 2025 00:28:43 +0100 Subject: [PATCH 36/57] Free up more disk space on Linux runners --- .github/workflows/test.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 16510b24b..1705246b8 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -40,7 +40,6 @@ jobs: needs: lint runs-on: ${{ matrix.os }} strategy: - fail-fast: false # FIXME remove once working matrix: os: [ubuntu-latest, ubuntu-24.04-arm, windows-latest, windows-11-arm, macos-13, macos-15] python_version: ['3.13'] @@ -64,7 +63,7 @@ jobs: if: runner.os == 'Linux' run: | docker system prune -a -f - sudo rm -rf $ANDROID_HOME/ndk/26.* /opt/hostedtoolcache/CodeQL \ + sudo rm -rf $ANDROID_HOME/ndk/{26,28}.* /opt/hostedtoolcache/CodeQL \ /usr/local/lib/node_modules /usr/local/share/chromium \ /usr/local/share/powershell df -h From 34623ff91ed2e0cfb9a0457ee8051698dedc143f Mon Sep 17 00:00:00 2001 From: Henry Schreiner Date: Tue, 3 Jun 2025 23:19:10 -0400 Subject: [PATCH 37/57] fix: minor fixups Signed-off-by: Henry Schreiner --- cibuildwheel/architecture.py | 6 +++--- cibuildwheel/platforms/android.py | 7 ++++--- docs/platforms.md | 2 +- test/utils.py | 2 +- 4 files changed, 9 insertions(+), 8 deletions(-) diff --git a/cibuildwheel/architecture.py b/cibuildwheel/architecture.py index ac364a1d0..6a13ce659 100644 --- a/cibuildwheel/architecture.py +++ b/cibuildwheel/architecture.py @@ -29,9 +29,9 @@ def arch_synonym(arch: str, from_platform: PlatformName, to_platform: PlatformName) -> str | None: - for arch_synonym in ARCH_SYNONYMS: - if arch == arch_synonym.get(from_platform): - return arch_synonym.get(to_platform, arch) + for arch_synonym_ in ARCH_SYNONYMS: + if arch == arch_synonym_.get(from_platform): + return arch_synonym_.get(to_platform, arch) return arch diff --git a/cibuildwheel/platforms/android.py b/cibuildwheel/platforms/android.py index 4ab4b8a81..0b1b5fdbe 100644 --- a/cibuildwheel/platforms/android.py +++ b/cibuildwheel/platforms/android.py @@ -240,7 +240,7 @@ def localize_sysconfigdata(sysconfigdata_path: Path, python_dir: Path) -> dict[s def localized_vars(orig_vars: dict[str, Any], prefix: Path) -> dict[str, Any]: orig_prefix = orig_vars["prefix"] - localized_vars = {} + localized_vars_ = {} for key, value in orig_vars.items(): # The host's sysconfigdata will include references to build-time paths. # Update these to refer to the current prefix. @@ -267,9 +267,9 @@ def localized_vars(orig_vars: dict[str, Any], prefix: Path) -> dict[str, Any]: elif key in ["LDSHARED", "LDCXXSHARED"]: final = final.removesuffix(" " + orig_vars["LDFLAGS"]) - localized_vars[key] = final + localized_vars_[key] = final - return localized_vars + return localized_vars_ def setup_android_env( @@ -353,6 +353,7 @@ def build_wheel(build_options: BuildOptions, build_path: Path, android_env: dict return built_wheel +# pylint: disable-next=too-many-positional-arguments def test_wheel( config: PythonConfiguration, build_options: BuildOptions, diff --git a/docs/platforms.md b/docs/platforms.md index e8378f514..e6c5852a2 100644 --- a/docs/platforms.md +++ b/docs/platforms.md @@ -179,7 +179,7 @@ If there are pre-releases available for a newer Python version, the `pyodide-pre Currently, it's recommended to run tests using a `python -m` entrypoint, rather than a command line entrypoint, or a shell script. This is because custom entrypoints have some issues in the Pyodide virtual environment. For example, `pytest` may not work as a command line entrypoint, but will work as a `python -m pytest` entrypoint. -## Android +## Android {: android} ### Prerequisites diff --git a/test/utils.py b/test/utils.py index e0d8962cb..03fa6c075 100644 --- a/test/utils.py +++ b/test/utils.py @@ -263,7 +263,7 @@ def _expected_wheels( python_abi_tags = ["cp312-cp312"] if EnableGroup.PyodidePrerelease in enable_groups: python_abi_tags.append("cp313-cp313") - elif platform in ["android", "ios"] and python_abi_tags is None: + elif platform in {"android", "ios"} and python_abi_tags is None: python_abi_tags = ["cp313-cp313"] elif python_abi_tags is None: python_abi_tags = [ From be841658149075b3bbf9a277c880643c129ca440 Mon Sep 17 00:00:00 2001 From: Malcolm Smith Date: Mon, 9 Jun 2025 18:46:46 +0100 Subject: [PATCH 38/57] Set sysconfig._BASE_PREFIX to support sysconfig.get_path("include") --- cibuildwheel/resources/_cross_venv.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/cibuildwheel/resources/_cross_venv.py b/cibuildwheel/resources/_cross_venv.py index 162245538..ab4a0f410 100644 --- a/cibuildwheel/resources/_cross_venv.py +++ b/cibuildwheel/resources/_cross_venv.py @@ -69,6 +69,11 @@ def cross_getandroidapilevel() -> int: # sysconfig ############################################################### # + # We don't change the actual sys.base_prefix and base_exec_prefix, because that + # could have unpredictable effects. Instead, we change the internal variables + # used to generate sysconfig.get_path("include"). + exec_prefix = sysconfig.get_config_var("exec_prefix") + sysconfig._BASE_PREFIX = sysconfig._BASE_EXEC_PREFIX = exec_prefix # type: ignore[attr-defined] sysconfig._init_config_vars() # type: ignore[attr-defined] # sysconfig.get_platform, which determines the wheel tag, is implemented in terms of From be2cf54078db2c6a23bb18c1277811da5cd2a6b0 Mon Sep 17 00:00:00 2001 From: Malcolm Smith Date: Mon, 9 Jun 2025 18:51:10 +0100 Subject: [PATCH 39/57] Get ANDROID_API_LEVEL from the build environment, not cibuildwheel's own environment --- cibuildwheel/platforms/android.py | 18 +++++++++++------- test/test_android.py | 16 +++++++++++++--- 2 files changed, 24 insertions(+), 10 deletions(-) diff --git a/cibuildwheel/platforms/android.py b/cibuildwheel/platforms/android.py index 4ab4b8a81..a4d2dfe5e 100644 --- a/cibuildwheel/platforms/android.py +++ b/cibuildwheel/platforms/android.py @@ -228,17 +228,21 @@ def make_extra_environ(self) -> dict[str, str]: return build_env, android_env -def localize_sysconfigdata(sysconfigdata_path: Path, python_dir: Path) -> dict[str, Any]: +def localize_sysconfigdata( + python_dir: Path, build_env: dict[str, str], sysconfigdata_path: Path +) -> dict[str, Any]: sysconfigdata: dict[str, Any] = run_path(str(sysconfigdata_path))["build_time_vars"] with sysconfigdata_path.open("w") as f: f.write("# Generated by cibuildwheel\n") f.write("build_time_vars = ") - sysconfigdata = localized_vars(sysconfigdata, python_dir / "prefix") + sysconfigdata = localized_vars(build_env, sysconfigdata, python_dir / "prefix") pprint(sysconfigdata, stream=f, compact=True) return sysconfigdata -def localized_vars(orig_vars: dict[str, Any], prefix: Path) -> dict[str, Any]: +def localized_vars( + build_env: dict[str, str], orig_vars: dict[str, Any], prefix: Path +) -> dict[str, Any]: orig_prefix = orig_vars["prefix"] localized_vars = {} for key, value in orig_vars.items(): @@ -249,7 +253,7 @@ def localized_vars(orig_vars: dict[str, Any], prefix: Path) -> dict[str, Any]: final = final.replace(orig_prefix, str(prefix)) if key == "ANDROID_API_LEVEL": - if api_level := os.environ.get(key): + if api_level := build_env.get(key): final = int(api_level) # Build systems vary in whether FLAGS variables are read from sysconfig, and if so, @@ -285,7 +289,7 @@ def setup_android_env( site_packages, ) ) - sysconfigdata = localize_sysconfigdata(sysconfigdata_path, python_dir) + sysconfigdata = localize_sysconfigdata(python_dir, build_env, sysconfigdata_path) android_env = build_env.copy() android_env["CIBW_CROSS_VENV"] = "1" # Activates the code in _cross_venv.py. @@ -308,9 +312,9 @@ def setup_android_env( for key in ["CFLAGS", "CXXFLAGS"]: android_env[key] += " " + opt - # Format the environment so it can be pasted into a shell. + # Format the environment so it can be pasted into a shell when debugging. for key, value in sorted(android_env.items()): - if build_env.get(key) != value: + if os.environ.get(key) != value: print(f"export {key}={shlex.quote(value)}") return android_env diff --git a/test/test_android.py b/test/test_android.py index a704c06b4..c41bd7638 100644 --- a/test/test_android.py +++ b/test/test_android.py @@ -277,16 +277,26 @@ def test_no_test_sources(tmp_path, capfd): @needs_emulator def test_api_level(tmp_path, capfd): - new_c_project().generate(tmp_path) + project = new_c_project() + project.files["pyproject.toml"] = dedent( + """\ + [build-system] + requires = ["setuptools"] + + [tool.cibuildwheel] + android.environment.ANDROID_API_LEVEL = "33" + android.environment.PIP_EXTRA_INDEX_URL = "https://chaquo.com/pypi-13.1" + """ + ) + project.generate(tmp_path) + wheels = cibuildwheel_run( tmp_path, add_env={ **cp313_env, - "ANDROID_API_LEVEL": "33", # Verify that Android dependencies can be installed from the Chaquopy repository, and # that wheels tagged with an older version of Android (in this case 24) are still # accepted. - "CIBW_ENVIRONMENT": "PIP_EXTRA_INDEX_URL=https://chaquo.com/pypi-13.1", "CIBW_TEST_REQUIRES": "bitarray==3.0.0", "CIBW_TEST_COMMAND": ( "python -c 'from bitarray import bitarray; print(~bitarray(\"01100\"))'" From 405fd06e280c379f8fd2ea7c504bf71e403236eb Mon Sep 17 00:00:00 2001 From: Malcolm Smith Date: Tue, 10 Jun 2025 17:23:17 +0100 Subject: [PATCH 40/57] Correct relative path of test-sources --- cibuildwheel/platforms/android.py | 2 +- test/test_android.py | 20 ++++++++++++++++++++ 2 files changed, 21 insertions(+), 1 deletion(-) diff --git a/cibuildwheel/platforms/android.py b/cibuildwheel/platforms/android.py index ea0d8d1a4..dbf284e13 100644 --- a/cibuildwheel/platforms/android.py +++ b/cibuildwheel/platforms/android.py @@ -413,7 +413,7 @@ def test_wheel( cwd_dir = build_path / "cwd" cwd_dir.mkdir() if build_options.test_sources: - copy_test_sources(build_options.test_sources, build_options.package_dir, cwd_dir) + copy_test_sources(build_options.test_sources, Path.cwd(), cwd_dir) else: (cwd_dir / "test_fail.py").write_text( resources.TEST_FAIL_CWD_FILE.read_text(), diff --git a/test/test_android.py b/test/test_android.py index c41bd7638..1d24bef2e 100644 --- a/test/test_android.py +++ b/test/test_android.py @@ -261,6 +261,26 @@ def test_test_command_bad(command, expected_output, tmp_path, spam_env, capfd): assert expected_output in capfd.readouterr().err +def test_package_subdir(tmp_path, spam_env, capfd): + spam_paths = list(tmp_path.iterdir()) + package_dir = tmp_path / "package" + package_dir.mkdir() + for path in spam_paths: + path.rename(package_dir / path.name) + + test_filename = "package/" + spam_env["CIBW_TEST_SOURCES"] + cibuildwheel_run( + tmp_path, + package_dir, + add_env={ + **spam_env, + "CIBW_TEST_SOURCES": test_filename, + "CIBW_TEST_COMMAND": f"python -m pytest {test_filename}", + }, + ) + assert "=== 1 passed in " in capfd.readouterr().out + + @needs_emulator def test_no_test_sources(tmp_path, capfd): new_c_project().generate(tmp_path) From db9796b72037ade2ae76b7a0a97fa0a61d47c8b3 Mon Sep 17 00:00:00 2001 From: Malcolm Smith Date: Wed, 11 Jun 2025 10:55:59 +0100 Subject: [PATCH 41/57] Pass a CMake toolchain file to the build --- cibuildwheel/platforms/android.py | 34 +++++++++++++++++++++++++++++++ examples/github-deploy.yml | 8 ++++++++ 2 files changed, 42 insertions(+) diff --git a/cibuildwheel/platforms/android.py b/cibuildwheel/platforms/android.py index dbf284e13..7c5dfe94a 100644 --- a/cibuildwheel/platforms/android.py +++ b/cibuildwheel/platforms/android.py @@ -8,6 +8,7 @@ from pathlib import Path from pprint import pprint from runpy import run_path +from textwrap import dedent from typing import Any from build import ProjectBuilder @@ -177,6 +178,7 @@ def setup_env( build_env = virtualenv( config.version, python_exe, venv_dir, dependency_constraint, use_uv=False ) + create_cmake_toolchain(config, build_path, python_dir, build_env) # Apply custom environment variables, and check environment is still valid build_env = build_options.environment.as_dictionary(build_env) @@ -228,6 +230,38 @@ def make_extra_environ(self) -> dict[str, str]: return build_env, android_env +def create_cmake_toolchain( + config: PythonConfiguration, build_path: Path, python_dir: Path, build_env: dict[str, str] +) -> None: + toolchain_path = build_path / "toolchain.cmake" + build_env["CMAKE_TOOLCHAIN_FILE"] = str(toolchain_path) + with open(toolchain_path, "w") as toolchain_file: + prefix = f"{python_dir}/prefix" + print( + dedent( + f"""\ + # To support as many build systems as possible, we use environment + # variables as the single source of truth for compiler flags and paths, + # so they don't need to be specified here. + + set(CMAKE_SYSTEM_NAME Android) + set(CMAKE_SYSTEM_PROCESSOR {config.arch}) + + # Inhibit all of CMake's own NDK handling code. + set(CMAKE_SYSTEM_VERSION 1) + + # Tell CMake where to look for headers and libraries. + list(INSERT CMAKE_FIND_ROOT_PATH 0 {prefix}) + set(CMAKE_FIND_ROOT_PATH_MODE_PROGRAM NEVER) + set(CMAKE_FIND_ROOT_PATH_MODE_LIBRARY ONLY) + set(CMAKE_FIND_ROOT_PATH_MODE_INCLUDE ONLY) + set(CMAKE_FIND_ROOT_PATH_MODE_PACKAGE ONLY) + """ + ), + file=toolchain_file, + ) + + def localize_sysconfigdata( python_dir: Path, build_env: dict[str, str], sysconfigdata_path: Path ) -> dict[str, Any]: diff --git a/examples/github-deploy.yml b/examples/github-deploy.yml index 9f27541fc..4a1a5d912 100644 --- a/examples/github-deploy.yml +++ b/examples/github-deploy.yml @@ -50,6 +50,14 @@ jobs: steps: - uses: actions/checkout@v4 + # https://github.blog/changelog/2024-04-02-github-actions-hardware-accelerated-android-virtualization-now-available/ + - name: Enable KVM for Android emulator + if: matrix.os == 'android-intel' + run: | + echo 'KERNEL=="kvm", GROUP="kvm", MODE="0666", OPTIONS+="static_node=kvm"' | sudo tee /etc/udev/rules.d/99-kvm4all.rules + sudo udevadm control --reload-rules + sudo udevadm trigger --name-match=kvm + - name: Build wheels uses: pypa/cibuildwheel@v3.0.0b5 env: From c63753c82f473c3c8617b85ab2fc4c7137775bc5 Mon Sep 17 00:00:00 2001 From: Malcolm Smith Date: Wed, 11 Jun 2025 23:11:15 +0100 Subject: [PATCH 42/57] Add "repair" step which adds libc++ to the wheel when necessary --- .pre-commit-config.yaml | 1 + cibuildwheel/platforms/android.py | 160 +++++++++++++++++++++++++++--- docs/options.md | 4 +- docs/platforms.md | 1 + pyproject.toml | 4 +- test/test_android.py | 41 ++++++++ test/test_projects/c.py | 2 +- 7 files changed, 196 insertions(+), 17 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 31632eebb..e83207412 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -34,6 +34,7 @@ repos: - nox>=2025.2.9 - orjson - packaging + - pyelftools - pygithub - pytest - rich diff --git a/cibuildwheel/platforms/android.py b/cibuildwheel/platforms/android.py index 7c5dfe94a..72cf92e2b 100644 --- a/cibuildwheel/platforms/android.py +++ b/cibuildwheel/platforms/android.py @@ -1,9 +1,12 @@ +import csv +import hashlib import os import platform import re import shlex import shutil import subprocess +from collections.abc import Iterable, Iterator from dataclasses import dataclass from pathlib import Path from pprint import pprint @@ -13,6 +16,8 @@ from build import ProjectBuilder from build.env import IsolatedEnv +from elftools.common.exceptions import ELFError +from elftools.elf.elffile import ELFFile from filelock import FileLock from .. import errors, platforms @@ -118,7 +123,10 @@ def build(options: Options, tmp_path: Path) -> None: else: build_env, android_env = setup_env(config, build_options, build_path, python_dir) before_build(build_options, build_env) - repaired_wheel = build_wheel(build_options, build_path, android_env) + built_wheel = build_wheel(build_options, build_path, android_env) + repaired_wheel = repair_wheel( + build_options, build_path, build_env, android_env, built_wheel + ) test_wheel( config, @@ -391,6 +399,132 @@ def build_wheel(build_options: BuildOptions, build_path: Path, android_env: dict return built_wheel +def repair_wheel( + build_options: BuildOptions, + build_path: Path, + build_env: dict[str, str], + android_env: dict[str, str], + built_wheel: Path, +) -> Path: + log.step("Repairing wheel...") + repaired_wheel_dir = build_path / "repaired_wheel" + repaired_wheel_dir.mkdir() + + if build_options.repair_command: + shell( + prepare_command( + build_options.repair_command, wheel=built_wheel, dest_dir=repaired_wheel_dir + ), + env=build_env, + ) + else: + repair_default(android_env, built_wheel, repaired_wheel_dir) + + repaired_wheels = list(repaired_wheel_dir.glob("*.whl")) + if len(repaired_wheels) == 0: + raise errors.RepairStepProducedNoWheelError() + if len(repaired_wheels) != 1: + msg = f"{repaired_wheel_dir} contains {len(repaired_wheels)} wheels; expected 1" + raise errors.FatalError(msg) + repaired_wheel = repaired_wheels[0] + + if repaired_wheel.name.endswith("none-any.whl"): + raise errors.NonPlatformWheelError() + return repaired_wheel + + +def repair_default( + android_env: dict[str, str], built_wheel: Path, repaired_wheel_dir: Path +) -> None: + """ + Adds libc++ to the wheel if anything links against it. In the future this should be + moved to auditwheel and generalized to support more libraries. + """ + if (match := re.search(r"^(.+?)-", built_wheel.name)) is None: + msg = f"Failed to parse wheel filename: {built_wheel.name}" + raise errors.FatalError(msg) + wheel_name = match[1] + + unpacked_dir = repaired_wheel_dir / "unpacked" + unpacked_dir.mkdir() + shutil.unpack_archive(built_wheel, unpacked_dir, format="zip") + + # Some build systems are inconsistent about name normalization, so don't assume the + # dist-info name is identical to the wheel name. + record_paths = list(unpacked_dir.glob("*.dist-info/RECORD")) + if len(record_paths) != 1: + msg = f"{built_wheel.name} contains {len(record_paths)} dist-info/RECORD files; expected 1" + raise errors.FatalError(msg) + + old_soname = "libc++_shared.so" + paths_to_patch = [] + for path, elffile in elf_file_filter( + unpacked_dir / filename + for filename, *_ in csv.reader(record_paths[0].read_text().splitlines()) + ): + if (dynamic := elffile.get_section_by_name(".dynamic")) and any( # type: ignore[no-untyped-call] + tag.entry.d_tag == "DT_NEEDED" and tag.needed == old_soname + for tag in dynamic.iter_tags() + ): + paths_to_patch.append(path) + + if not paths_to_patch: + shutil.copyfile(built_wheel, repaired_wheel_dir / built_wheel.name) + else: + # Android doesn't support DT_RPATH, but supports DT_RUNPATH since API level 24 + # (https://github.com/aosp-mirror/platform_bionic/blob/master/android-changes-for-ndk-developers.md). + if int(sysconfig_print('get_config_var("ANDROID_API_LEVEL")', android_env)) < 24: + msg = f"Adding {old_soname} requires ANDROID_API_LEVEL to be at least 24" + raise errors.FatalError(msg) + + toolchain = Path(android_env["CC"]).parent.parent + src_path = toolchain / f"sysroot/usr/lib/{android_env['HOST']}/{old_soname}" + + # Use the same library location as auditwheel would. + libs_dir = unpacked_dir / (wheel_name + ".libs") + libs_dir.mkdir() + new_soname = soname_with_hash(src_path) + dst_path = libs_dir / new_soname + shutil.copyfile(src_path, dst_path) + call("patchelf", "--set-soname", new_soname, dst_path) + + for path in paths_to_patch: + call("patchelf", "--replace-needed", old_soname, new_soname, path) + call( + "patchelf", + "--set-rpath", + f"${{ORIGIN}}/{libs_dir.relative_to(path.parent)}", + path, + ) + call("wheel", "pack", unpacked_dir, "-d", repaired_wheel_dir) + + +def elf_file_filter(paths: Iterable[Path]) -> Iterator[tuple[Path, ELFFile]]: + """Filter through an iterator of filenames and load up only ELF files""" + for path in paths: + if path.name.endswith(".py"): + continue + else: + try: + with open(path, "rb") as f: + candidate = ELFFile(f) # type: ignore[no-untyped-call] + yield path, candidate + except ELFError: + # not an elf file + continue + + +def soname_with_hash(src_path: Path) -> str: + """Return the same library filename as auditwheel would""" + shorthash = hashlib.sha256(src_path.read_bytes()).hexdigest()[:8] + src_name = src_path.name + base, ext = src_name.split(".", 1) + if not base.endswith(f"-{shorthash}"): + return f"{base}-{shorthash}.{ext}" + else: + return src_name + + # pylint: disable-next=too-many-positional-arguments def test_wheel( config: PythonConfiguration, @@ -416,18 +550,6 @@ def test_wheel( shell_prepared(build_options.before_test, build_options=build_options, env=build_env) # Install the wheel and test-requires. - platform_tag = ( - call( - "python", - "-c", - "import sysconfig; print(sysconfig.get_platform())", - env=android_env, - capture_stdout=True, - ) - .strip() - .replace("-", "_") - ) - site_packages_dir = build_path / "site-packages" site_packages_dir.mkdir() call( @@ -435,7 +557,7 @@ def test_wheel( "install", "--only-binary=:all:", "--platform", - platform_tag, + sysconfig_print("get_platform()", android_env).replace("-", "_"), "--target", site_packages_dir, f"{wheel}{build_options.test_extras}", @@ -491,3 +613,13 @@ def test_wheel( *test_args, env=build_env, ) + + +def sysconfig_print(method_call: str, env: dict[str, str]) -> str: + return call( + "python", + "-c", + f'import sysconfig; print(sysconfig.{method_call}, end="")', + env=env, + capture_stdout=True, + ) diff --git a/docs/options.md b/docs/options.md index c76f76487..e3c0fe256 100644 --- a/docs/options.md +++ b/docs/options.md @@ -890,6 +890,8 @@ Default: - on Linux: `'auditwheel repair -w {dest_dir} {wheel}'` - on macOS: `'delocate-wheel --require-archs {delocate_archs} -w {dest_dir} -v {wheel}'` +- on Android: There is no default command, but cibuildwheel will add `libc++` to the + wheel if anything links against it. Setting a command will replace this behavior. - on other platforms: `''` A shell command to repair a built wheel by copying external library dependencies into the wheel tree and relinking them. @@ -1639,7 +1641,7 @@ A space-separated list of environment variables to set in the test environment. The syntax is the same as for [`environment`](#environment). Platform-specific environment variables are also available:
-`CIBW_TEST_ENVIRONMENT_MACOS` | `CIBW_TEST_ENVIRONMENT_WINDOWS` | `CIBW_TEST_ENVIRONMENT_LINUX` | `CIBW_TEST_ENVIRONMENT_IOS` | `CIBW_TEST_ENVIRONMENT_PYODIDE` +`CIBW_TEST_ENVIRONMENT_MACOS` | `CIBW_TEST_ENVIRONMENT_WINDOWS` | `CIBW_TEST_ENVIRONMENT_LINUX` | `CIBW_TEST_ENVIRONMENT_ANDROID` |`CIBW_TEST_ENVIRONMENT_IOS` | `CIBW_TEST_ENVIRONMENT_PYODIDE` #### Examples diff --git a/docs/platforms.md b/docs/platforms.md index e6c5852a2..157070f44 100644 --- a/docs/platforms.md +++ b/docs/platforms.md @@ -204,6 +204,7 @@ It also requires the following commands to be on the `PATH`: * `curl` * `java` (or set the `JAVA_HOME` environment variable) +* `patchelf` (if the wheel links against any external libraries) ### Android version compatibility diff --git a/pyproject.toml b/pyproject.toml index 11980ef28..782ff6efb 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -44,7 +44,9 @@ dependencies = [ "dependency-groups>=1.2", "filelock", "packaging>=20.9", - "platformdirs" + "pyelftools>=0.29", + "platformdirs", + "wheel>=0.33.6", ] [project.optional-dependencies] diff --git a/test/test_android.py b/test/test_android.py index 1d24bef2e..35d060776 100644 --- a/test/test_android.py +++ b/test/test_android.py @@ -2,8 +2,10 @@ import platform import re from dataclasses import dataclass +from shutil import rmtree from subprocess import CalledProcessError from textwrap import dedent +from zipfile import ZipFile import pytest @@ -325,3 +327,42 @@ def test_api_level(tmp_path, capfd): ) assert wheels == [f"spam-0.1.0-cp313-cp313-android_33_{native_arch.android_abi}.whl"] assert "bitarray('10011')" in capfd.readouterr().out + + +@needs_emulator +def test_libcxx(tmp_path, capfd): + project_dir = tmp_path / "project" + output_dir = tmp_path / "output" + + # A C++ package should include libc++, and the extension module should be able to + # find it using DT_RUNPATH. + new_c_project(setup_py_extension_args_add="language='c++'").generate(project_dir) + script = 'import spam; print(", ".join(f"{s}: {spam.filter(s)}" for s in ["ham", "spam"]))' + cp313_test_env = {**cp313_env, "CIBW_TEST_COMMAND": f"python -c '{script}'"} + + # Including external libraries requires API level 24. + with pytest.raises(CalledProcessError): + cibuildwheel_run(project_dir, add_env=cp313_test_env, output_dir=output_dir) + assert "libc++_shared.so requires ANDROID_API_LEVEL to be at least 24" in capfd.readouterr().err + + wheels = cibuildwheel_run( + project_dir, + add_env={**cp313_test_env, "ANDROID_API_LEVEL": "24"}, + output_dir=output_dir, + ) + assert len(wheels) == 1 + names = ZipFile(output_dir / wheels[0]).namelist() + libcxx_names = [ + name for name in names if re.fullmatch(r"spam\.libs/libc\+\+_shared-[0-9a-f]{8}\.so", name) + ] + assert len(libcxx_names) == 1 + assert "ham: 1, spam: 0" in capfd.readouterr().out + + # A C package should not include libc++. + rmtree(project_dir) + rmtree(output_dir) + new_c_project().generate(project_dir) + wheels = cibuildwheel_run(project_dir, add_env=cp313_env, output_dir=output_dir) + assert len(wheels) == 1 + for name in ZipFile(output_dir / wheels[0]).namelist(): + assert ".libs" not in name diff --git a/test/test_projects/c.py b/test/test_projects/c.py index 55d759875..9643e4cda 100644 --- a/test/test_projects/c.py +++ b/test/test_projects/c.py @@ -17,7 +17,7 @@ return NULL; // Spam should not be allowed through the filter. - sts = strcmp(content, "spam"); + sts = strcmp(content, "spam") != 0; {{ spam_c_function_add | indent(4) }} From 60eb070d2ed7249c7537797ea40121800d9061b9 Mon Sep 17 00:00:00 2001 From: Malcolm Smith Date: Thu, 12 Jun 2025 00:13:11 +0100 Subject: [PATCH 43/57] Add missing needs_emulator decorator --- test/test_android.py | 1 + 1 file changed, 1 insertion(+) diff --git a/test/test_android.py b/test/test_android.py index 35d060776..36d570f37 100644 --- a/test/test_android.py +++ b/test/test_android.py @@ -263,6 +263,7 @@ def test_test_command_bad(command, expected_output, tmp_path, spam_env, capfd): assert expected_output in capfd.readouterr().err +@needs_emulator def test_package_subdir(tmp_path, spam_env, capfd): spam_paths = list(tmp_path.iterdir()) package_dir = tmp_path / "package" From a16d47af29ea1897ac8b61e814b06c3a19bd5a1a Mon Sep 17 00:00:00 2001 From: Malcolm Smith Date: Mon, 16 Jun 2025 14:48:26 +0100 Subject: [PATCH 44/57] Provide useful error message if ANDROID_HOME is not set --- cibuildwheel/platforms/android.py | 7 +++++++ test/test_android.py | 10 ++++++++++ 2 files changed, 17 insertions(+) diff --git a/cibuildwheel/platforms/android.py b/cibuildwheel/platforms/android.py index 72cf92e2b..af2af7599 100644 --- a/cibuildwheel/platforms/android.py +++ b/cibuildwheel/platforms/android.py @@ -95,6 +95,13 @@ def before_all(options: Options, python_configurations: list[PythonConfiguration def build(options: Options, tmp_path: Path) -> None: + if "ANDROID_HOME" not in os.environ: + msg = ( + "ANDROID_HOME environment variable is not set. For instructions, see " + "https://cibuildwheel.pypa.io/en/stable/platforms/#android" + ) + raise errors.FatalError(msg) + configs = get_python_configurations( options.globals.build_selector, options.globals.architectures ) diff --git a/test/test_android.py b/test/test_android.py index 36d570f37..005f22c0e 100644 --- a/test/test_android.py +++ b/test/test_android.py @@ -83,6 +83,16 @@ class Architecture: } +def test_android_home(tmp_path, capfd): + new_c_project().generate(tmp_path) + env = os.environ.copy() + del env["ANDROID_HOME"] + + with pytest.raises(CalledProcessError): + cibuildwheel_run(tmp_path, env={**env, **cp313_env}) + assert "ANDROID_HOME environment variable is not set" in capfd.readouterr().err + + def test_frontend_good(tmp_path): new_c_project().generate(tmp_path) wheels = cibuildwheel_run( From 0ba32814c286d31aa709ce7c60f3720c7eabe1a2 Mon Sep 17 00:00:00 2001 From: Malcolm Smith Date: Mon, 16 Jun 2025 16:37:02 +0100 Subject: [PATCH 45/57] Remove use of HOST environment variable --- cibuildwheel/platforms/android.py | 13 +++++++------ cibuildwheel/resources/_cross_venv.py | 12 +++++++----- 2 files changed, 14 insertions(+), 11 deletions(-) diff --git a/cibuildwheel/platforms/android.py b/cibuildwheel/platforms/android.py index af2af7599..f53010add 100644 --- a/cibuildwheel/platforms/android.py +++ b/cibuildwheel/platforms/android.py @@ -210,7 +210,7 @@ def setup_env( call(command, "--version", env=build_env) # Construct an altered environment which simulates running on Android. - android_env = setup_android_env(python_dir, venv_dir, build_env) + android_env = setup_android_env(config, python_dir, venv_dir, build_env) # Install build tools build_frontend = build_options.build_frontend @@ -260,7 +260,7 @@ def create_cmake_toolchain( # so they don't need to be specified here. set(CMAKE_SYSTEM_NAME Android) - set(CMAKE_SYSTEM_PROCESSOR {config.arch}) + set(CMAKE_SYSTEM_PROCESSOR {android_triplet(config.identifier).split("-")[0]}) # Inhibit all of CMake's own NDK handling code. set(CMAKE_SYSTEM_VERSION 1) @@ -326,7 +326,7 @@ def localized_vars( def setup_android_env( - python_dir: Path, venv_dir: Path, build_env: dict[str, str] + config: PythonConfiguration, python_dir: Path, venv_dir: Path, build_env: dict[str, str] ) -> dict[str, str]: site_packages = next(venv_dir.glob("lib/python*/site-packages")) for suffix in ["pth", "py"]: @@ -340,8 +340,9 @@ def setup_android_env( ) sysconfigdata = localize_sysconfigdata(python_dir, build_env, sysconfigdata_path) + # Activate the code in _cross_venv.py. android_env = build_env.copy() - android_env["CIBW_CROSS_VENV"] = "1" # Activates the code in _cross_venv.py. + android_env["CIBW_HOST_TRIPLET"] = android_triplet(config.identifier) env_output = call(python_dir / "android.py", "env", env=build_env, capture_stdout=True) for line in env_output.splitlines(): @@ -480,12 +481,12 @@ def repair_default( else: # Android doesn't support DT_RPATH, but supports DT_RUNPATH since API level 24 # (https://github.com/aosp-mirror/platform_bionic/blob/master/android-changes-for-ndk-developers.md). - if int(sysconfig_print('get_config_var("ANDROID_API_LEVEL")', android_env)) < 24: + if int(sysconfig_print('get_config_vars()["ANDROID_API_LEVEL"]', android_env)) < 24: msg = f"Adding {old_soname} requires ANDROID_API_LEVEL to be at least 24" raise errors.FatalError(msg) toolchain = Path(android_env["CC"]).parent.parent - src_path = toolchain / f"sysroot/usr/lib/{android_env['HOST']}/{old_soname}" + src_path = toolchain / f"sysroot/usr/lib/{android_env['CIBW_HOST_TRIPLET']}/{old_soname}" # Use the same library location as auditwheel would. libs_dir = unpacked_dir / (wheel_name + ".libs") diff --git a/cibuildwheel/resources/_cross_venv.py b/cibuildwheel/resources/_cross_venv.py index ab4a0f410..84237a2eb 100644 --- a/cibuildwheel/resources/_cross_venv.py +++ b/cibuildwheel/resources/_cross_venv.py @@ -11,7 +11,7 @@ def initialize() -> None: - if os.environ.get("CIBW_CROSS_VENV", "0") != "1": + if not (host_triplet := os.environ.get("CIBW_HOST_TRIPLET")): return # os ###################################################################### @@ -24,7 +24,7 @@ def cross_os_uname() -> os.uname_result: # realistic values anyway (from an API level 24 emulator). "3.18.91+", "#1 SMP PREEMPT Tue Jan 9 20:35:43 UTC 2018", - os.environ["HOST"].split("-")[0], + host_triplet.split("-")[0], ) ) @@ -57,11 +57,10 @@ def cross_getandroidapilevel() -> int: # Some packages may recognize sys.cross_compiling from the crossenv tool. sys.cross_compiling = True # type: ignore[attr-defined] sys.getandroidapilevel = cross_getandroidapilevel # type: ignore[attr-defined] - sys.implementation._multiarch = os.environ["HOST"] # type: ignore[attr-defined] + sys.implementation._multiarch = host_triplet # type: ignore[attr-defined] sys.platform = "android" - # _get_sysconfigdata_name is implemented in terms of sys.abiflags, sys.platform and - # sys.implementation._multiarch. Determine the abiflags from the filename. + # Determine the abiflags from the sysconfigdata filename. sysconfigdata_path = next(Path(__file__).parent.glob("_sysconfigdata_*.py")) abiflags_match = re.match(r"_sysconfigdata_(.*?)_", sysconfigdata_path.name) assert abiflags_match is not None @@ -74,6 +73,9 @@ def cross_getandroidapilevel() -> int: # used to generate sysconfig.get_path("include"). exec_prefix = sysconfig.get_config_var("exec_prefix") sysconfig._BASE_PREFIX = sysconfig._BASE_EXEC_PREFIX = exec_prefix # type: ignore[attr-defined] + + # Reload the sysconfigdata file, generating its name from sys.abiflags, + # sys.platform, and sys.implementation._multiarch. sysconfig._init_config_vars() # type: ignore[attr-defined] # sysconfig.get_platform, which determines the wheel tag, is implemented in terms of From a1aedd574a83367052e453104b71cc160acd0a4d Mon Sep 17 00:00:00 2001 From: Malcolm Smith Date: Fri, 11 Jul 2025 21:35:59 +0100 Subject: [PATCH 46/57] Update to Python 3.15.5 --- cibuildwheel/resources/build-platforms.toml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cibuildwheel/resources/build-platforms.toml b/cibuildwheel/resources/build-platforms.toml index a4d105654..73967ffc1 100644 --- a/cibuildwheel/resources/build-platforms.toml +++ b/cibuildwheel/resources/build-platforms.toml @@ -226,8 +226,8 @@ python_configurations = [ [android] python_configurations = [ - { identifier = "cp313-android_arm64_v8a", version = "3.13", url = "https://repo.maven.apache.org/maven2/com/chaquo/python/python/3.13.3.20250603.183345/python-3.13.3.20250603.183345-aarch64-linux-android.tar.gz" }, - { identifier = "cp313-android_x86_64", version = "3.13", url = "https://repo.maven.apache.org/maven2/com/chaquo/python/python/3.13.3.20250603.183345/python-3.13.3.20250603.183345-x86_64-linux-android.tar.gz" }, + { identifier = "cp313-android_arm64_v8a", version = "3.13", url = "https://repo.maven.apache.org/maven2/com/chaquo/python/python/3.13.5/python-3.13.5-aarch64-linux-android.tar.gz" }, + { identifier = "cp313-android_x86_64", version = "3.13", url = "https://repo.maven.apache.org/maven2/com/chaquo/python/python/3.13.5/python-3.13.5-x86_64-linux-android.tar.gz" }, ] [ios] From 8a41764a4e81ab3ada1b827216998059eda6744e Mon Sep 17 00:00:00 2001 From: Malcolm Smith Date: Fri, 11 Jul 2025 22:00:30 +0100 Subject: [PATCH 47/57] Fix PyLint warnings, clarify comment --- cibuildwheel/platforms/android.py | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/cibuildwheel/platforms/android.py b/cibuildwheel/platforms/android.py index f53010add..72f301ba6 100644 --- a/cibuildwheel/platforms/android.py +++ b/cibuildwheel/platforms/android.py @@ -219,11 +219,14 @@ def setup_env( raise errors.FatalError(msg) call("pip", "install", "build", *constraint_flags(dependency_constraint), env=build_env) - # Install build-time requirements. These must be installed for the build platform, - # but queried while simulating Android. The `build` CLI doesn't support this - # combination, so we use its API to query the requirements, but install them - # ourselves. We'll later run `build` in the same environment, passing the - # `--no-isolation` option. + # Build-time requirements must be queried within android_env, because + # `get_requires_for_build` can run arbitrary code in setup.py scripts, which may be + # affected by the target platform. However, the requirements must be installed + # within build_env, because they're going to run on the build machine. + # + # The `build` CLI doesn't support this combination, so we use its API to query the + # requirements, and then install them ourselves with pip. We'll later run `build` in + # the same environment, passing the `--no-isolation` option. class AndroidEnv(IsolatedEnv): @property def python_executable(self) -> str: @@ -250,7 +253,7 @@ def create_cmake_toolchain( ) -> None: toolchain_path = build_path / "toolchain.cmake" build_env["CMAKE_TOOLCHAIN_FILE"] = str(toolchain_path) - with open(toolchain_path, "w") as toolchain_file: + with open(toolchain_path, "w", encoding="UTF-8") as toolchain_file: prefix = f"{python_dir}/prefix" print( dedent( @@ -281,7 +284,7 @@ def localize_sysconfigdata( python_dir: Path, build_env: dict[str, str], sysconfigdata_path: Path ) -> dict[str, Any]: sysconfigdata: dict[str, Any] = run_path(str(sysconfigdata_path))["build_time_vars"] - with sysconfigdata_path.open("w") as f: + with sysconfigdata_path.open("w", encoding="UTF-8") as f: f.write("# Generated by cibuildwheel\n") f.write("build_time_vars = ") sysconfigdata = localized_vars(build_env, sysconfigdata, python_dir / "prefix") @@ -510,16 +513,13 @@ def repair_default( def elf_file_filter(paths: Iterable[Path]) -> Iterator[tuple[Path, ELFFile]]: """Filter through an iterator of filenames and load up only ELF files""" for path in paths: - if path.name.endswith(".py"): - continue - else: + if not path.name.endswith(".py"): try: with open(path, "rb") as f: candidate = ELFFile(f) # type: ignore[no-untyped-call] yield path, candidate except ELFError: - # not an elf file - continue + pass # Not an ELF file def soname_with_hash(src_path: Path) -> str: From bffdb0461f8967f08a69c1923778ffac0c94394b Mon Sep 17 00:00:00 2001 From: Malcolm Smith Date: Tue, 15 Jul 2025 11:27:29 +0100 Subject: [PATCH 48/57] Group common arguments into a dataclass --- cibuildwheel/platforms/android.py | 139 ++++++++++++++---------------- 1 file changed, 66 insertions(+), 73 deletions(-) diff --git a/cibuildwheel/platforms/android.py b/cibuildwheel/platforms/android.py index 72f301ba6..89de954cb 100644 --- a/cibuildwheel/platforms/android.py +++ b/cibuildwheel/platforms/android.py @@ -94,6 +94,16 @@ def before_all(options: Options, python_configurations: list[PythonConfiguration ) +@dataclass(frozen=True) +class BuildState: + config: PythonConfiguration + options: BuildOptions + build_path: Path + python_dir: Path + build_env: dict[str, str] + android_env: dict[str, str] + + def build(options: Options, tmp_path: Path) -> None: if "ANDROID_HOME" not in os.environ: msg = ( @@ -117,8 +127,11 @@ def build(options: Options, tmp_path: Path) -> None: build_options = options.build_options(config.identifier) build_path = tmp_path / config.identifier build_path.mkdir() - python_dir = setup_target_python(config, build_path) + build_env, android_env = setup_env(config, build_options, build_path, python_dir) + state = BuildState( + config, build_options, build_path, python_dir, build_env, android_env + ) compatible_wheel = find_compatible_wheel(built_wheels, config.identifier) if compatible_wheel: @@ -128,22 +141,11 @@ def build(options: Options, tmp_path: Path) -> None: ) repaired_wheel = compatible_wheel else: - build_env, android_env = setup_env(config, build_options, build_path, python_dir) - before_build(build_options, build_env) - built_wheel = build_wheel(build_options, build_path, android_env) - repaired_wheel = repair_wheel( - build_options, build_path, build_env, android_env, built_wheel - ) + before_build(state) + built_wheel = build_wheel(state) + repaired_wheel = repair_wheel(state, built_wheel) - test_wheel( - config, - build_options, - build_path, - python_dir, - build_env, - android_env, - repaired_wheel, - ) + test_wheel(state, repaired_wheel) if compatible_wheel is None: built_wheels.append( @@ -254,7 +256,6 @@ def create_cmake_toolchain( toolchain_path = build_path / "toolchain.cmake" build_env["CMAKE_TOOLCHAIN_FILE"] = str(toolchain_path) with open(toolchain_path, "w", encoding="UTF-8") as toolchain_file: - prefix = f"{python_dir}/prefix" print( dedent( f"""\ @@ -269,7 +270,7 @@ def create_cmake_toolchain( set(CMAKE_SYSTEM_VERSION 1) # Tell CMake where to look for headers and libraries. - list(INSERT CMAKE_FIND_ROOT_PATH 0 {prefix}) + list(INSERT CMAKE_FIND_ROOT_PATH 0 {python_dir}/prefix) set(CMAKE_FIND_ROOT_PATH_MODE_PROGRAM NEVER) set(CMAKE_FIND_ROOT_PATH_MODE_LIBRARY ONLY) set(CMAKE_FIND_ROOT_PATH_MODE_INCLUDE ONLY) @@ -373,30 +374,34 @@ def setup_android_env( return android_env -def before_build(build_options: BuildOptions, env: dict[str, str]) -> None: - if build_options.before_build: +def before_build(state: BuildState) -> None: + if state.options.before_build: log.step("Running before_build...") - shell_prepared(build_options.before_build, build_options=build_options, env=env) + shell_prepared( + state.options.before_build, + build_options=state.options, + env=state.build_env, + ) -def build_wheel(build_options: BuildOptions, build_path: Path, android_env: dict[str, str]) -> Path: +def build_wheel(state: BuildState) -> Path: log.step("Building wheel...") - built_wheel_dir = build_path / "built_wheel" + built_wheel_dir = state.build_path / "built_wheel" call( "python", "-m", "build", - build_options.package_dir, + state.options.package_dir, "--wheel", "--no-isolation", "--skip-dependency-check", f"--outdir={built_wheel_dir}", *get_build_frontend_extra_flags( - build_options.build_frontend, - build_options.build_verbosity, - build_options.config_settings, + state.options.build_frontend, + state.options.build_verbosity, + state.options.config_settings, ), - env=android_env, + env=state.android_env, ) built_wheels = list(built_wheel_dir.glob("*.whl")) @@ -410,26 +415,20 @@ def build_wheel(build_options: BuildOptions, build_path: Path, android_env: dict return built_wheel -def repair_wheel( - build_options: BuildOptions, - build_path: Path, - build_env: dict[str, str], - android_env: dict[str, str], - built_wheel: Path, -) -> Path: +def repair_wheel(state: BuildState, built_wheel: Path) -> Path: log.step("Repairing wheel...") - repaired_wheel_dir = build_path / "repaired_wheel" + repaired_wheel_dir = state.build_path / "repaired_wheel" repaired_wheel_dir.mkdir() - if build_options.repair_command: + if state.options.repair_command: shell( prepare_command( - build_options.repair_command, wheel=built_wheel, dest_dir=repaired_wheel_dir + state.options.repair_command, wheel=built_wheel, dest_dir=repaired_wheel_dir ), - env=build_env, + env=state.build_env, ) else: - repair_default(android_env, built_wheel, repaired_wheel_dir) + repair_default(state.android_env, built_wheel, repaired_wheel_dir) repaired_wheels = list(repaired_wheel_dir.glob("*.whl")) if len(repaired_wheels) == 0: @@ -533,84 +532,78 @@ def soname_with_hash(src_path: Path) -> str: return src_name -# pylint: disable-next=too-many-positional-arguments -def test_wheel( - config: PythonConfiguration, - build_options: BuildOptions, - build_path: Path, - python_dir: Path, - build_env: dict[str, str], - android_env: dict[str, str], - wheel: Path, -) -> None: - if not (build_options.test_command and build_options.test_selector(config.identifier)): +def test_wheel(state: BuildState, wheel: Path) -> None: + test_command = state.options.test_command + if not (test_command and state.options.test_selector(state.config.identifier)): return log.step("Testing wheel...") native_arch = arch_synonym(platform.machine(), platforms.native_platform(), "android") - if config.arch != native_arch: + if state.config.arch != native_arch: log.warning( - f"Skipping tests for {config.arch}, as the build machine only supports {native_arch}" + f"Skipping tests for {state.config.arch}, as the build machine only " + f"supports {native_arch}" ) return - if build_options.before_test: - shell_prepared(build_options.before_test, build_options=build_options, env=build_env) + if state.options.before_test: + shell_prepared( + state.options.before_test, + build_options=state.options, + env=state.build_env, + ) # Install the wheel and test-requires. - site_packages_dir = build_path / "site-packages" + site_packages_dir = state.build_path / "site-packages" site_packages_dir.mkdir() call( "pip", "install", "--only-binary=:all:", "--platform", - sysconfig_print("get_platform()", android_env).replace("-", "_"), + sysconfig_print("get_platform()", state.android_env).replace("-", "_"), "--target", site_packages_dir, - f"{wheel}{build_options.test_extras}", - *build_options.test_requires, - env=build_env, + f"{wheel}{state.options.test_extras}", + *state.options.test_requires, + env=state.build_env, ) # Copy test-sources. - cwd_dir = build_path / "cwd" + cwd_dir = state.build_path / "cwd" cwd_dir.mkdir() - if build_options.test_sources: - copy_test_sources(build_options.test_sources, Path.cwd(), cwd_dir) + if state.options.test_sources: + copy_test_sources(state.options.test_sources, Path.cwd(), cwd_dir) else: (cwd_dir / "test_fail.py").write_text( resources.TEST_FAIL_CWD_FILE.read_text(), ) # Android doesn't support placeholders in the test command. - if any( - ("{" + placeholder + "}") in build_options.test_command - for placeholder in ["project", "package"] - ): + if any(("{" + placeholder + "}") in test_command for placeholder in ["project", "package"]): msg = ( - f"Test command '{build_options.test_command}' with a " + f"Test command '{test_command}' with a " "'{project}' or '{package}' placeholder is not supported on Android, " "because the source directory is not visible on the emulator." ) raise errors.FatalError(msg) # Parse test-command. - test_args = shlex.split(build_options.test_command) + test_args = shlex.split(test_command) if test_args[:2] in [["python", "-c"], ["python", "-m"]]: test_args[:3] = [test_args[1], test_args[2], "--"] elif test_args[0] in ["pytest"]: test_args[:1] = ["-m", test_args[0], "--"] else: msg = ( - f"Test command '{build_options.test_command}' is not supported on " - f"Android. Supported commands are 'python -m', 'python -c' and 'pytest'." + f"Test command '{test_command}' is not supported on Android. " + f"Supported commands are 'python -m', 'python -c' and 'pytest'." ) raise errors.FatalError(msg) # Run the test app. call( - python_dir / "android.py", + state.python_dir / "android.py", "test", "--managed", "maxVersion", @@ -619,7 +612,7 @@ def test_wheel( "--cwd", cwd_dir, *test_args, - env=build_env, + env=state.build_env, ) From 822dda7abe86969fdcf050c76ac6d0dfac89be8d Mon Sep 17 00:00:00 2001 From: Malcolm Smith Date: Tue, 15 Jul 2025 11:49:32 +0100 Subject: [PATCH 49/57] Handle environment variables containing newlines --- cibuildwheel/platforms/android.py | 18 +++++++++++++----- 1 file changed, 13 insertions(+), 5 deletions(-) diff --git a/cibuildwheel/platforms/android.py b/cibuildwheel/platforms/android.py index 89de954cb..891a50738 100644 --- a/cibuildwheel/platforms/android.py +++ b/cibuildwheel/platforms/android.py @@ -348,12 +348,20 @@ def setup_android_env( android_env = build_env.copy() android_env["CIBW_HOST_TRIPLET"] = android_triplet(config.identifier) + # Get the environment variables needed to build for Android (CC, CFLAGS, etc). These are + # generated by https://github.com/python/cpython/blob/main/Android/android-env.sh. env_output = call(python_dir / "android.py", "env", env=build_env, capture_stdout=True) - for line in env_output.splitlines(): - key, value = line.removeprefix("export ").split("=", 1) - value_split = shlex.split(value) - assert len(value_split) == 1, value_split - android_env[key] = value_split[0] + + # shlex.split should produce a sequence alternating between: + # * the word "export" + # * a key=value string, without quotes + for i, token in enumerate(shlex.split(env_output)): + if i % 2 == 0: + assert token == "export", token + else: + key, sep, value = token.partition("=") + assert sep == "=", token + android_env[key] = value # localized_vars cleared the CFLAGS and CXXFLAGS in the sysconfigdata, but most # packages take their optimization flags from these variables. Pass these flags via From bfb3e5e93e1289ceef711c418f0ffa29ac3b41e8 Mon Sep 17 00:00:00 2001 From: Malcolm Smith Date: Tue, 15 Jul 2025 12:41:56 +0100 Subject: [PATCH 50/57] Discourage the use of `pytest` test commands without `python -m` --- cibuildwheel/platforms/android.py | 13 ++++++++++--- docs/options.md | 1 - test/test_android.py | 18 +++++++++++++----- 3 files changed, 23 insertions(+), 9 deletions(-) diff --git a/cibuildwheel/platforms/android.py b/cibuildwheel/platforms/android.py index 891a50738..4d858c5fa 100644 --- a/cibuildwheel/platforms/android.py +++ b/cibuildwheel/platforms/android.py @@ -590,7 +590,7 @@ def test_wheel(state: BuildState, wheel: Path) -> None: # Android doesn't support placeholders in the test command. if any(("{" + placeholder + "}") in test_command for placeholder in ["project", "package"]): msg = ( - f"Test command '{test_command}' with a " + f"Test command {test_command!r} with a " "'{project}' or '{package}' placeholder is not supported on Android, " "because the source directory is not visible on the emulator." ) @@ -601,11 +601,18 @@ def test_wheel(state: BuildState, wheel: Path) -> None: if test_args[:2] in [["python", "-c"], ["python", "-m"]]: test_args[:3] = [test_args[1], test_args[2], "--"] elif test_args[0] in ["pytest"]: + # We transform some commands into the `python -m` form, but this is deprecated. + msg = ( + f"Test command {test_command!r} is not supported on Android. " + "cibuildwheel will try to execute it as if it started with `python -m`. " + "If this works, all you need to do is add that to your test command." + ) + log.warning(msg) test_args[:1] = ["-m", test_args[0], "--"] else: msg = ( - f"Test command '{test_command}' is not supported on Android. " - f"Supported commands are 'python -m', 'python -c' and 'pytest'." + f"Test command {test_command!r} is not supported on Android. " + f"Supported commands are 'python -m' and 'python -c'." ) raise errors.FatalError(msg) diff --git a/docs/options.md b/docs/options.md index accbba6fc..ad2a41b8d 100644 --- a/docs/options.md +++ b/docs/options.md @@ -1347,7 +1347,6 @@ be in one of the following forms: * `python -c command ...` (Android only) * `python -m module-name ...` -* `pytest ...` (deprecated; converted to `python -m pytest ...`) Platform-specific environment variables are also available:
`CIBW_TEST_COMMAND_MACOS` | `CIBW_TEST_COMMAND_WINDOWS` | `CIBW_TEST_COMMAND_LINUX` | `CIBW_TEST_COMMAND_ANDROID` | `CIBW_TEST_COMMAND_IOS` | `CIBW_TEST_COMMAND_PYODIDE` diff --git a/test/test_android.py b/test/test_android.py index 005f22c0e..71749c876 100644 --- a/test/test_android.py +++ b/test/test_android.py @@ -240,21 +240,27 @@ def test_spam(): ) def test_test_command_good(command, expected_output, tmp_path, spam_env, capfd): cibuildwheel_run(tmp_path, add_env={**spam_env, "CIBW_TEST_COMMAND": command}) - assert expected_output in capfd.readouterr().out + stdout, stderr = capfd.readouterr() + assert expected_output in stdout + + if not command.startswith("python"): + assert ( + f"Test command {command!r} is not supported on Android. cibuildwheel " + "will try to execute it as if it started with `python -m`." + ) in stderr @needs_emulator @pytest.mark.parametrize( ("command", "expected_output"), [ - # Build-time failure + # Build-time failure: unrecognized command ( "./test_spam.py", "Test command './test_spam.py' is not supported on Android. " - "Supported commands are 'python -m', 'python -c' and 'pytest'.", + "Supported commands are 'python -m' and 'python -c'.", ), - # Runtime failure - ("pytest test_ham.py", "not found: test_ham.py"), + # Build-time failure: unrecognized placeholder ( "pytest {project}", "Test command 'pytest {project}' with a '{project}' or '{package}' " @@ -265,6 +271,8 @@ def test_test_command_good(command, expected_output, tmp_path, spam_env, capfd): "Test command 'pytest {package}' with a '{project}' or '{package}' " "placeholder is not supported on Android", ), + # Runtime failure + ("pytest test_ham.py", "not found: test_ham.py"), ], ) def test_test_command_bad(command, expected_output, tmp_path, spam_env, capfd): From eda16378f1cb79fa08a5fec3cc1b313d21e79011 Mon Sep 17 00:00:00 2001 From: Malcolm Smith Date: Tue, 15 Jul 2025 13:03:19 +0100 Subject: [PATCH 51/57] Use single quotes in user-visible messages --- cibuildwheel/platforms/android.py | 2 +- test/test_android.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/cibuildwheel/platforms/android.py b/cibuildwheel/platforms/android.py index 4d858c5fa..07e8aec85 100644 --- a/cibuildwheel/platforms/android.py +++ b/cibuildwheel/platforms/android.py @@ -604,7 +604,7 @@ def test_wheel(state: BuildState, wheel: Path) -> None: # We transform some commands into the `python -m` form, but this is deprecated. msg = ( f"Test command {test_command!r} is not supported on Android. " - "cibuildwheel will try to execute it as if it started with `python -m`. " + "cibuildwheel will try to execute it as if it started with 'python -m'. " "If this works, all you need to do is add that to your test command." ) log.warning(msg) diff --git a/test/test_android.py b/test/test_android.py index 71749c876..661ea7880 100644 --- a/test/test_android.py +++ b/test/test_android.py @@ -246,7 +246,7 @@ def test_test_command_good(command, expected_output, tmp_path, spam_env, capfd): if not command.startswith("python"): assert ( f"Test command {command!r} is not supported on Android. cibuildwheel " - "will try to execute it as if it started with `python -m`." + "will try to execute it as if it started with 'python -m'." ) in stderr From cc6251c2d7eaf22a9cc2298882839590adf68222 Mon Sep 17 00:00:00 2001 From: Malcolm Smith Date: Wed, 16 Jul 2025 11:35:25 +0100 Subject: [PATCH 52/57] Improve testing documentation --- README.md | 2 +- docs/platforms.md | 38 +++++++++++++++++--------------------- test/test_android.py | 8 ++------ 3 files changed, 20 insertions(+), 28 deletions(-) diff --git a/README.md b/README.md index 6622451fc..81769f3cf 100644 --- a/README.md +++ b/README.md @@ -68,7 +68,7 @@ Usage ¹ [Requires emulation](https://cibuildwheel.pypa.io/en/stable/faq/#emulation), distributed separately. Other services may also support Linux ARM through emulation or third-party build hosts, but these are not tested in our CI.
² [Uses cross-compilation](https://cibuildwheel.pypa.io/en/stable/faq/#windows-arm64). It is not possible to test `arm64` on this CI platform.
³ Requires a macOS runner; runs tests on the simulator for the runner's architecture.
-⁴ Building for Android requires the runner to be Linux x86_64, macOS ARM64 or macOS x86_64. Testing is supported on the same platforms, but also requires the runner to either be bare-metal, or support nested virtualization. CI platforms known to meet this requirement are: GitHub Actions Linux x86_64.
+⁴ Building for Android requires the runner to be Linux x86_64, macOS ARM64 or macOS x86_64. Testing has [additional requirements](https://cibuildwheel.pypa.io/en/stable/platforms/#android).
Example setup diff --git a/docs/platforms.md b/docs/platforms.md index 8e9327fe6..d67da7a8c 100644 --- a/docs/platforms.md +++ b/docs/platforms.md @@ -183,9 +183,10 @@ Currently, it's recommended to run tests using a `python -m` entrypoint, rather ### Prerequisites -cibuildwheel can build and test Android wheels on any POSIX platform supported by the -Android development tools, which currently means Linux x86_64, macOS ARM64 or macOS -x86_64. +cibuildwheel can build Android wheels on any POSIX platform supported by the Android +development tools, which currently means Linux x86_64, macOS ARM64 or macOS x86_64. Any +of these platforms can be used to build wheels for any Android architecture supported by +Python. However, *testing* wheels has additional requirements: see the section below. If you already have an Android SDK, export the `ANDROID_HOME` environment variable to point at its location. Otherwise, here's how to install it: @@ -220,28 +221,23 @@ Android builds only support the `build` frontend. In principle, support for the cross-platform builds](https://github.com/astral-sh/uv/issues/7957), and [doesn't have support for iOS or Android wheel tags](https://github.com/astral-sh/uv/issues/8029). -### Cross platform builds - -Android builds are *cross platform builds*, as cibuildwheel does not support running -compilers and other build tools "on device". Any supported build platform (listed -above) can be used to build wheels for any supported Android architecture. However, -wheels can only be *tested* on a machine of the same architecture – see the section -below. - ### Tests -If tests have been configured, the test suite will be executed on a Gradle-managed -emulator matching the architecture of the build machine – for example, if you're -building on an ARM64 machine, an ARM64 wheel can be tested on an ARM64 emulator. -Cross-architecture testing is not supported. +Tests are executed on a Gradle-managed emulator matching the architecture of the build +machine – for example, if you're building on an ARM64 machine, then you can test an +ARM64 wheel. Wheels of other architectures can still be built, but testing will +automatically be skipped. + +Running an emulator requires the build machine to either be bare-metal or support +nested virtualization. CI platforms known to meet this requirement are: -On Linux, the emulator needs access to the KVM virtualization interface. If the -emulator fails to start, try running `$ANDROID_HOME/emulator/emulator -accel-check` -for advice. +* GitHub Actions Linux x86_64 -The test process uses the same testbed used by CPython itself to run the CPython test -suite. It is a Gradle project that has been configured to have a single JUnit test, -the result of which reports the success or failure of running the test command. +On Linux, the emulator needs access to the KVM virtualization interface. This may +require adding your user to a group, or [changing your udev +rules](https://github.blog/changelog/2024-04-02-github-actions-hardware-accelerated-android-virtualization-now-available/). +If the emulator fails to start, try running `$ANDROID_HOME/emulator/emulator +-accel-check`. The Android test environment can't support running shell scripts, so the [`test-command`](options.md#test-command) must be a Python command – see its diff --git a/test/test_android.py b/test/test_android.py index 661ea7880..d176b65bd 100644 --- a/test/test_android.py +++ b/test/test_android.py @@ -27,6 +27,7 @@ allow_module_level=True, ) +# Detect CI services which have the Android SDK pre-installed. ci_supports_build = any( key in os.environ for key in [ @@ -42,12 +43,7 @@ else: pytest.skip(msg, allow_module_level=True) -# Running an emulator requires the build machine to either be bare-metal or support -# nested virtualization. Many CI services don't support this. GitHub Actions only -# supports it on Linux -# (https://docs.github.com/en/actions/using-github-hosted-runners/using-github-hosted-runners/about-github-hosted-runners#supported-runners-and-hardware-resources), -# but this may extend to macOS once M3 runners are available -# (https://github.com/ReactiveCircus/android-emulator-runner/issues/350). +# Many CI services don't support running the Android emulator: see platforms.md. ci_supports_emulator = "GITHUB_ACTIONS" in os.environ and platform.system() == "Linux" From 691556fcaa080ee57f8c7ab7a03fd4dcea5c59be Mon Sep 17 00:00:00 2001 From: Malcolm Smith Date: Wed, 16 Jul 2025 11:48:27 +0100 Subject: [PATCH 53/57] Pass wheel filename to `log.build_end` --- cibuildwheel/platforms/android.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/cibuildwheel/platforms/android.py b/cibuildwheel/platforms/android.py index 07e8aec85..3eb1b6573 100644 --- a/cibuildwheel/platforms/android.py +++ b/cibuildwheel/platforms/android.py @@ -147,13 +147,15 @@ def build(options: Options, tmp_path: Path) -> None: test_wheel(state, repaired_wheel) + output_wheel: Path | None = None if compatible_wheel is None: - built_wheels.append( - move_file(repaired_wheel, build_options.output_dir / repaired_wheel.name) + output_wheel = move_file( + repaired_wheel, build_options.output_dir / repaired_wheel.name ) + built_wheels.append(output_wheel) shutil.rmtree(build_path) - log.build_end() + log.build_end(output_wheel) except subprocess.CalledProcessError as error: msg = f"Command {error.cmd} failed with code {error.returncode}. {error.stdout or ''}" From d83bb42095aa29e5f43aa21240da7728b05a644d Mon Sep 17 00:00:00 2001 From: Malcolm Smith Date: Wed, 16 Jul 2025 13:22:50 +0100 Subject: [PATCH 54/57] In GitHub Actions example, skip Android tests on macOS --- examples/github-deploy.yml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/examples/github-deploy.yml b/examples/github-deploy.yml index b9fbe0388..134020bc5 100644 --- a/examples/github-deploy.yml +++ b/examples/github-deploy.yml @@ -47,6 +47,11 @@ jobs: steps: - uses: actions/checkout@v4 + # GitHub Actions can't currently run the Android emulator on macOS. + - name: Skip Android tests on macOS + if: matrix.os == 'android-arm' + run: echo "CIBW_TEST_COMMAND=" >> "$GITHUB_ENV" + # https://github.blog/changelog/2024-04-02-github-actions-hardware-accelerated-android-virtualization-now-available/ - name: Enable KVM for Android emulator if: matrix.os == 'android-intel' From cee4b1671bd2d2eba024a370483085dcbf685cd2 Mon Sep 17 00:00:00 2001 From: Malcolm Smith Date: Thu, 17 Jul 2025 22:33:50 +0100 Subject: [PATCH 55/57] Correct relative paths in `patchelf --set-rpath` --- cibuildwheel/platforms/android.py | 3 ++- pyproject.toml | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/cibuildwheel/platforms/android.py b/cibuildwheel/platforms/android.py index 3eb1b6573..52d4898cb 100644 --- a/cibuildwheel/platforms/android.py +++ b/cibuildwheel/platforms/android.py @@ -8,6 +8,7 @@ import subprocess from collections.abc import Iterable, Iterator from dataclasses import dataclass +from os.path import relpath from pathlib import Path from pprint import pprint from runpy import run_path @@ -513,7 +514,7 @@ def repair_default( call( "patchelf", "--set-rpath", - f"${{ORIGIN}}/{libs_dir.relative_to(path.parent)}", + f"${{ORIGIN}}/{relpath(libs_dir, path.parent)}", path, ) call("wheel", "pack", unpacked_dir, "-d", repaired_wheel_dir) diff --git a/pyproject.toml b/pyproject.toml index 178e1c46b..abe0a92eb 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -45,8 +45,8 @@ dependencies = [ "filelock", "humanize", "packaging>=20.9", - "pyelftools>=0.29", "platformdirs", + "pyelftools>=0.29", "wheel>=0.33.6", ] From d1aae4bc92a652423b920b8f3d9d1ba837101c36 Mon Sep 17 00:00:00 2001 From: Malcolm Smith Date: Sun, 20 Jul 2025 19:22:00 +0100 Subject: [PATCH 56/57] Clarify `test-sources` docs --- README.md | 12 ++++++------ docs/changelog.md | 6 +++--- docs/options.md | 10 +++++----- 3 files changed, 14 insertions(+), 14 deletions(-) diff --git a/README.md b/README.md index 81769f3cf..6ac4156fc 100644 --- a/README.md +++ b/README.md @@ -151,7 +151,7 @@ The following diagram summarises the steps that cibuildwheel takes on each platf | | [`pyodide-version`](https://cibuildwheel.pypa.io/en/stable/options/#pyodide-version) | Specify the Pyodide version to use for `pyodide` platform builds | | **Testing** | [`test-command`](https://cibuildwheel.pypa.io/en/stable/options/#test-command) | The command to test each built wheel | | | [`before-test`](https://cibuildwheel.pypa.io/en/stable/options/#before-test) | Execute a shell command before testing each wheel | -| | [`test-sources`](https://cibuildwheel.pypa.io/en/stable/options/#test-sources) | Files and folders from the source tree that are copied into an isolated tree before running the tests | +| | [`test-sources`](https://cibuildwheel.pypa.io/en/stable/options/#test-sources) | Paths that are copied into the working directory of the tests | | | [`test-requires`](https://cibuildwheel.pypa.io/en/stable/options/#test-requires) | Install Python dependencies before running the tests | | | [`test-extras`](https://cibuildwheel.pypa.io/en/stable/options/#test-extras) | Install your wheel for testing using `extras_require` | | | [`test-groups`](https://cibuildwheel.pypa.io/en/stable/options/#test-groups) | Specify test dependencies from your project's `dependency-groups` | @@ -162,7 +162,7 @@ The following diagram summarises the steps that cibuildwheel takes on each platf | | [`build-verbosity`](https://cibuildwheel.pypa.io/en/stable/options/#build-verbosity) | Increase/decrease the output of the build | - + These options can be specified in a pyproject.toml file, or as environment variables, see [configuration docs](https://cibuildwheel.pypa.io/en/latest/configuration/). @@ -245,10 +245,10 @@ See @henryiii's [release post](https://iscinumpy.dev/post/cibuildwheel-3-0-0/) f - ✨ Adds CPython 3.14 support, under the [`enable` option](https://cibuildwheel.pypa.io/en/stable/options/#enable) `cpython-prerelease`. This version of cibuildwheel uses 3.14.0b2. (#2390) _While CPython is in beta, the ABI can change, so your wheels might not be compatible with the final release. For this reason, we don't recommend distributing wheels until RC1, at which point 3.14 will be available in cibuildwheel without the flag._ (#2390) -- ✨ Adds the [test-sources option](https://cibuildwheel.pypa.io/en/stable/options/#test-sources), and changes the working directory for tests. (#2062, #2284, #2437) - - If this option is set, cibuildwheel will copy the files and folders specified in `test-sources` into the temporary directory we run from. This is required for iOS builds, but also useful for other platforms, as it allows you to avoid placeholders. - - If this option is not set, behaviour matches v2.x - cibuildwheel will run the tests from a temporary directory, and you can use the `{project}` placeholder in the `test-command` to refer to the project directory. (#2420) +- ✨ Adds the [test-sources option](https://cibuildwheel.pypa.io/en/stable/options/#test-sources), which copies files and folders into the temporary working directory we run tests from. (#2062, #2284, #2420, #2437) + + This is particularly important for iOS builds, which do not support placeholders in the `test-command`, but can also be useful for other platforms. - ✨ Adds [`dependency-versions`](https://cibuildwheel.pypa.io/en/stable/options/#dependency-versions) inline syntax (#2122) - ✨ Improves support for Pyodide builds and adds the experimental [`pyodide-version`](https://cibuildwheel.pypa.io/en/stable/options/#pyodide-version) option, which allows you to specify the version of Pyodide to use for builds. (#2002) @@ -292,7 +292,7 @@ _15 March 2025_ - ⚠️ Added warnings when the shorthand values `manylinux1`, `manylinux2010`, `manylinux_2_24`, and `musllinux_1_1` are used to specify the images in linux builds. The shorthand to these (unmaintainted) images will be removed in v3.0. If you want to keep using these images, explicitly opt-in using the full image URL, which can be found in [this file](https://github.com/pypa/cibuildwheel/blob/v2.23.1/cibuildwheel/resources/pinned_docker_images.cfg). (#2312) - 🛠 Dependency updates, including a manylinux update which fixes an [issue with rustup](https://github.com/pypa/cibuildwheel/issues/2303). (#2315) - + --- diff --git a/docs/changelog.md b/docs/changelog.md index a6d61d49e..56eaf599e 100644 --- a/docs/changelog.md +++ b/docs/changelog.md @@ -22,10 +22,10 @@ See @henryiii's [release post](https://iscinumpy.dev/post/cibuildwheel-3-0-0/) f - ✨ Adds CPython 3.14 support, under the [`enable` option](https://cibuildwheel.pypa.io/en/stable/options/#enable) `cpython-prerelease`. This version of cibuildwheel uses 3.14.0b2. (#2390) _While CPython is in beta, the ABI can change, so your wheels might not be compatible with the final release. For this reason, we don't recommend distributing wheels until RC1, at which point 3.14 will be available in cibuildwheel without the flag._ (#2390) -- ✨ Adds the [test-sources option](https://cibuildwheel.pypa.io/en/stable/options/#test-sources), and changes the working directory for tests. (#2062, #2284, #2437) - - If this option is set, cibuildwheel will copy the files and folders specified in `test-sources` into the temporary directory we run from. This is required for iOS builds, but also useful for other platforms, as it allows you to avoid placeholders. - - If this option is not set, behaviour matches v2.x - cibuildwheel will run the tests from a temporary directory, and you can use the `{project}` placeholder in the `test-command` to refer to the project directory. (#2420) +- ✨ Adds the [test-sources option](https://cibuildwheel.pypa.io/en/stable/options/#test-sources), which copies files and folders into the temporary working directory we run tests from. (#2062, #2284, #2420, #2437) + + This is particularly important for iOS builds, which do not support placeholders in the `test-command`, but can also be useful for other platforms. - ✨ Adds [`dependency-versions`](https://cibuildwheel.pypa.io/en/stable/options/#dependency-versions) inline syntax (#2122) - ✨ Improves support for Pyodide builds and adds the experimental [`pyodide-version`](https://cibuildwheel.pypa.io/en/stable/options/#pyodide-version) option, which allows you to specify the version of Pyodide to use for builds. (#2002) diff --git a/docs/options.md b/docs/options.md index 9ffd77e8a..0a98b44b3 100644 --- a/docs/options.md +++ b/docs/options.md @@ -1318,8 +1318,8 @@ Shell command to run tests after the build. The wheel will be installed automatically and available for import from the tests. If this variable is not set, your wheel will not be installed after building. -To ensure the wheel is imported by your tests (instead of your source copy), -**Tests are executed from a temporary directory**, outside of your source +To ensure your tests import the wheel (instead of your source tree), +**tests are executed from a temporary working directory**, outside of your source tree. To access your test code, you have a couple of options: - You can use the [`test-sources`](#test-sources) setting to copy specific @@ -1462,12 +1462,12 @@ Platform-specific environment variables are also available:
### `test-sources` {: #test-sources env-var toml} -> Files and folders from the source tree that are copied into an isolated tree before running the tests +> Paths that are copied into the working directory of the tests A space-separated list of files and folders, relative to the root of the project, required for running the tests. If specified, these files and folders -will be copied into a temporary folder, and that temporary folder will be used -as the working directory for running the test suite. +will be copied into the temporary folder which is used as the working directory +for running the test suite. For more details, see [`test-command`](#test-command). Platform-specific environment variables are also available:
`CIBW_TEST_SOURCES_MACOS` | `CIBW_TEST_SOURCES_WINDOWS` | `CIBW_TEST_SOURCES_LINUX` | `CIBW_TEST_SOURCES_ANDROID` | `CIBW_TEST_SOURCES_IOS` | `CIBW_TEST_SOURCES_PYODIDE` From 416ef032201730a4975bc15330946be3f4e4380f Mon Sep 17 00:00:00 2001 From: Malcolm Smith Date: Tue, 22 Jul 2025 23:04:49 +0100 Subject: [PATCH 57/57] Update to Python 3.13.5+20250722.214220 --- cibuildwheel/resources/build-platforms.toml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cibuildwheel/resources/build-platforms.toml b/cibuildwheel/resources/build-platforms.toml index 8a436e995..371ae0d2a 100644 --- a/cibuildwheel/resources/build-platforms.toml +++ b/cibuildwheel/resources/build-platforms.toml @@ -226,8 +226,8 @@ python_configurations = [ [android] python_configurations = [ - { identifier = "cp313-android_arm64_v8a", version = "3.13", url = "https://repo.maven.apache.org/maven2/com/chaquo/python/python/3.13.5/python-3.13.5-aarch64-linux-android.tar.gz" }, - { identifier = "cp313-android_x86_64", version = "3.13", url = "https://repo.maven.apache.org/maven2/com/chaquo/python/python/3.13.5/python-3.13.5-x86_64-linux-android.tar.gz" }, + { identifier = "cp313-android_arm64_v8a", version = "3.13", url = "https://repo.maven.apache.org/maven2/com/chaquo/python/python/3.13.5+20250722.214220/python-3.13.5+20250722.214220-aarch64-linux-android.tar.gz" }, + { identifier = "cp313-android_x86_64", version = "3.13", url = "https://repo.maven.apache.org/maven2/com/chaquo/python/python/3.13.5+20250722.214220/python-3.13.5+20250722.214220-x86_64-linux-android.tar.gz" }, ] [ios]