diff --git a/.github/workflows/linux.yml b/.github/workflows/linux.yml index f2bf45fb..901233ad 100644 --- a/.github/workflows/linux.yml +++ b/.github/workflows/linux.yml @@ -168,7 +168,7 @@ jobs: - generate-matrix - pythonbuild - image - runs-on: depot-ubuntu-22.04 + runs-on: ${{ matrix.runner }} strategy: matrix: ${{ fromJson(needs.generate-matrix.outputs.matrix) }} fail-fast: false diff --git a/.github/workflows/macos.yml b/.github/workflows/macos.yml index 122202e8..7aafefa9 100644 --- a/.github/workflows/macos.yml +++ b/.github/workflows/macos.yml @@ -90,7 +90,7 @@ jobs: strategy: matrix: ${{ fromJson(needs.generate-matrix.outputs.matrix) }} fail-fast: false - runs-on: depot-macos-latest + runs-on: ${{ matrix.runner }} name: ${{ matrix.target_triple }} / ${{ matrix.python }} / ${{ matrix.build_options }} steps: - uses: actions/checkout@v4 diff --git a/.github/workflows/windows.yml b/.github/workflows/windows.yml index 3d5a7bd0..c625b410 100644 --- a/.github/workflows/windows.yml +++ b/.github/workflows/windows.yml @@ -88,7 +88,7 @@ jobs: needs: - generate-matrix - pythonbuild - runs-on: windows-latest-large + runs-on: ${{ matrix.runner }} strategy: matrix: ${{ fromJson(needs.generate-matrix.outputs.matrix) }} fail-fast: false diff --git a/ci-matrix.py b/ci-matrix.py index 21363cc2..e9b882fa 100644 --- a/ci-matrix.py +++ b/ci-matrix.py @@ -14,6 +14,7 @@ from packaging.version import Version CI_TARGETS_YAML = "ci-targets.yaml" +CI_RUNNERS_YAML = "ci-runners.yaml" CI_EXTRA_SKIP_LABELS = ["documentation"] @@ -88,6 +89,7 @@ def should_include_entry(entry: dict[str, str], filters: dict[str, set[str]]) -> def generate_matrix_entries( config: dict[str, Any], + runners: dict[str, Any], platform_filter: Optional[str] = None, label_filters: Optional[dict[str, set[str]]] = None, ) -> list[dict[str, str]]: @@ -103,6 +105,7 @@ def generate_matrix_entries( target_triple, target_config, platform, + runners, label_filters.get("directives", set()), ) @@ -117,22 +120,50 @@ def generate_matrix_entries( return matrix_entries +def find_runner(runners: dict[str, Any], platform: str, arch: str) -> str: + # Find a matching platform first + match_platform = [ + runner for runner in runners if runners[runner]["platform"] == platform + ] + + # Then, find a matching architecture + match_arch = [ + runner for runner in match_platform if runners[runner]["arch"] == arch + ] + + # If there's a matching architecture, use that + if match_arch: + return match_arch[0] + + # Otherwise, use the first with a matching platform + if match_platform: + return match_platform[0] + + raise RuntimeError(f"No runner found for platform {platform!r} and arch {arch!r}") + + def add_matrix_entries_for_config( matrix_entries: list[dict[str, str]], target_triple: str, config: dict[str, Any], platform: str, + runners: dict[str, Any], directives: set[str], ) -> None: python_versions = config["python_versions"] build_options = config["build_options"] arch = config["arch"] + runner = find_runner(runners, platform, arch) # Create base entry that will be used for all variants base_entry = { "arch": arch, "target_triple": target_triple, "platform": platform, + "runner": runner, + # If `run` is in the config, use that — otherwise, default to if the + # runner architecture matches the build architecture + "run": str(config.get("run", runners[runner]["arch"] == arch)).lower(), } # Add optional fields if they exist @@ -142,8 +173,6 @@ def add_matrix_entries_for_config( base_entry["libc"] = config["libc"] if "vcvars" in config: base_entry["vcvars"] = config["vcvars"] - if "run" in config: - base_entry["run"] = str(config["run"]).lower() if "dry-run" in directives: base_entry["dry-run"] = "true" @@ -191,6 +220,11 @@ def parse_args() -> argparse.Namespace: "--labels", help="Comma-separated list of labels to filter by (e.g., 'platform:darwin,python:3.13,build:debug'), all must match.", ) + parser.add_argument( + "--free-runners", + action="store_true", + help="If only free runners should be used.", + ) return parser.parse_args() @@ -201,9 +235,21 @@ def main() -> None: with open(CI_TARGETS_YAML, "r") as f: config = yaml.safe_load(f) + with open(CI_RUNNERS_YAML, "r") as f: + runners = yaml.safe_load(f) + + # If only free runners are allowed, reduce to a subset + if args.free_runners: + runners = { + runner: runner_config + for runner, runner_config in runners.items() + if runner_config.get("free") + } + matrix = { "include": generate_matrix_entries( config, + runners, args.platform, labels, ) diff --git a/ci-runners.yaml b/ci-runners.yaml new file mode 100644 index 00000000..aeb0e5ca --- /dev/null +++ b/ci-runners.yaml @@ -0,0 +1,37 @@ +# Describes the runners that the CI system can use + +depot-ubuntu-22.04: + arch: x86_64 + platform: linux + free: false + +# TODO: Enable this runner to perform native builds for aarch64 +# depot-ubuntu-22.04-arm: +# arch: aarch64 +# platform: linux +# free: false + +depot-macos-latest: + arch: x86_64 + platform: darwin + free: false + +ubuntu-latest: + arch: x86_64 + platform: linux + free: true + +macos-latest: + arch: x86_64 + platform: darwin + free: true + +windows-latest-large: + arch: x86_64 + platform: windows + free: false + +windows-latest: + arch: x86_64 + platform: windows + free: true