deps, native: share rustpython engines and host helpers #7207
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: pyre CI | |
| on: | |
| push: | |
| branches: [ "main", "py3.[0-9]+" ] | |
| paths: | |
| - "Cargo.toml" | |
| - "Cargo.lock" | |
| - "majit/**/*" | |
| - "pyre/**/*" | |
| - "lib-python/**" | |
| - "scripts/install-charon.py" | |
| - "scripts/extract-llbc.py" | |
| # Decides whether the CodSpeed job measures at all, so a change to it | |
| # has to reach the workflow that consults it. | |
| - "scripts/codspeed-environment.py" | |
| # The engine `extract-llbc.py` imports; most of the extraction logic | |
| # lives here, so a change to it alone must still trigger this workflow. | |
| - "scripts/llbc_extract.py" | |
| - ".github/workflows/pyre-ci.yml" | |
| pull_request: | |
| branches: [ "main", "py3.[0-9]+" ] | |
| paths: | |
| - "Cargo.toml" | |
| - "Cargo.lock" | |
| - "majit/**/*" | |
| - "pyre/**/*" | |
| - "lib-python/**" | |
| - "scripts/install-charon.py" | |
| - "scripts/extract-llbc.py" | |
| # Decides whether the CodSpeed job measures at all, so a change to it | |
| # has to reach the workflow that consults it. | |
| - "scripts/codspeed-environment.py" | |
| - "scripts/llbc_extract.py" | |
| - ".github/workflows/pyre-ci.yml" | |
| workflow_dispatch: | |
| concurrency: | |
| # One group per PR and one per tracked branch, so only the newest run of each | |
| # survives: a PR's runs supersede each other, and a merge train on `main` | |
| # cancels the predecessor commit's run rather than keeping every landed commit | |
| # under test. A commit that is superseded before it finishes is therefore | |
| # covered by its successor's run, not by one of its own. | |
| group: pyre-ci-${{ github.event.pull_request.number || github.ref }} | |
| cancel-in-progress: true | |
| permissions: | |
| contents: read | |
| jobs: | |
| cargo-fmt: | |
| name: cargo fmt --check | |
| runs-on: ubuntu-24.04 | |
| steps: | |
| - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 | |
| with: | |
| persist-credentials: false | |
| - uses: dtolnay/rust-toolchain@29eef336d9b2848a0b548edc03f92a220660cdb8 # stable | |
| with: | |
| components: rustfmt | |
| - name: Run cargo fmt --check | |
| run: cargo fmt --all -- --check | |
| - name: Check majit ownership boundary | |
| run: python3 scripts/check-majit-boundary.py | |
| - name: Check synthetic fixture headers | |
| # `pyre/bench/synth/` is shared, so a directive `check.py` requires reds | |
| # every fixture that arrives without one. Asked here because the pass | |
| # reads files and builds nothing (0.2 s over 479 fixtures), while the | |
| # synthetic suite cannot answer until the backend build it sits behind | |
| # has finished. | |
| run: python3 pyre/check.py --check-headers | |
| cpyext-abi: | |
| name: cpyext ABI | |
| runs-on: ubuntu-24.04 | |
| steps: | |
| - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 | |
| with: | |
| persist-credentials: false | |
| - name: Check the exported C declarations | |
| # An extension is compiled against CPython's headers everywhere else, so | |
| # a parameter pyre declares differently is a calling-convention mismatch | |
| # rather than a style difference -- and one no other test here can see, | |
| # because every fixture is compiled against pyre's own header. Needs no | |
| # cargo and no CPython checkout: the declarations are recorded in | |
| # pyre/scripts/cpython-abi.txt. | |
| shell: bash | |
| run: | | |
| python3 pyre/scripts/cpyext-abi.py check | |
| python3 pyre/scripts/cpyext-abi.py generate --check | |
| prepare-charon-llbc-linux: | |
| name: prepare Charon/LLBC (ubuntu-24.04) | |
| runs-on: ubuntu-24.04 | |
| # Every expensive job in this workflow hangs off one of the three prepare | |
| # jobs, so naming `cargo fmt --check` here is what holds a whole run behind | |
| # a 33 s check. Waiting costs no runner-seconds of its own -- a queued job | |
| # occupies no runner -- only the wall time fmt spends queued, which on a | |
| # saturated pool is already the larger half of every job's latency. | |
| # | |
| # Measured over the 95 finished runs of 2026-08-24: `cargo fmt --check` was | |
| # the first job to go red in 5 of the 20 failures, and those five runs spent | |
| # 97751 runner-seconds after it had already failed, against 1545805 for the | |
| # day. `cpyext ABI` is the other job cheap enough to gate on and is left | |
| # ungated: it went red in none of the 95. | |
| needs: cargo-fmt | |
| env: &charon-llbc-env | |
| # Redirect the shared charon/ullbc build dir INTO the workspace so | |
| # actions/cache can reach it. Locally PYRE_SHARED_BUILD is unset and | |
| # both scripts default to ../.pyre-build (shared across worktrees). | |
| PYRE_SHARED_BUILD: ${{ github.workspace }}/.pyre-build | |
| # Single source of truth for the pinned charon tag. MUST mirror | |
| # CHARON_VERSION_DEFAULT in scripts/install-charon.py: it drives both | |
| # the cache keys below AND install-charon.py's .installed-version | |
| # stamp, so a pin bump here misses the cache and re-installs. | |
| CHARON_VERSION: nightly-2026.05.29 | |
| steps: &prepare-charon-llbc-steps | |
| - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 | |
| with: | |
| persist-credentials: false | |
| - name: Check the LLBC fingerprint input walk | |
| # Ahead of the Charon install so a regression reports in seconds. An | |
| # `include*!` spelling the walk cannot resolve widens no digest and fails | |
| # nothing downstream — the artefact just reads as fresh — so this is the | |
| # only place that behaviour is observable. Needs no cargo and no git. | |
| shell: bash | |
| run: scripts/llbc_extract_selftest.py | |
| - uses: dtolnay/rust-toolchain@29eef336d9b2848a0b548edc03f92a220660cdb8 # stable | |
| - uses: Swatinem/rust-cache@e18b497796c12c097a38f9edb9d0641fb99eee32 # v2 | |
| with: | |
| cache-bin: false | |
| shared-key: ${{ runner.arch }} | |
| # Restore-only on PRs (they pull main's cache via restore-keys); save | |
| # only on main so per-PR target caches don't multiply across refs past | |
| # GitHub's 10 GB cache budget and trigger LRU eviction. | |
| save-if: ${{ github.ref == 'refs/heads/main' }} | |
| - name: Cache Charon binary | |
| id: charon-cache | |
| uses: actions/cache@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5.0.5 | |
| with: | |
| # Only the installed charon/charon-driver + .installed-version stamp — | |
| # all the skip check (charon_bin + stamp) and the main build need. | |
| # The Windows from-source tree (.pyre-build/charon-src) is NOT cached: | |
| # it shares this key with the binary, so it never aids a miss (a miss | |
| # restores nothing and rebuilds fully regardless) and only bloats the | |
| # entry against the repo-wide 10 GB cache budget. | |
| path: .pyre-build/charon | |
| key: charon-bin-${{ runner.os }}-${{ runner.arch }}-${{ env.CHARON_VERSION }} | |
| restore-keys: | | |
| charon-bin-${{ runner.os }}-${{ runner.arch }}- | |
| - name: Install Charon | |
| # install-charon.py self-skips on a restored stamp; the guard also | |
| # avoids the network round-trip / from-source rebuild on an exact hit. | |
| if: steps.charon-cache.outputs.cache-hit != 'true' | |
| shell: bash | |
| run: scripts/install-charon.py | |
| - name: Upload Charon artifact | |
| # Hand the installed Charon tree to downstream jobs within this run. | |
| # The cache above is still the cross-run source of truth; this artifact | |
| # avoids late consumer jobs depending on a second cache restore. | |
| uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 | |
| with: | |
| name: charon-${{ runner.os }}-${{ runner.arch }} | |
| path: .pyre-build/charon | |
| retention-days: 1 | |
| include-hidden-files: true | |
| if-no-files-found: error | |
| - name: Compute LLBC fingerprints | |
| id: llbc-fingerprint | |
| shell: bash | |
| # One cache key per Charon target crate. `build/llbc` is empty this early, | |
| # so no crate has a `.readfiles` yet and `source=` is the whole cargo | |
| # closure — which is what a key wants: it must be computable before the | |
| # artefact it will name exists. The narrow digest the stamp records is a | |
| # freshness verdict, not a key, and is not available here. | |
| # | |
| # Take each crate's `source[<crate>]=` line alone. `--fingerprint` prints | |
| # three fields per crate, and `$GITHUB_OUTPUT` parses one `name=value` | |
| # per line, so passing the whole block through would file `closure=` and | |
| # `external=` as outputs of their own and leave the key holding a stray | |
| # `source=` prefix. One `--per-crate` process for all four, not one | |
| # process per crate: the cargo metadata and include walks are shared, and | |
| # four processes cost 42 s on the windows runner. | |
| run: | | |
| # Outside the checkout: the redirect creates the file before the walk | |
| # that would otherwise see it as an untracked input. | |
| fingerprints="$RUNNER_TEMP/llbc-fingerprints.txt" | |
| scripts/extract-llbc.py --fingerprint --per-crate \ | |
| majit-rlib pyre-object pyre-interpreter pyre-jit > "$fingerprints" | |
| for crate in majit-rlib pyre-object pyre-interpreter pyre-jit; do | |
| value="$(sed -n "s/^source\[$crate\]=//p" "$fingerprints")" | |
| test -n "$value" || { echo "no source[$crate]= for $crate" >&2; exit 1; } | |
| echo "${crate//-/_}=${value}" >> "$GITHUB_OUTPUT" | |
| done | |
| # Per-crate caches (per-OS/arch because cfg(target_os)/cfg(target_arch) | |
| # bodies differ). Splitting the former single build/llbc entry lets the | |
| # unchanged crates restore while only the changed crate re-extracts; the | |
| # Extract step below refills any missed crate, so the downstream artefact | |
| # always carries the whole ullbc set. | |
| # | |
| # Each crate's entry also carries its `*.layouts.ullbc` sidecars — the | |
| # same crate re-extracted for a cross target and reduced to its type | |
| # declarations, so a wasm32 build reads wasm32 field offsets rather than | |
| # the extraction host's. They must restore alongside the artefact they | |
| # accompany: a cache hit skips extraction, and `pyre-jit-trace`'s build | |
| # script refuses a cross-target build whose sidecars are absent. | |
| # | |
| # `.ullbc.readfiles` restores for the same reason and is equally required: | |
| # it carries the artefact's own file table, which is what `source=` now | |
| # hashes. Restored without it, every consumer falls back to the whole cargo | |
| # closure and computes a digest that CANNOT equal the narrow one the stamp | |
| # beside it records, so all four crates read STALE while the artefact is | |
| # current. The prefix carries a generation, and it retires the keys whose | |
| # payload predates it — the key is the wide digest, which no edit to the | |
| # extraction driver moves, so without a new namespace the shapeless entries | |
| # would restore forever, and an exact hit skips the Extract step that would | |
| # otherwise re-judge them. `llbc3-` is the generation whose stamps carry | |
| # `artefacts=`, the field recording the artefact bytes the stamp was | |
| # written beside; a stamp without that field is refused, not compared. | |
| # | |
| # Each entry also takes a `restore-keys` prefix, so a miss falls back to | |
| # this crate's most recent entry instead of restoring nothing. The exact | |
| # key is the WIDE digest — `Compute LLBC fingerprints` runs before any | |
| # `.readfiles` exists, so `source=` is the whole cargo closure there and | |
| # any edit anywhere misses every crate. The fallback is what lets the | |
| # narrow digest decide instead: `extract-llbc.py` recomputes each crate's | |
| # stamp against the current tree, using the restored `.readfiles` to | |
| # narrow `source=`, and compares it to the restored stamp as exact text. | |
| # A crate whose own closure did not move skips; every other crate | |
| # re-extracts. So the key only chooses what is OFFERED — a stale | |
| # restore costs a download and is then re-extracted, it cannot survive | |
| # into the artefact. | |
| - name: Cache LLBC (majit-rlib) | |
| id: llbc-cache-rlib | |
| uses: actions/cache@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5.0.5 | |
| with: | |
| path: | | |
| build/llbc/majit-rlib.ullbc | |
| build/llbc/majit-rlib.ullbc.fingerprint | |
| build/llbc/majit-rlib.ullbc.readfiles | |
| build/llbc/majit-rlib.*.layouts.ullbc | |
| key: llbc3-majit-rlib-${{ runner.os }}-${{ runner.arch }}-${{ env.CHARON_VERSION }}-${{ steps.llbc-fingerprint.outputs.majit_rlib }} | |
| restore-keys: | | |
| llbc3-majit-rlib-${{ runner.os }}-${{ runner.arch }}-${{ env.CHARON_VERSION }}- | |
| - name: Cache LLBC (pyre-object) | |
| id: llbc-cache-object | |
| uses: actions/cache@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5.0.5 | |
| with: | |
| path: | | |
| build/llbc/pyre-object.ullbc | |
| build/llbc/pyre-object.ullbc.fingerprint | |
| build/llbc/pyre-object.ullbc.readfiles | |
| build/llbc/pyre-object.*.layouts.ullbc | |
| key: llbc3-pyre-object-${{ runner.os }}-${{ runner.arch }}-${{ env.CHARON_VERSION }}-${{ steps.llbc-fingerprint.outputs.pyre_object }} | |
| restore-keys: | | |
| llbc3-pyre-object-${{ runner.os }}-${{ runner.arch }}-${{ env.CHARON_VERSION }}- | |
| - name: Cache LLBC (pyre-interpreter) | |
| id: llbc-cache-interpreter | |
| uses: actions/cache@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5.0.5 | |
| with: | |
| path: | | |
| build/llbc/pyre-interpreter.ullbc | |
| build/llbc/pyre-interpreter.ullbc.fingerprint | |
| build/llbc/pyre-interpreter.ullbc.readfiles | |
| build/llbc/pyre-interpreter.*.layouts.ullbc | |
| key: llbc3-pyre-interpreter-${{ runner.os }}-${{ runner.arch }}-${{ env.CHARON_VERSION }}-${{ steps.llbc-fingerprint.outputs.pyre_interpreter }} | |
| restore-keys: | | |
| llbc3-pyre-interpreter-${{ runner.os }}-${{ runner.arch }}-${{ env.CHARON_VERSION }}- | |
| - name: Cache LLBC (pyre-jit) | |
| id: llbc-cache-jit | |
| uses: actions/cache@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5.0.5 | |
| with: | |
| path: | | |
| build/llbc/pyre-jit.ullbc | |
| build/llbc/pyre-jit.ullbc.fingerprint | |
| build/llbc/pyre-jit.ullbc.readfiles | |
| key: llbc3-pyre-jit-${{ runner.os }}-${{ runner.arch }}-${{ env.CHARON_VERSION }}-${{ steps.llbc-fingerprint.outputs.pyre_jit }} | |
| restore-keys: | | |
| llbc3-pyre-jit-${{ runner.os }}-${{ runner.arch }}-${{ env.CHARON_VERSION }}- | |
| - name: Extract LLBC | |
| # Runs only when some crate's cache missed. extract-llbc.py self-skips | |
| # the crates whose .ullbc + matching .fingerprint stamp were restored | |
| # above, so only the changed crate(s) recompile. | |
| if: >- | |
| steps.llbc-cache-rlib.outputs.cache-hit != 'true' || | |
| steps.llbc-cache-object.outputs.cache-hit != 'true' || | |
| steps.llbc-cache-interpreter.outputs.cache-hit != 'true' || | |
| steps.llbc-cache-jit.outputs.cache-hit != 'true' | |
| shell: bash | |
| run: scripts/extract-llbc.py majit-rlib pyre-object pyre-interpreter pyre-jit | |
| - name: Upload LLBC artifact | |
| # Hand the ullbc set (freshly extracted, or restored from the cache | |
| # above) to the downstream jobs as a run-scoped artifact. Unlike | |
| # actions/cache, an artifact is exempt from the repo-wide 10 GB cache | |
| # budget and its LRU eviction, so a small ullbc set cannot be evicted | |
| # between this job and a late-scheduled consumer. The LLBC cache above | |
| # is kept only to skip re-extraction across runs. | |
| uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 | |
| with: | |
| name: llbc-${{ runner.os }}-${{ runner.arch }} | |
| path: build/llbc | |
| retention-days: 1 | |
| if-no-files-found: error | |
| prepare-charon-llbc-macos: | |
| name: prepare Charon/LLBC (macos-latest) | |
| runs-on: macos-latest | |
| # `cargo-fmt` is the only Linux job the two platforms wait for, and the | |
| # reason is latency. Job end times on run 32793420243, relative to the | |
| # first job's start: `cargo fmt --check` +47 s, `cargo test (ubuntu)` | |
| # +7977 s, `pyre/check.py dynasm (ubuntu)` +13327 s. Hanging the platform | |
| # prepare on either of the latter two defers everything behind it: that | |
| # run's macOS/Windows legs started at +13330 s and it finished at | |
| # +18269 s, against roughly +5000 s for the same work started after fmt. | |
| # | |
| # A Linux `check.py` gate was tried (PR #1440) and removed. It cost more | |
| # than the round trip: over the 380 finished runs of 2026-08-21..25, 83 | |
| # failure runs had a green Linux `cargo test` and a red `check.py`, and 44 | |
| # of those 83 were ALSO red on a platform leg -- so it withheld a platform | |
| # answer that existed in 53% of the runs it applied to, while tripling the | |
| # wall time of every run that was going to be green anyway. Restricting the | |
| # two platforms to `main` is worse again: 7 runs in that window went red | |
| # only on macOS or Windows, and under that rule they would have landed | |
| # first. | |
| needs: cargo-fmt | |
| env: *charon-llbc-env | |
| steps: *prepare-charon-llbc-steps | |
| prepare-charon-llbc-windows: | |
| name: prepare Charon/LLBC (windows-latest) | |
| # On windows-latest install-charon.py builds Charon from source | |
| # (no Windows release asset is published); see scripts/install-charon.py. | |
| runs-on: windows-latest | |
| # See prepare-charon-llbc-macos for why `cargo-fmt` is the only wait. | |
| needs: cargo-fmt | |
| env: *charon-llbc-env | |
| steps: *prepare-charon-llbc-steps | |
| cargo-test-linux: | |
| name: cargo test (ubuntu-24.04) | |
| runs-on: ubuntu-24.04 | |
| needs: prepare-charon-llbc-linux | |
| if: ${{ !cancelled() && needs.prepare-charon-llbc-linux.result == 'success' }} | |
| # Two cargo passes, one per backend. They measured ~27 min together, so | |
| # the cap bounds a hang without bounding the work. | |
| timeout-minutes: 60 | |
| env: | |
| # See prepare-charon-llbc: downstream jobs download the prepared Charon | |
| # artifact into this workspace path. | |
| PYRE_SHARED_BUILD: ${{ github.workspace }}/.pyre-build | |
| CHARON_VERSION: nightly-2026.05.29 | |
| steps: &cargo-test-steps | |
| - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 | |
| with: | |
| persist-credentials: false | |
| - uses: dtolnay/rust-toolchain@29eef336d9b2848a0b548edc03f92a220660cdb8 # stable | |
| - uses: Swatinem/rust-cache@e18b497796c12c097a38f9edb9d0641fb99eee32 # v2 | |
| with: | |
| cache-bin: false | |
| shared-key: ${{ runner.arch }} | |
| # Restore-only on PRs (they pull main's cache via restore-keys); save | |
| # only on main so per-PR target caches don't multiply across refs past | |
| # GitHub's 10 GB cache budget and trigger LRU eviction. | |
| save-if: ${{ github.ref == 'refs/heads/main' }} | |
| - name: Download Charon artifact | |
| # Run-scoped handoff from prepare-charon-llbc. Cross-run reuse still | |
| # comes from the prepare job's cache; consumers avoid cache eviction | |
| # races by downloading the prepared install tree directly. | |
| uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 | |
| with: | |
| name: charon-${{ runner.os }}-${{ runner.arch }} | |
| path: .pyre-build/charon | |
| - name: Restore Charon executable permissions (Unix) | |
| if: runner.os != 'Windows' | |
| shell: bash | |
| run: chmod +x .pyre-build/charon/*/charon .pyre-build/charon/*/charon-driver | |
| - name: Download LLBC artifact | |
| # Run-scoped handoff from prepare-charon-llbc. Unlike actions/cache, | |
| # an artifact is exempt from the repo-wide 10 GB cache budget and its | |
| # LRU eviction, so a small ullbc set cannot vanish between prepare and | |
| # a late-scheduled consumer (the macOS legs start well after the | |
| # Linux/Windows legs — long enough for a cached entry to be evicted). | |
| uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 | |
| with: | |
| name: llbc-${{ runner.os }}-${{ runner.arch }} | |
| path: build/llbc | |
| - name: Verify prepared Charon/LLBC | |
| shell: bash | |
| run: | | |
| test -d .pyre-build/charon | |
| for crate in majit-rlib pyre-object pyre-interpreter pyre-jit; do | |
| test -s "build/llbc/${crate}.ullbc" | |
| # The sidecar carries the artefact's own file table, which is what | |
| # `source=` hashes. Without it the fingerprint falls back to the whole | |
| # cargo closure and every consumer reports LLBC STALE, so fail here | |
| # where the cause is still visible. | |
| test -s "build/llbc/${crate}.ullbc.readfiles" | |
| done | |
| - name: Free up runner disk space | |
| if: runner.os == 'Linux' | |
| shell: bash | |
| # `cargo test --all` builds every workspace member, and the majit | |
| # demo interpreters (tlr/tl/tla/tiny2/tiny3/tinyframe/braininterp/ | |
| # tlc/calc) plus the wasm crates each statically link the full | |
| # selected backend. On the cranelift leg those extra binaries | |
| # overflow the ubuntu runner's default free disk during the link | |
| # (`No space left on device` -> ld terminates with SIGBUS). Reclaim | |
| # the preinstalled toolchains CI never uses (~20+ GB) so the whole | |
| # workspace's tests build and run instead of excluding crates. | |
| run: | | |
| sudo rm -rf /usr/local/lib/android /usr/share/dotnet /opt/ghc \ | |
| /opt/hostedtoolcache/CodeQL /usr/local/.ghcup /usr/local/share/boost | |
| sudo docker image prune --all --force || true | |
| df -h / | |
| - name: Build test binaries (dynasm) | |
| shell: bash | |
| # Separate compilation failures from test failures in the job log. The | |
| # following test command reuses these artifacts. | |
| run: cargo test --all --no-run --no-default-features --features dynasm,cpyext | |
| - name: Run cargo tests (dynasm) | |
| shell: bash | |
| # Only ONE full-workspace run. The dynasm pass runs the whole workspace | |
| # (--all): that is the single run of every | |
| # backend-independent crate's tests (majit-translate/-ir/-gc/-macros/ | |
| # -trace, pyre-object/-native/-module, the backend crates' own tests, …). | |
| # None of those are gated by the dynasm/cranelift feature, so re-running | |
| # them under cranelift only reproduced byte-identical results while | |
| # relinking ~15 extra test binaries (the majit-translate harness alone | |
| # links >2900 tests). The backend-independent cpyext tests also run here | |
| # once, rather than changing the feature set of the cranelift pass. | |
| run: cargo test --all --no-default-features --features dynasm,cpyext | |
| - name: Run process-exclusive tests (dynasm, serial) | |
| shell: bash | |
| # Eleven tests that are `#[ignore]`d for driving process-global state — a | |
| # registered mutator, stop-the-world — so they cannot share a process | |
| # with the parallel run above. Not skipped for being slow or broken: they | |
| # all pass, and executing them takes under a second. `--test-threads=1` | |
| # is what makes them safe, and it is why they get their own step rather | |
| # than a filter on the run above. majit-gc carries no backend feature of | |
| # its own, so naming one here is rejected outright; selecting it alone | |
| # recompiles its harness, about fifteen seconds. The cranelift backend's | |
| # two equivalents run at the end of the cranelift pass below, where the | |
| # artefacts they need already exist. | |
| run: | | |
| cargo test -p majit-gc --lib -- --ignored --test-threads=1 | |
| cargo test --no-default-features --features dynasm \ | |
| -p majit-backend-dynasm --lib -- --ignored --test-threads=1 | |
| - name: Check the LLBC fingerprint format against its producer | |
| if: runner.os == 'Linux' | |
| shell: bash | |
| # `real_driver_output_parses` is the one test that asks | |
| # `scripts/extract-llbc.py --fingerprint` what it actually prints instead | |
| # of asserting a shape its author believed in, so a producer that extends | |
| # the format cannot silently turn every consumer into "not checked". It | |
| # is `#[ignore]`d because it spawns `python3`, which is why it is pinned | |
| # to Linux: `python3` is not a name that resolves on the windows runner. | |
| # The test target is named under the same `--all` feature set as the | |
| # two steps above so it reuses what they compiled: `-p pyre-jit-trace` | |
| # resolves `pyre-interpreter` differently and rebuilt 61 crates, and | |
| # `pyre-jit-trace` cannot name `cpyext` itself -- the feature belongs | |
| # to `pyre-interpreter`. `prepass` need not be named either: the | |
| # workspace selection enables it through `pyrex`, where naming it | |
| # beside `-p pyre-jit-trace` was what kept the build script off its | |
| # placeholder path. Measured locally, 139s and 281s for those two | |
| # forms against 3s for this one, with 0 crates compiled. | |
| run: | | |
| cargo test --all --no-default-features --features dynasm,cpyext \ | |
| --test llbc_fingerprint_format_test -- --ignored | |
| - name: Build test binaries (cranelift) | |
| shell: bash | |
| # Keep this package list identical to the cranelift test step below. | |
| run: | | |
| cargo test --no-run --no-default-features --features cranelift \ | |
| -p majit-metainterp -p majit-backend-cranelift | |
| - name: Run cargo tests (cranelift) | |
| shell: bash | |
| # This is a separate step so GitHub reports and reruns the backend passes | |
| # independently. Steps remain in one job, in order, so this pass starts | |
| # only after dynasm finishes and reuses that pass's Cargo target cache. | |
| # Exercise the cranelift backend and the metainterp's backend-selection | |
| # path. The dynasm pass above already covers the rest of the workspace; | |
| # rebuilding the pyre stack and every example with a second backend adds | |
| # substantial compile and link cost without covering backend code that | |
| # these two crates do not already exercise. | |
| # `--no-fail-fast` because these are several separate test binaries: | |
| # without it cargo stops at the first one that fails and the run names a | |
| # single failure where there may be more, which costs a round trip per | |
| # hidden sibling. | |
| run: | | |
| cargo test --no-fail-fast --no-default-features --features cranelift \ | |
| -p majit-metainterp -p majit-backend-cranelift | |
| # The two label-selector tests set PYRE_CL_NO_CLOSING_JUMP, a | |
| # process-global env var read while compiling, so they are `#[ignore]`d | |
| # out of the parallel run above and need a serial process of their own. | |
| # The name filter narrows the run to those two tests and reuses the | |
| # backend test harness built above. | |
| cargo test --no-fail-fast -p majit-backend-cranelift \ | |
| --lib -- --ignored --test-threads=1 \ | |
| compiler::tests::test_host_loop_external_jump_to | |
| cargo-test-macos: | |
| name: cargo test (macos-latest) | |
| runs-on: macos-latest | |
| needs: prepare-charon-llbc-macos | |
| if: ${{ !cancelled() && needs.prepare-charon-llbc-macos.result == 'success' }} | |
| env: | |
| PYRE_SHARED_BUILD: ${{ github.workspace }}/.pyre-build | |
| CHARON_VERSION: nightly-2026.05.29 | |
| steps: *cargo-test-steps | |
| cargo-test-windows: | |
| name: cargo test (windows-latest) | |
| runs-on: windows-latest | |
| needs: prepare-charon-llbc-windows | |
| if: ${{ !cancelled() && needs.prepare-charon-llbc-windows.result == 'success' }} | |
| env: | |
| PYRE_SHARED_BUILD: ${{ github.workspace }}/.pyre-build | |
| CHARON_VERSION: nightly-2026.05.29 | |
| steps: *cargo-test-steps | |
| dispatcher-graph-linux: | |
| name: dispatcher-graph acceptance (ubuntu-24.04) | |
| runs-on: ubuntu-24.04 | |
| needs: prepare-charon-llbc-linux | |
| if: ${{ !cancelled() && needs.prepare-charon-llbc-linux.result == 'success' }} | |
| # `slow_generated_jitcodes_preserve_complete_dispatcher_graph` self-ignores | |
| # under `debug_assertions`, so the `cargo test --all` job never runs it: it | |
| # translates the full LLBC set, which is far too slow unoptimized. Left | |
| # unrun it rots — its dispatcher arm counts drifted stale once and masked a | |
| # live lowering defect. | |
| # | |
| # It rode the `pyre/check.py` job on the premise that check.py leaves a warm | |
| # release profile behind. It does not: the test harness pulls dev | |
| # dependencies that are outside pyrex's graph, so the step recompiles 66 | |
| # crates either way — 628s on the longest job in the workflow. The only | |
| # thing it needs from the rest of CI is the extracted LLBC, which is an | |
| # artifact download. | |
| timeout-minutes: 45 | |
| env: | |
| # See prepare-charon-llbc: downstream jobs download the prepared Charon | |
| # artifact into this workspace path. | |
| PYRE_SHARED_BUILD: ${{ github.workspace }}/.pyre-build | |
| CHARON_VERSION: nightly-2026.05.29 | |
| steps: | |
| - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 | |
| with: | |
| persist-credentials: false | |
| - uses: dtolnay/rust-toolchain@29eef336d9b2848a0b548edc03f92a220660cdb8 # stable | |
| - uses: Swatinem/rust-cache@e18b497796c12c097a38f9edb9d0641fb99eee32 # v2 | |
| with: | |
| cache-bin: false | |
| shared-key: ${{ runner.arch }} | |
| # Restore-only on PRs (they pull main's cache via restore-keys); save | |
| # only on main so per-PR target caches don't multiply across refs past | |
| # GitHub's 10 GB cache budget and trigger LRU eviction. | |
| save-if: ${{ github.ref == 'refs/heads/main' }} | |
| - name: Download Charon artifact | |
| uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 | |
| with: | |
| name: charon-${{ runner.os }}-${{ runner.arch }} | |
| path: .pyre-build/charon | |
| - name: Restore Charon executable permissions | |
| shell: bash | |
| run: chmod +x .pyre-build/charon/*/charon .pyre-build/charon/*/charon-driver | |
| - name: Download LLBC artifact | |
| uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 | |
| with: | |
| name: llbc-${{ runner.os }}-${{ runner.arch }} | |
| path: build/llbc | |
| - name: Verify prepared Charon/LLBC | |
| shell: bash | |
| run: | | |
| test -d .pyre-build/charon | |
| for crate in majit-rlib pyre-object pyre-interpreter pyre-jit; do | |
| test -s "build/llbc/${crate}.ullbc" | |
| # The sidecar carries the artefact's own file table, which is what | |
| # `source=` hashes. Without it the fingerprint falls back to the whole | |
| # cargo closure and every consumer reports LLBC STALE, so fail here | |
| # where the cause is still visible. | |
| test -s "build/llbc/${crate}.ullbc.readfiles" | |
| done | |
| - name: Run release-only dispatcher-graph acceptance test | |
| run: | | |
| cargo test --release -p majit-translate \ | |
| --test test_make_jitcodes_produces_graph_keyed_output | |
| # The hand-written GC root brackets, held to what the analysis can prove. | |
| # This job is where the gate is cheapest: all four artefacts are already on | |
| # disk and the scan is ~25 s. Hanging it off `prepare` instead would put it | |
| # on every downstream job's critical path. | |
| - name: Check the GC root brackets | |
| env: | |
| # The checkout is shallow and holds no `main` ref, so the gate cannot | |
| # work out for itself which base its numbers were measured over. | |
| PYRE_GC_GATE_BASE: ${{ github.event.pull_request.base.sha }} | |
| run: | | |
| cargo build -p majit-translate --release --example gc-root-reachability | |
| python3 scripts/check-gc-root-brackets.py | |
| # The graphs the two-phase rtyper prepass still Skips to the legacy walker | |
| # (#346), held to a shrinking list of names rather than a count: a family | |
| # closed and a family reopened cancel in a total, and the reopening is the | |
| # one worth catching. | |
| # | |
| # The reading costs a build script rerun, not a second pipeline: the | |
| # prepass already runs inside `pyre-jit-trace`'s, and MAJIT_DECLINE_LOG | |
| # only asks the gate it already passes through to print what it declined. | |
| # This job is where that is cheapest -- the artefacts are on disk and | |
| # `majit-translate` is warm from the acceptance test above. | |
| # | |
| # The baseline is per platform, and only this job measures one, so a fresh | |
| # platform has none and the gate says so rather than passing. The refreshed | |
| # file is uploaded below so seeding it is a download, not a Linux box. | |
| - name: Check the rtyper skip subjects | |
| id: rtyper-skip | |
| run: python3 scripts/check-rtyper-skip-subjects.py | |
| # Scoped to the step above, not to the job: an earlier step failing leaves | |
| # no census on disk, and refreshing then would report a second failure | |
| # about the first one's absence. | |
| - name: Refresh the skip-subject baseline for download | |
| if: failure() && steps.rtyper-skip.outcome == 'failure' | |
| # `--no-build` reads the stderr the failed step already produced, so this | |
| # re-reads the run that failed rather than measuring a second one. | |
| run: python3 scripts/check-rtyper-skip-subjects.py --no-build --update | |
| - name: Upload the refreshed skip-subject baseline | |
| if: failure() && steps.rtyper-skip.outcome == 'failure' | |
| uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 | |
| with: | |
| name: rtyper-skip-subjects-${{ runner.os }}-${{ runner.arch }} | |
| path: majit/rtyper-skip-subjects.*.txt | |
| retention-days: 7 | |
| if-no-files-found: error | |
| codspeed-benchmarks: | |
| name: CodSpeed benchmarks (ubuntu-24.04) | |
| runs-on: ubuntu-24.04 | |
| needs: prepare-charon-llbc-linux | |
| if: ${{ !cancelled() && needs.prepare-charon-llbc-linux.result == 'success' }} | |
| # The benchmark binary is the interpreter + JIT, so it needs the same | |
| # extracted LLBC every other build here does. That is why this job lives | |
| # in this workflow rather than one of its own: `prepare-charon-llbc-linux` | |
| # hands its artefacts to jobs of the same run, and a separate workflow | |
| # would have to install Charon and re-extract the set for itself. | |
| permissions: | |
| contents: read | |
| # CodSpeed's uploader authenticates with an OIDC token; the | |
| # workflow-level `contents: read` alone does not grant one. | |
| id-token: write | |
| # Simulation mode runs each benchmark under Valgrind, which costs roughly | |
| # two orders of magnitude over native. The six embedded benchmarks it | |
| # replaced measured 9 min end to end in May 2026; the ten real scripts are | |
| # 4.10s of user CPU between them, so the same ~100x lands near 10 min. | |
| timeout-minutes: 60 | |
| env: | |
| # See prepare-charon-llbc: downstream jobs download the prepared Charon | |
| # artifact into this workspace path. | |
| PYRE_SHARED_BUILD: ${{ github.workspace }}/.pyre-build | |
| CHARON_VERSION: nightly-2026.05.29 | |
| steps: | |
| - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 | |
| with: | |
| persist-credentials: false | |
| - uses: dtolnay/rust-toolchain@29eef336d9b2848a0b548edc03f92a220660cdb8 # stable | |
| - name: Restore the machine main last measured on | |
| # A recording that cannot be fetched is read as "none yet", which the step | |
| # below treats as nothing to disagree with. Failing here instead would | |
| # turn a cache outage into a red benchmark job. | |
| continue-on-error: true | |
| uses: actions/cache/restore@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 | |
| with: | |
| path: build/codspeed-environment-reference.json | |
| # One entry per attempt, read back by prefix: the newest recording | |
| # main wrote is what a later run on any other ref is held to. The | |
| # attempt is part of the key because a cache entry cannot be | |
| # overwritten -- a re-run that draws a second machine would otherwise | |
| # leave the first attempt's recording standing as the reference while | |
| # the baseline beside it was measured on the second. | |
| key: codspeed-environment-${{ runner.os }}-${{ runner.arch }}-${{ github.run_id }}-${{ github.run_attempt }} | |
| restore-keys: codspeed-environment-${{ runner.os }}-${{ runner.arch }}- | |
| - name: Name the machine this run drew | |
| id: environment | |
| shell: bash | |
| run: | | |
| reference=build/codspeed-environment-reference.json | |
| # main is not held to a machine, it names one: CodSpeed compares a | |
| # branch against main's latest run, so a rule that stopped main from | |
| # measuring would freeze the base every branch is read against. It | |
| # writes over the recording it restored, and the step below saves that. | |
| # Every other ref measures only when it drew the same machine. | |
| if [ "$GITHUB_REF" = refs/heads/main ]; then | |
| args=(--record "$reference") | |
| else | |
| args=(--record build/codspeed-environment.json --reference "$reference") | |
| fi | |
| python3 scripts/codspeed-environment.py "${args[@]}" --github-output | |
| - uses: Swatinem/rust-cache@e18b497796c12c097a38f9edb9d0641fb99eee32 # v2 | |
| if: ${{ steps.environment.outputs.match != 'false' }} | |
| with: | |
| cache-bin: false | |
| shared-key: ${{ runner.arch }} | |
| # Restore-only on PRs (they pull main's cache via restore-keys); save | |
| # only on main so per-PR target caches don't multiply across refs past | |
| # GitHub's 10 GB cache budget and trigger LRU eviction. | |
| save-if: ${{ github.ref == 'refs/heads/main' }} | |
| - name: Download Charon artifact | |
| if: ${{ steps.environment.outputs.match != 'false' }} | |
| uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 | |
| with: | |
| name: charon-${{ runner.os }}-${{ runner.arch }} | |
| path: .pyre-build/charon | |
| - name: Restore Charon executable permissions | |
| if: ${{ steps.environment.outputs.match != 'false' }} | |
| shell: bash | |
| run: chmod +x .pyre-build/charon/*/charon .pyre-build/charon/*/charon-driver | |
| - name: Download LLBC artifact | |
| if: ${{ steps.environment.outputs.match != 'false' }} | |
| uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 | |
| with: | |
| name: llbc-${{ runner.os }}-${{ runner.arch }} | |
| path: build/llbc | |
| - name: Verify prepared Charon/LLBC | |
| if: ${{ steps.environment.outputs.match != 'false' }} | |
| shell: bash | |
| run: | | |
| test -d .pyre-build/charon | |
| for crate in majit-rlib pyre-object pyre-interpreter pyre-jit; do | |
| test -s "build/llbc/${crate}.ullbc" | |
| # The sidecar carries the artefact's own file table, which is what | |
| # `source=` hashes. Without it the fingerprint falls back to the whole | |
| # cargo closure and every consumer reports LLBC STALE, so fail here | |
| # where the cause is still visible. | |
| test -s "build/llbc/${crate}.ullbc.readfiles" | |
| done | |
| - name: Build pyre-dynasm | |
| if: ${{ steps.environment.outputs.match != 'false' }} | |
| # Through check.py rather than a bare cargo line so the feature set the | |
| # benchmarks measure is the one check.py scores, named in one place. | |
| # `--build=only` also stamps the artefact, so what CodSpeed measures is | |
| # provably built from this tree. | |
| run: python3 pyre/check.py --backend dynasm --build=only | |
| - name: Run the benchmarks | |
| if: ${{ steps.environment.outputs.match != 'false' }} | |
| uses: CodSpeedHQ/action@3194d9a39c4d46684cb44bf7207fc56626aad8fd # v4 | |
| with: | |
| # Simulation rather than walltime: this repository is on a personal | |
| # account, so no dedicated bare-metal (`codspeed-macro`) runner is | |
| # available and a wall-clock measurement on a shared runner would be | |
| # dominated by neighbour noise. See | |
| # https://codspeed.io/docs/instruments/walltime#usage-on-personal-github-accounts | |
| mode: simulation | |
| # No `run:` -- the targets come from codspeed.yml, which spends one | |
| # process per benchmark so the JIT state one builds cannot reach the | |
| # next. See that file for why the harness is `exec` and not cargo. | |
| config: codspeed.yml | |
| - name: Record the machine this baseline was measured on | |
| # Only after the upload succeeded: a recording no run stands behind would | |
| # hold every branch to a machine no base was ever measured on. | |
| if: ${{ success() && github.ref == 'refs/heads/main' }} | |
| uses: actions/cache/save@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 | |
| with: | |
| # The same path the restore above reads, because a cache entry restores | |
| # to the path it was archived from and not to the one asked for. | |
| path: build/codspeed-environment-reference.json | |
| key: codspeed-environment-${{ runner.os }}-${{ runner.arch }}-${{ github.run_id }}-${{ github.run_attempt }} | |
| pyre-check-linux: | |
| name: pyre/check.py ${{ matrix.label }} (ubuntu-24.04) | |
| runs-on: ubuntu-24.04 | |
| needs: prepare-charon-llbc-linux | |
| if: ${{ !cancelled() && needs.prepare-charon-llbc-linux.result == 'success' }} | |
| # The dynasm leg carries the pip end-to-end and the CPython suite as well, | |
| # and measured 48 min without them against the 11 min they take. The cap | |
| # comes from the job they used to be, and still bounds a hang rather than | |
| # the work. | |
| timeout-minutes: 90 | |
| strategy: &pyre-check-backends | |
| # Linux runs one job per backend and macOS runs both backends in one job; | |
| # Windows runs dynasm alone. `check.py` builds each backend's release | |
| # binary itself, and | |
| # splitting the two builds across jobs only pays off where the second | |
| # build is cheap. Runner-seconds spent on check.py, one backend per job | |
| # (run 32657938846) against both in one (run 32647581403): | |
| # ubuntu 3990s vs 4243s, windows 5449s vs 4170s, macos 3704s vs 3136s. | |
| # Only Linux comes out ahead: on the other two the second job pays for a | |
| # full release build over a target directory the first job never warmed, | |
| # and that costs more than the two backends cost in sequence. Collapsing | |
| # the two back into one job on macOS and Windows is worth 1847 runner- | |
| # seconds a run against the 253 the Linux split saves. | |
| # Re-measured on run 32712894722, where both collapsed jobs are green. | |
| # Windows confirms it at 3957s against the 5449s it split into. macOS read | |
| # 4089s and confirms nothing either way: `cargo test (macos-latest)`, which | |
| # nothing here touches, went 1516s to 2757s across the same pair of runs, | |
| # so that runner was about 1.8x slower and the two macOS figures are not | |
| # comparable. The mechanism is the same on all three -- the split builds | |
| # the shared release workspace twice and the collapsed job builds it once | |
| # -- and macOS has the slowest build of the three. | |
| # `fail-fast: false` because one backend going red says nothing about the | |
| # other, and cancelling it would hide a second failure. | |
| fail-fast: false | |
| matrix: | |
| include: | |
| - backend: dynasm | |
| label: dynasm | |
| only: --dynasm-only | |
| - backend: cranelift | |
| label: cranelift | |
| only: --cranelift-only | |
| env: | |
| # See prepare-charon-llbc: downstream jobs download the prepared Charon | |
| # artifact into this workspace path. | |
| PYRE_SHARED_BUILD: ${{ github.workspace }}/.pyre-build | |
| CHARON_VERSION: nightly-2026.05.29 | |
| steps: &pyre-check-steps | |
| - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 | |
| with: | |
| persist-credentials: false | |
| - uses: dtolnay/rust-toolchain@29eef336d9b2848a0b548edc03f92a220660cdb8 # stable | |
| - name: Set up CPython | |
| id: cpython | |
| uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0 | |
| with: | |
| python-version: "3.14" | |
| - name: Set up PyPy | |
| id: pypy | |
| uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0 | |
| with: | |
| python-version: "pypy3.11" | |
| - uses: Swatinem/rust-cache@e18b497796c12c097a38f9edb9d0641fb99eee32 # v2 | |
| with: | |
| cache-bin: false | |
| shared-key: ${{ runner.arch }} | |
| # Restore-only on PRs (they pull main's cache via restore-keys); save | |
| # only on main so per-PR target caches don't multiply across refs past | |
| # GitHub's 10 GB cache budget and trigger LRU eviction. | |
| save-if: ${{ github.ref == 'refs/heads/main' }} | |
| - name: Download Charon artifact | |
| # Run-scoped handoff from prepare-charon-llbc. Cross-run reuse still | |
| # comes from the prepare job's cache; consumers avoid cache eviction | |
| # races by downloading the prepared install tree directly. | |
| uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 | |
| with: | |
| name: charon-${{ runner.os }}-${{ runner.arch }} | |
| path: .pyre-build/charon | |
| - name: Restore Charon executable permissions (Unix) | |
| if: runner.os != 'Windows' | |
| shell: bash | |
| run: chmod +x .pyre-build/charon/*/charon .pyre-build/charon/*/charon-driver | |
| - name: Download LLBC artifact | |
| # Run-scoped handoff from prepare-charon-llbc. Unlike actions/cache, | |
| # an artifact is exempt from the repo-wide 10 GB cache budget and its | |
| # LRU eviction, so a small ullbc set cannot vanish between prepare and | |
| # a late-scheduled consumer (the macOS legs start well after the | |
| # Linux/Windows legs — long enough for a cached entry to be evicted). | |
| uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 | |
| with: | |
| name: llbc-${{ runner.os }}-${{ runner.arch }} | |
| path: build/llbc | |
| - name: Verify prepared Charon/LLBC | |
| shell: bash | |
| run: | | |
| test -d .pyre-build/charon | |
| for crate in majit-rlib pyre-object pyre-interpreter pyre-jit; do | |
| test -s "build/llbc/${crate}.ullbc" | |
| # The sidecar carries the artefact's own file table, which is what | |
| # `source=` hashes. Without it the fingerprint falls back to the whole | |
| # cargo closure and every consumer reports LLBC STALE, so fail here | |
| # where the cause is still visible. | |
| test -s "build/llbc/${crate}.ullbc.readfiles" | |
| done | |
| - name: Only frame construction accesses PyFrame virtualizable fields without a pointer (Linux only) | |
| # A red here means a virtualizable field of `PyFrame` is now reached off | |
| # a non-dereferenced local aggregate somewhere the census does not expect | |
| # — or that frame construction stopped doing so. The codewriter treats | |
| # that shape as not reaching a live virtualizable and keeps it off the | |
| # `getfield_vable_*` / `setfield_vable_*` path, so the set of places with | |
| # that shape is a soundness-relevant invariant rather than a style rule. | |
| # `FrameBox::new` taking its frame by value is what puts its twelve | |
| # projections there; `pyframe.rs` carries a compile-time guard on that | |
| # signature, and this is the corpus-level half. | |
| # | |
| # Reads the extracted LLBC that the steps above already downloaded — no | |
| # corpus build. | |
| # | |
| # The Linux gate is required, not an optimization. This step sits in | |
| # `&pyre-check-steps`, which the macOS and Windows legs alias, so it is | |
| # copied into all three jobs; the census resolves charon per platform and | |
| # has no Windows mapping, and would hard-fail that leg ungated. Running | |
| # once is also sufficient — the property is platform-independent, and | |
| # so is the LLBC it reads, so it rides the dynasm leg of the backend | |
| # matrix instead of running once per backend. | |
| if: runner.os == 'Linux' && matrix.backend == 'dynasm' | |
| shell: bash | |
| run: python3 pyre/scripts/vable-projection-census.py build/llbc/pyre-object.ullbc build/llbc/pyre-interpreter.ullbc build/llbc/pyre-jit.ullbc | |
| - name: Add wasm32 target (Linux only) | |
| # check.py adds wasm to its default backends when this target is | |
| # installed, so this is what makes the Linux leg build and run the wasm | |
| # backend too. wasm output is platform-independent, so exercising it on | |
| # one OS is enough — macOS/Windows keep no wasm32 target and stay on the | |
| # native backends. It rides the dynasm leg of the backend matrix | |
| # because the wasm codegen tests below compare a bench run on wasm | |
| # against the same bench run on dynasm. | |
| if: runner.os == 'Linux' && matrix.backend == 'dynasm' | |
| run: rustup target add wasm32-unknown-unknown | |
| - name: Build pyre backends | |
| # Split from the run below only so that the two are timed apart. One | |
| # invocation reports one duration for a build plus a measurement, and | |
| # this job's 90-minute cap bounds their sum, so nothing today says which | |
| # of the two is growing -- or whether a leg that got slower compiled | |
| # more or ran more. Two steps put both numbers in the run summary, for | |
| # every leg, every run. | |
| # | |
| # It is not a speed change: the same build runs once either way, on the | |
| # same runner over the same target directory. The second invocation pays | |
| # one extra `build_inputs_fingerprint` (under a second; see check.py) and | |
| # nothing else. | |
| # | |
| # `--build=only` rather than a bare `cargo build` because the build is | |
| # what stamps each artefact with its inputs. Built outside check.py the | |
| # artefacts carry no stamp, and the `--build=no` run below drops from | |
| # verifying freshness to noting that it cannot -- which is the gate added | |
| # after this script measured binaries from two different trees and | |
| # printed ALL PASSED. Split this way the gate gets *more* exercise than | |
| # before: the process that measures is no longer the one that built, so | |
| # every run reads the stamp across a process boundary. | |
| # | |
| # The `--backend` expression is duplicated below and the two must stay in | |
| # lockstep: building one set and measuring another would quietly measure | |
| # fewer backends than the leg is named for. It is spelled twice rather | |
| # than hoisted because `runner` is not available in a job-level `env`. | |
| # | |
| # Naming the backend is what makes the matrix legs disjoint: left to | |
| # its default each leg runs every backend it finds a target for. The | |
| # Linux dynasm leg carries wasm too, because it is the leg that | |
| # installed the wasm32 target and the one whose release `pyre-dynasm` | |
| # the wasm codegen tests below need. | |
| env: | |
| PYRE_CHECK_PYTHON3: ${{ steps.cpython.outputs.python-path }} | |
| PYRE_CHECK_PYPY3: ${{ steps.pypy.outputs.python-path }} | |
| run: ${{ steps.cpython.outputs.python-path }} pyre/check.py --backend ${{ (runner.os == 'Linux' && matrix.backend == 'dynasm') && 'dynasm,wasm' || matrix.backend }} --build=only | |
| - name: Run pyre/check.py | |
| # `--build=no` measures the artefacts the step above built and stamped. | |
| # Keep the `--backend` list identical to that step's. | |
| env: | |
| PYRE_CHECK_PYTHON3: ${{ steps.cpython.outputs.python-path }} | |
| PYRE_CHECK_PYPY3: ${{ steps.pypy.outputs.python-path }} | |
| run: ${{ steps.cpython.outputs.python-path }} pyre/check.py --backend ${{ (runner.os == 'Linux' && matrix.backend == 'dynasm') && 'dynasm,wasm' || matrix.backend }} --build=no | |
| - name: Run pyre/extra_tests/parity_tests | |
| # Every script must exit 0 and print "OK" as its last line under CPython | |
| # AND under each pyre backend, so a divergence from CPython observable | |
| # semantics fails here. check.py has just built this leg's release | |
| # binary, so this costs only the per-script process spawns. The `-only` | |
| # flag is what holds the leg to that binary: the runner drives whatever | |
| # it finds in `target/release`, and a restored Cargo cache can hold the | |
| # other backend's binary from a build this job never ran. | |
| # | |
| # Run it even when check.py above failed. The two gates answer different | |
| # questions and check.py is the one that goes red on a jitstats drift | |
| # nobody has re-recorded yet; while it does, every later step in this job | |
| # is skipped and the parity suite reports nothing at all. Windows spent | |
| # its whole history that way: thirteen real parity failures sat unseen | |
| # until one run happened to get past check.py. The job still fails on | |
| # either — this only stops one from hiding the other. | |
| if: ${{ !cancelled() }} | |
| env: | |
| PYRE_CHECK_PYTHON3: ${{ steps.cpython.outputs.python-path }} | |
| run: ${{ steps.cpython.outputs.python-path }} pyre/extra_tests/parity_tests/run.py ${{ matrix.only }} | |
| - name: Run pyre/extra_tests/snippets (gated subset) | |
| # The snippet corpus is mostly imported from RustPython and is not all | |
| # green, so only the scripts declaring `# pyre-check: gate=1` run here — | |
| # the pyre-authored language/builtin gaps, every one of which passes | |
| # under CPython and both backends. They live here rather than as | |
| # `#[cfg(test)]` cases because CPython is one of the runners: a snippet | |
| # that drifts from CPython fails on its own reference, which a Rust test | |
| # asserting the same thing cannot notice. | |
| # | |
| # Same `!cancelled()` reasoning as the parity step above. | |
| if: ${{ !cancelled() }} | |
| env: | |
| PYRE_CHECK_PYTHON3: ${{ steps.cpython.outputs.python-path }} | |
| run: ${{ steps.cpython.outputs.python-path }} pyre/extra_tests/run.py --gated-only ${{ matrix.only }} | |
| - name: Run the vendored root extra_tests | |
| # PyPy's own `extra_tests/` tree, run in place through a driver that | |
| # supplies the pytest names those files use. Only `run.py`'s ENABLED list | |
| # runs; the rest of the tree needs pytest surface, third-party packages, | |
| # or PyPy modules that are not here. Windows selects nothing today and | |
| # the runner exits 0 saying so. | |
| env: | |
| PYRE_CHECK_PYTHON3: ${{ steps.cpython.outputs.python-path }} | |
| run: ${{ steps.cpython.outputs.python-path }} pyre/extra_tests/upstream/run.py ${{ matrix.only }} | |
| - name: JIT structural stats vs committed baseline (Linux, informational) | |
| # Diff each benchmark's [jit-stats] counters against the committed | |
| # pyre/bench/<name>.<backend>.jitstats baselines. The counters are decided | |
| # by the platform-independent tracer, so a diff here means a change moved | |
| # what the JIT compiles. | |
| # | |
| # The sign-stable badness subset (loops_aborted, internal_compile_panics) | |
| # is already HARD-gated by the "Run pyre/check.py" step above: check.py's | |
| # regression floor fails the default run when either counter rises above | |
| # its committed 0, so an inline-abort/compile-panic regression reddens CI | |
| # immediately without this step. That subset is flake-free because the | |
| # healthy value is 0 on every platform. | |
| # | |
| # This step stays informational for the COUNT-VALUED fields | |
| # (guard_failures, loops_compiled, bridges_compiled): the baselines were | |
| # recorded on macOS aarch64 and cross-runner/cross-platform determinism of | |
| # those absolute counts is not yet confirmed in CI, so they must not gate | |
| # main until the summaries stay clean across real PRs. The .out/.time | |
| # baselines are not committed, so their snapshot checks report "missing" | |
| # and only the jit-stats diff is exercised. Runs on Linux only (one OS is | |
| # enough for a platform-independent signal). Use the runner's standard | |
| # grep instead of assuming an optional ripgrep installation. | |
| if: runner.os == 'Linux' | |
| continue-on-error: true | |
| env: | |
| PYRE_CHECK_PYTHON3: ${{ steps.cpython.outputs.python-path }} | |
| PYRE_CHECK_PYPY3: ${{ steps.pypy.outputs.python-path }} | |
| run: | | |
| set -uo pipefail | |
| { | |
| echo '### JIT structural stats vs committed baseline' | |
| echo '#### ${{ matrix.backend }}' | |
| echo '```' | |
| out=$(${{ steps.cpython.outputs.python-path }} pyre/check.py \ | |
| --backend '${{ matrix.backend }}' --build=no --no-synthetic \ | |
| --snapshot-diff 2>&1 || true) | |
| echo "$out" | grep -Ei 'jit-stats diff|snapshot diff' \ | |
| || echo '(no jit-stats output)' | |
| echo '```' | |
| } >> "$GITHUB_STEP_SUMMARY" | |
| - name: Run wasm runtime codegen regression tests (Linux only) | |
| # majit-backend-wasm/tests/codegen_test.rs holds six #[ignore]d runtime | |
| # integration tests that run a bench on both the dynasm and wasm backends | |
| # and compare, so they need the release pyre-dynasm + pyre-wasm-runner and | |
| # the wasm-host module. check.py has just built all three on Linux (it adds | |
| # the wasm32 target above; WASM_MODULE_PATH is the .wasm-host.wasm the test | |
| # loads), so run the ignored tests against them here. The plain | |
| # `cargo test --all` job cannot: it never builds those artifacts. | |
| # | |
| # Left at the default fan-out. All six set PYRE_WASM_JIT_STATS, so they | |
| # share one wasmtime `.cwasm` variant, and both hazards of that sharing | |
| # are closed: check.py warms it, so the five that use the cache | |
| # deserialize instead of each recompiling the ~14MB module (the sixth | |
| # sets PYRE_WASM_NO_CACHE and never touches it), and the runner publishes | |
| # the artifact by renaming a per-process temporary over it, so no run can | |
| # truncate the file another has mapped. Writing it in place is what | |
| # previously left a runner here exiting non-zero having written nothing | |
| # at all, in a different test each time; run_runtime_program now names | |
| # the signal when a child dies on one. | |
| # | |
| # The dynasm leg of the backend matrix is the one holding both halves: | |
| # it is where check.py builds wasm alongside the native backend. | |
| if: runner.os == 'Linux' && matrix.backend == 'dynasm' | |
| run: cargo test -p majit-backend-wasm --test codegen_test -- --ignored | |
| - name: Run pyre/extra_tests/pip (hermetic pip end-to-end, Linux only) | |
| # Drives the release `pyre-dynasm` through venv, ensurepip, a wheel | |
| # install, a PEP 517 build under real isolation, and an uninstall. | |
| # Everything it resolves is a wheel already in the checkout, so it never | |
| # reaches an index -- and one of its checks asserts that, by requiring a | |
| # plain `pip download` to fail. | |
| # | |
| # This and the suite below had a job of their own, which opened by | |
| # running `cargo build --release -p pyrex --bin pyre-dynasm | |
| # --no-default-features --features dynasm` -- the identical command | |
| # `check.py` runs here, 894s in each of the two jobs. The dynasm leg of | |
| # this matrix is the one place in the workflow where that binary already | |
| # exists. It is also the longest Linux job, so appending 11 minutes to it | |
| # is the whole cost: the run is bounded by `cargo test (windows-latest)` | |
| # at 4153s against this leg's 3490s. | |
| # | |
| # `--dynasm-only` because a restored target directory can hold a | |
| # `pyre-cranelift` this leg never built. Ahead of the suite so its verdict | |
| # lands early rather than after the suite's wall time. | |
| if: ${{ !cancelled() && runner.os == 'Linux' && matrix.backend == 'dynasm' }} | |
| shell: bash | |
| run: ${{ steps.cpython.outputs.python-path }} pyre/extra_tests/pip/run.py --dynasm-only | |
| - name: Run CPython suite (gate regressions, JIT on, Linux only) | |
| # This is the only place CI runs the suite -- `pyre/check.py` keeps the | |
| # stage behind `--cpython-suite`, which no job passes, so its wall time is | |
| # paid once. The runner gates against the shared baseline plus its own | |
| # `baseline.linux-x86_64.json` overlay; `PLATFORM_GATED` only handles | |
| # modules CPython skips wholesale on this host. | |
| # | |
| # The runner has 3 cores, so `--jobs 4` oversubscribes them and stretches | |
| # the wall time of whichever module is running when the extra job lands. | |
| # `--timeout` is per module: `test.test_asyncio` alone takes ~117s of CPU | |
| # and ~2m47s of wall time, which does not fit the former 120s limit. | |
| # | |
| # Last, as the longest step here, so every other verdict in this job lands | |
| # before it. It reports whatever check.py and the pip run above did; the | |
| # one case it cannot report on is check.py failing to build the binary, | |
| # where it goes red naming the same cause a second time. | |
| # | |
| # `vm.overcommit_memory=1` before the run: a module that spawns a child | |
| # does it from a process holding the whole JIT heap -- `test_zipfile`'s | |
| # `TestExecutablePrependedZip` runs after the lzma and zip64 classes, so | |
| # its `subprocess.check_output` forks at a ~1.9GB peak. Under the default | |
| # `OVERCOMMIT_GUESS`, `fork` charges the parent's committed size against | |
| # *currently free* memory, so three sibling jobs near their own peaks can | |
| # push that below the bar and the call returns ENOMEM even though nothing | |
| # is close to exhausting the 16GB runner. The child execs immediately and | |
| # touches none of the copied pages, and upstream `fork_exec` takes plain | |
| # `os.fork()` while documenting that `allow_vfork` is a no-op, so the | |
| # heuristic has nothing real to protect here. Keep it non-fatal: if a | |
| # future image refuses the write, the suite should still run and report. | |
| if: ${{ !cancelled() && runner.os == 'Linux' && matrix.backend == 'dynasm' }} | |
| shell: bash | |
| run: | | |
| sudo sysctl -w vm.overcommit_memory=1 || echo "note: could not relax overcommit; fork-heavy modules may report ENOMEM" | |
| ${{ steps.cpython.outputs.python-path }} pyre/cpython_tests/run.py --backend dynasm --baseline pyre/cpython_tests/baseline.json --jobs 3 --timeout 300 | |
| pyre-check-macos: | |
| name: pyre/check.py ${{ matrix.label }} (macos-latest) | |
| runs-on: macos-latest | |
| needs: prepare-charon-llbc-macos | |
| if: ${{ !cancelled() && needs.prepare-charon-llbc-macos.result == 'success' }} | |
| strategy: | |
| # Both backends in one job rather than one job each: see the runner-second | |
| # measurements on `&pyre-check-backends`. The single entry keeps | |
| # `matrix.backend` / `matrix.label` / `matrix.only` defined for the shared | |
| # steps, and the steps that name one backend are gated on Linux anyway. | |
| matrix: | |
| include: | |
| - backend: dynasm,cranelift | |
| label: dynasm+cranelift | |
| only: "" | |
| env: | |
| PYRE_SHARED_BUILD: ${{ github.workspace }}/.pyre-build | |
| CHARON_VERSION: nightly-2026.05.29 | |
| steps: *pyre-check-steps | |
| pyre-check-windows: | |
| name: pyre/check.py ${{ matrix.label }} (windows-latest) | |
| runs-on: windows-latest | |
| needs: prepare-charon-llbc-windows | |
| if: ${{ !cancelled() && needs.prepare-charon-llbc-windows.result == 'success' }} | |
| strategy: | |
| # dynasm alone here. cranelift keeps its own ubuntu leg and rides the | |
| # collapsed macOS job, so what this drops is cranelift on a Windows host, | |
| # and what it saves is this job's second release build. The single entry | |
| # keeps `matrix.backend` / `matrix.label` / `matrix.only` defined for the | |
| # shared steps, and the steps that name one backend are gated on Linux | |
| # anyway. | |
| matrix: | |
| include: | |
| - backend: dynasm | |
| label: dynasm | |
| only: --dynasm-only | |
| env: | |
| PYRE_SHARED_BUILD: ${{ github.workspace }}/.pyre-build | |
| CHARON_VERSION: nightly-2026.05.29 | |
| steps: *pyre-check-steps | |
| sandbox-build: | |
| name: sandbox e2e + wasm web build (ubuntu-24.04) | |
| runs-on: ubuntu-24.04 | |
| needs: prepare-charon-llbc-linux | |
| if: ${{ !cancelled() && needs.prepare-charon-llbc-linux.result == 'success' }} | |
| env: | |
| # See prepare-charon-llbc: downstream jobs download the prepared Charon | |
| # artifact into this workspace path. The sandbox `pyre` binary still | |
| # pulls pyre-jit, whose build needs the extracted LLBC. | |
| PYRE_SHARED_BUILD: ${{ github.workspace }}/.pyre-build | |
| CHARON_VERSION: nightly-2026.05.29 | |
| steps: | |
| - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 | |
| with: | |
| persist-credentials: false | |
| - uses: dtolnay/rust-toolchain@29eef336d9b2848a0b548edc03f92a220660cdb8 # stable | |
| - uses: Swatinem/rust-cache@e18b497796c12c097a38f9edb9d0641fb99eee32 # v2 | |
| with: | |
| cache-bin: false | |
| shared-key: ${{ runner.arch }} | |
| # Restore-only on PRs (they pull main's cache via restore-keys); save | |
| # only on main so per-PR target caches don't multiply across refs past | |
| # GitHub's 10 GB cache budget and trigger LRU eviction. | |
| save-if: ${{ github.ref == 'refs/heads/main' }} | |
| - name: Download Charon artifact | |
| uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 | |
| with: | |
| name: charon-${{ runner.os }}-${{ runner.arch }} | |
| path: .pyre-build/charon | |
| - name: Restore Charon executable permissions (Unix) | |
| if: runner.os != 'Windows' | |
| shell: bash | |
| run: chmod +x .pyre-build/charon/*/charon .pyre-build/charon/*/charon-driver | |
| - name: Download LLBC artifact | |
| uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 | |
| with: | |
| name: llbc-${{ runner.os }}-${{ runner.arch }} | |
| path: build/llbc | |
| - name: Verify prepared Charon/LLBC | |
| shell: bash | |
| run: | | |
| test -d .pyre-build/charon | |
| for crate in majit-rlib pyre-object pyre-interpreter pyre-jit; do | |
| test -s "build/llbc/${crate}.ullbc" | |
| # The sidecar carries the artefact's own file table, which is what | |
| # `source=` hashes. Without it the fingerprint falls back to the whole | |
| # cargo closure and every consumer reports LLBC STALE, so fail here | |
| # where the cause is still visible. | |
| test -s "build/llbc/${crate}.ullbc.readfiles" | |
| done | |
| - name: Add wasm32 target | |
| run: rustup target add wasm32-unknown-unknown | |
| - name: Build pyre-wasm web binding | |
| # Compile check only: nothing here runs the browser binding, and the | |
| # `pyre/check.py` job overwrites pyre_wasm.wasm with its own wasm-host | |
| # build before it snapshots and runs that module. It runs here, and | |
| # first, because this job already carries the checkout, toolchain, Cargo | |
| # cache and prepared Charon/LLBC set the build needs, and finishes well | |
| # inside the wall time of the check.py leg it used to sit in. | |
| # | |
| # It was placed ahead of check.py on the premise that the two wasm | |
| # flavours would share compiled dependencies. Measured: the web build | |
| # takes 6m and leaves the later wasm-host build 50s cheaper (2m32 -> | |
| # 1m42), because check.py sets RUSTFLAGS the web build does not and half | |
| # the graph recompiles under the difference. So the sharing is real and | |
| # costs about seven times what it returns. | |
| run: | | |
| cargo build --release -p pyre-wasm \ | |
| --target wasm32-unknown-unknown \ | |
| --no-default-features --features web | |
| - name: Build pyre --features sandbox | |
| # A green sandbox build is the fails-closed proof: every mediated module | |
| # names libc through host_seam::sys, so any direct syscall left outside | |
| # the seam fails to compile here. On Linux `sandbox` installs the seccomp | |
| # backstop by default, so this also compile-checks the install path. | |
| run: cargo build --release -p pyrex --bin pyre --features sandbox | |
| - name: Sandbox compile-out fence (clippy) | |
| # Extends the fails-closed proof beyond libc. host_seam::sys already makes | |
| # a stray `libc::` syscall fail to compile under sandbox; this forbids the | |
| # non-libc host surface too (std::fs / std::env / std::io stdio / | |
| # std::process / std::net) in the untrusted interpreter. `--no-deps` scopes | |
| # it to pyre-interpreter's own code — the trusted `pyre-sandbox` controller | |
| # and the host-side build script are exempt. CLIPPY_CONF_DIR points clippy | |
| # at ci/clippy-sandbox/clippy.toml, which normal clippy never discovers, so | |
| # a green run proves no raw host call survives in sandbox-live code. | |
| env: | |
| CLIPPY_CONF_DIR: ${{ github.workspace }}/ci/clippy-sandbox | |
| run: cargo clippy -p pyre-interpreter --no-deps --features sandbox,dynasm -- -A clippy::all -D clippy::disallowed_methods -D clippy::disallowed_types | |
| - name: Run sandbox end-to-end suite | |
| # Exercises the compile-out sandbox (virtual FS + escape blocking) through | |
| # the controller. On Linux the e2e binary (built `--features sandbox`) | |
| # installs the seccomp allowlist, so this run also RUNTIME-VALIDATES it: a | |
| # syscall the interpreter needs but the allowlist omits makes the child | |
| # SIGSYS (exit 159) and fails the suite. | |
| run: cargo test --release -p pyre-sandbox --test e2e_interact -- --ignored --nocapture |