diff --git a/.github/workflows/pyre-ci.yml b/.github/workflows/pyre-ci.yml index d3e0be5a906..2ae6f9a717d 100644 --- a/.github/workflows/pyre-ci.yml +++ b/.github/workflows/pyre-ci.yml @@ -576,9 +576,9 @@ jobs: # leg; waiting for macOS races the Linux upload. This job is the only place # CI runs the suite — `pyre/check.py` keeps the stage behind # `--cpython-suite`, which no job passes, so its wall time is paid once. - # Separately, the baseline is darwin-arm64-specific - # (`CPYTHON_SUITE_BASELINE_HOST`); `PLATFORM_GATED` only handles modules - # CPython skips wholesale on this host. + # Separately, this runner gates against the shared baseline plus its own + # `baseline.linux-x86_64.json` overlay; `PLATFORM_GATED` only handles + # modules CPython skips wholesale on this host. needs: prepare-charon-llbc-linux if: ${{ !cancelled() && needs.prepare-charon-llbc-linux.result == 'success' }} timeout-minutes: 30 diff --git a/pyre/check.py b/pyre/check.py index 6474875b9fa..05e964be0c0 100644 --- a/pyre/check.py +++ b/pyre/check.py @@ -8,7 +8,6 @@ import difflib import math import os -import platform import re import shutil import statistics @@ -146,11 +145,6 @@ def _detect_pyre_stdlib(): BENCH_DIR = "pyre/bench" SYNTHETIC_BENCH_DIR = "pyre/bench/synth" CPYTHON_SUITE_BASELINE = "pyre/cpython_tests/baseline.json" -# The baseline holds one verdict per module per backend, and dynasm emits -# arch-specific code, so a verdict only compares against the host it was -# observed on. `.github/workflows/pyre-ci.yml` pins its CPython suite job to -# `macos-latest` for the same reason. -CPYTHON_SUITE_BASELINE_HOST = ("darwin", "arm64") # Per module, matching the CI job. `test.test_asyncio` alone needs ~2m47s of # wall time, so a smaller per-module limit turns a slow module into a fake # regression. @@ -2430,10 +2424,11 @@ def run_cpython_suite(self): specialised-pair subscript fold while the whole synthetic corpus and every parity fixture stayed green. - The baseline records one verdict per module per backend, observed on - darwin-arm64, and dynasm's codegen is arch-specific -- so the - comparison only means anything there. On any other host the stage - reports that it did not run instead of counting as a pass. + The baseline records one verdict per module per backend and follows + the host: `run.py` reads a `baseline.-.json` + overlay before the shared file, so a verdict a host disagrees with + (dynasm's codegen is arch-specific) is recorded there rather than + making the stage unusable off one machine. Off by default and reached only through `--cpython-suite`: the suite costs more wall time than every other stage here put together, and the @@ -2449,15 +2444,6 @@ def run_cpython_suite(self): print(dim("skip (backend not enabled)")) self._append_comparison(backend, name, "-", "-", "skip") return - host = (sys.platform, platform.machine()) - if host != CPYTHON_SUITE_BASELINE_HOST: - sys.stdout.write(f" {backend:<10s}") - print(dim( - f"skip (baseline observed on {'-'.join(CPYTHON_SUITE_BASELINE_HOST)}, " - f"host is {'-'.join(host)})" - )) - self._append_comparison(backend, name, "-", "-", "skip") - return sys.stdout.write(f" {backend:<10s}") sys.stdout.flush() output, elapsed, code, stderr = run_timed( diff --git a/pyre/cpython_tests/run.py b/pyre/cpython_tests/run.py index fb7482ddcb6..f6a63e0810a 100644 --- a/pyre/cpython_tests/run.py +++ b/pyre/cpython_tests/run.py @@ -37,9 +37,13 @@ `--strict-baseline` (gates on unrecorded improvements), `--full`, or `--update-baseline` to detect them. `SKIP` baseline entries are not run. -Baseline entries carry no platform. `PLATFORM_GATED` excludes modules CPython -skips wholesale on this host, preventing another host's PASS from becoming a -false `PASS -> SKIP` regression; it does not make the baseline portable. +The baseline is a shared file plus a per-host overlay, +`baseline.-.json`, consulted first. Whichever host +records a (module, backend) first sets the shared verdict; every other host +writes an entry only where it disagrees, and drops it again once the two +agree. Two separate mechanisms sit next to this: `PLATFORM_GATED` excludes +modules CPython skips wholesale on a host, and `KNOWN_SKIPS` is a decision +about the module rather than the host, so it stays shared. Usage: python3 pyre/cpython_tests/run.py [--backend dynasm|cranelift] @@ -54,6 +58,7 @@ import concurrent.futures import json import os +import platform import signal import subprocess import sys @@ -66,6 +71,7 @@ TESTDIR = ROOT / "lib-python" / "3" / "test" STDLIB_VERSION_FILE = ROOT / "lib-python" / "stdlib-version.txt" DEFAULT_BASELINE = HERE / "baseline.json" +HOST_TAG = f"{sys.platform}-{platform.machine()}" EXE = ".exe" if sys.platform == "win32" else "" BIN_NAME = {"dynasm": "pyre-dynasm", "cranelift": "pyre-cranelift"} @@ -453,11 +459,30 @@ def load_baseline(path: Path) -> dict: return json.loads(path.read_text(encoding="utf-8")) -def expected_status(baseline: dict, module: str, backend: str) -> str | None: - entry = baseline.get("modules", {}).get(module) - if entry is None: - return None - return entry.get(backend) or entry.get("dynasm") +def host_baseline_path(path: Path) -> Path: + """Per-host overlay beside the shared baseline, `baseline..json`. + + Same shape as the shared file and consulted first, so a host records only + the modules it genuinely disagrees with. The jit-stats baselines overlay + on `sys.platform` alone; a verdict also has to separate the architectures + within a platform, because the dynasm backend emits different machine code + on each and a miscompile is not portable. + """ + return path.with_name(f"{path.stem}.{HOST_TAG}{path.suffix}") + + +def expected_status(baseline: dict, overlay: dict, module: str, + backend: str) -> str | None: + """Recorded verdict for `module`, the host overlay winning over the shared + file. Within one file `dynasm` stands in for a backend with no entry.""" + for source in (overlay, baseline): + entry = source.get("modules", {}).get(module) + if entry is None: + continue + status = entry.get(backend) or entry.get("dynasm") + if status is not None: + return status + return None # ── main ───────────────────────────────────────────────────────────── @@ -518,6 +543,8 @@ def main() -> int: return 2 baseline = load_baseline(args.baseline) + overlay_path = host_baseline_path(args.baseline) + overlay = load_baseline(overlay_path) if overlay_path.exists() else {"modules": {}} modules = discover_modules(args.filter) if args.list: @@ -558,7 +585,7 @@ def main() -> int: off_platform.append((m, gate_reason)) skipped.append(m) continue - exp = expected_status(baseline, m, args.backend) + exp = expected_status(baseline, overlay, m, args.backend) is_skip = (exp == "SKIP") or (m in KNOWN_SKIPS) if is_skip and not args.full and not args.update_baseline: skipped.append(m) @@ -571,6 +598,10 @@ def main() -> int: print(f"pyre CPython suite — backend={args.backend} mode={args.mode} " f"jit={'off' if args.no_jit else 'on'} jobs={args.jobs}") print(f"binary: {binary}") + overlay_count = len(overlay.get("modules", {})) + print(f"baseline: {args.baseline.name} + " + + (f"{overlay_path.name} ({overlay_count} host entries)" + if overlay_count else f"no {HOST_TAG} overlay")) for m, reason in off_platform: print(f" off-platform on {sys.platform}: {m} ({reason})") extra = f", {deselected} not gated (non-PASS)" if deselected else "" @@ -632,7 +663,7 @@ def main() -> int: regressions: list[str] = [] improvements: list[str] = [] for m, (status, detail) in sorted(results.items()): - exp = expected_status(baseline, m, args.backend) + exp = expected_status(baseline, overlay, m, args.backend) if exp == "PASS" and status != "PASS": regressions.append(f"{m}: PASS -> {status} {detail}") elif exp != "PASS" and status == "PASS": @@ -659,9 +690,11 @@ def main() -> int: print(f"\nreport written: {args.report}") if args.update_baseline: - write_baseline(args.baseline, baseline, results, args.backend) - print(f"\nbaseline written: {args.baseline} " - f"({sum(1 for s, _ in results.values() if s == 'PASS')} PASS recorded)") + written = write_baseline(args.baseline, baseline, overlay, results, + args.backend) + recorded = sum(1 for s, _ in results.values() if s == "PASS") + for target in written: + print(f"\nbaseline written: {target} ({recorded} PASS recorded)") return 0 if regressions: @@ -682,25 +715,68 @@ def main() -> int: return 0 -def write_baseline(path: Path, baseline: dict, results: dict, backend: str) -> None: +def write_baseline(path: Path, baseline: dict, overlay: dict, results: dict, + backend: str) -> list[Path]: + """Record `results`, splitting them between the shared baseline and this + host's overlay. Returns the files actually written. + + A verdict the shared file has never seen establishes the shared answer, so + whichever host records first sets it and no host is privileged. After + that a host writes to its own overlay only where it disagrees, and a run + that comes back into agreement drops the overlay entry again -- so an + overlay never outlives the divergence that created it. + """ modules = baseline.setdefault("modules", {}) + overlay_modules = overlay.setdefault("modules", {}) baseline["stdlib_version"] = stdlib_version() + overlay_dirty = False for m, (status, _detail) in results.items(): # Defensive: never overwrite another platform's recorded result. if platform_gate(m) is not None: continue - entry = modules.setdefault(m, {}) # A curated KNOWN_SKIP stays SKIP regardless of what the run observed # (it is a "do not run" decision, not a result). Modules absent from # `results` (phantom skips that no longer exist) are simply not added. + # It is a decision about the module rather than about this host, so it + # is shared and never overlaid. if m in KNOWN_SKIPS: + entry = modules.setdefault(m, {}) entry[backend] = "SKIP" entry.setdefault("reason", KNOWN_SKIPS[m]) - else: - entry[backend] = status + overlay_dirty |= overlay_modules.pop(m, None) is not None + continue + shared = modules.get(m, {}).get(backend) + if shared is None: + modules.setdefault(m, {})[backend] = status + continue + if status == shared: + host_entry = overlay_modules.get(m) + if host_entry is not None and host_entry.pop(backend, None) is not None: + overlay_dirty = True + if not host_entry: + del overlay_modules[m] + continue + if overlay_modules.setdefault(m, {}).get(backend) != status: + overlay_modules[m][backend] = status + overlay_dirty = True + + written = [path] path.parent.mkdir(parents=True, exist_ok=True) path.write_text(json.dumps(baseline, indent=2, sort_keys=True) + "\n", encoding="utf-8") + overlay_path = host_baseline_path(path) + if overlay_dirty or overlay_path.exists(): + if overlay_modules: + overlay["host"] = HOST_TAG + overlay["stdlib_version"] = stdlib_version() + overlay_path.write_text( + json.dumps(overlay, indent=2, sort_keys=True) + "\n", + encoding="utf-8") + written.append(overlay_path) + elif overlay_path.exists(): + overlay_path.unlink() + written.append(overlay_path) + return written if __name__ == "__main__": diff --git a/pyre/pyre-interpreter/src/module/_blake2/_blake2_app.py b/pyre/pyre-interpreter/src/module/_blake2/_blake2_app.py index 11ca6130a45..0946fa3bcbe 100644 --- a/pyre/pyre-interpreter/src/module/_blake2/_blake2_app.py +++ b/pyre/pyre-interpreter/src/module/_blake2/_blake2_app.py @@ -26,6 +26,19 @@ def __setattr__(cls, name, value): _MISSING = object() +def _buffer_size(value): + """Byte length of a buffer parameter, the `Py_buffer.len` the clinic + converter measures. + + Not `len()`: a memoryview over a wider itemsize counts items, so + `array('I', [0] * 5)` is 5 there and 20 bytes here, and only the byte + count decides whether the salt fits. + """ + if type(value) is bytes: + return len(value) + return memoryview(value).nbytes + + def _make_blake_type(class_name, _salt_size, _person_size, _key_size, _digest_size, max_offset, _block_size): class _Blake(metaclass=_Immutable): @@ -99,6 +112,17 @@ def __new__(cls, *args, **kwargs): "digest_size must be between 1 and %d bytes" % cls.MAX_DIGEST_SIZE ) + # Salt and person are rejected before the tree parameters and the + # key after them, the order lib_pypy/_blake2 sets each field in. + # `blake2b(salt=b'x' * 17, fanout=256)` reports the salt. + if _buffer_size(salt) > cls.SALT_SIZE: + raise ValueError( + "maximum salt length is %d bytes" % cls.SALT_SIZE + ) + if _buffer_size(person) > cls.PERSON_SIZE: + raise ValueError( + "maximum person length is %d bytes" % cls.PERSON_SIZE + ) if not 0 <= fanout <= 255: raise ValueError("fanout must be between 0 and 255") if not 1 <= depth <= 255: @@ -118,6 +142,10 @@ def __new__(cls, *args, **kwargs): "inner_size must be between 0 and %d" % cls.MAX_DIGEST_SIZE ) + if _buffer_size(key) > cls.MAX_KEY_SIZE: + raise ValueError( + "maximum key length is %d bytes" % cls.MAX_KEY_SIZE + ) # Both clinic bool converters are observable through __bool__. bool(usedforsecurity) diff --git a/pyre/pyre-interpreter/src/typedef.rs b/pyre/pyre-interpreter/src/typedef.rs index aa2dbcbe613..c60ad9382d4 100644 --- a/pyre/pyre-interpreter/src/typedef.rs +++ b/pyre/pyre-interpreter/src/typedef.rs @@ -2833,7 +2833,9 @@ fn module_descr_init(args: &[PyObjectRef]) -> Result( // when the name resolves there. Requires the live frame operand. // The builtins fallback needs the module `pick_builtin(w_globals)` picks // (`frame.get_builtin()`). A live frame supplies it directly and also lets - // us double-check the name is absent from the frame's AUTHORITATIVE globals + // us double-check the operand against the frame's AUTHORITATIVE globals // — `bh_load_global_fn` re-resolves the globals it consults from the LIVE // frame (`frame.get_w_globals()` when the frame owns `w_code`, else the // code's bound globals) and IGNORES the `namespace_ptr` operand. The - // `ns_ptr` hint usually equals that live dict, but a present name there must - // resolve from globals (residual), not the builtin, or the fold would be - // wrong. An INLINED callee has no materialised frame (`frame_ptr == 0`, its + // `ns_ptr` hint usually equals that live dict; when it does not, nothing + // here can prove what the residual would read, so decline. + // An INLINED callee has no materialised frame (`frame_ptr == 0`, its // `portal_frame_reg` unseeded); derive the builtin module from the concrete // globals' `__builtins__` cell instead — the same object `pick_builtin` // resolves (baseobjspace.rs:9716) and the one the interpreter fallback would @@ -13157,10 +13157,15 @@ pub(crate) fn try_walker_load_global_cell_fold( pyre_interpreter::w_code_get_w_globals(w_code_ptr as pyre_object::PyObjectRef) } }; - if !live_globals.is_null() - && live_globals as usize != w_globals as usize - && crate::state::module_dict_cell_slot_direct(live_globals, &name).is_some() - { + // Only the SAME dict makes the absence provable. `module_dict_cell_slot_direct` + // answers `None` both for a name that is absent and for a dict it cannot + // read at all — a plain dict, or a module dict that ran + // `switch_to_object_strategy` — so on a different dict its `None` says + // nothing. Guard (a) below pins `w_globals`' version, which watches the + // wrong dict in that case, and the residual it replaces resolves + // `live_globals`; a name present there would read the builtin instead of + // the global. + if live_globals.is_null() || live_globals as usize != w_globals as usize { return Ok(false); } frame.get_builtin() diff --git a/pyre/pyre-object/src/module.rs b/pyre/pyre-object/src/module.rs index af9882805c7..e8e3bafb542 100644 --- a/pyre/pyre-object/src/module.rs +++ b/pyre/pyre-object/src/module.rs @@ -5,6 +5,7 @@ #![allow(unsafe_op_in_unsafe_fn)] use crate::pyobject::*; +use rustpython_wtf8::{Wtf8, Wtf8Buf}; /// Python module object. /// @@ -20,8 +21,10 @@ use crate::pyobject::*; #[repr(C)] pub struct Module { pub ob_header: PyObject, - /// Heap-allocated module name string. - pub name: *mut String, + /// Heap-allocated module name. WTF-8 rather than a Rust `String`: a name + /// reaches here straight from `module.__init__`, and surrogateescape + /// decoding of an undecodable filename puts a lone surrogate in it. + pub name: *mut Wtf8Buf, /// Authoritative dict object (`PyPy module.w_dict`). Always non-null /// after construction. pub w_dict: PyObjectRef, @@ -81,7 +84,7 @@ fn module_value(name: &str) -> Module { // `w_module_dict_new`; `pypy/objspace/std/celldict.py` strategy semantics // (`get_global_cache`, `invalidate_caches`, // `switch_to_object_strategy`) cover the module surface. - let name_box = crate::lltype::malloc_raw(name.to_string()); + let name_box = crate::lltype::malloc_raw(Wtf8Buf::from_string(name.to_string())); let w_dict = crate::dictmultiobject::w_module_dict_new(); if !name.is_empty() { unsafe { @@ -175,7 +178,7 @@ fn module_aliasing_dict_value(name: &str, w_dict_object: PyObjectRef) -> Module ); } } - let name = crate::lltype::malloc_raw(name.to_string()); + let name = crate::lltype::malloc_raw(Wtf8Buf::from_string(name.to_string())); Module { ob_header: PyObject { ob_type: &MODULE_TYPE as *const PyType, @@ -190,7 +193,7 @@ fn module_aliasing_dict_value(name: &str, w_dict_object: PyObjectRef) -> Module /// /// # Safety /// `obj` must point to a valid `Module`. -pub unsafe fn w_module_get_name(obj: PyObjectRef) -> &'static str { +pub unsafe fn w_module_get_name(obj: PyObjectRef) -> &'static Wtf8 { let module = &*(obj as *const Module); &*module.name } @@ -202,10 +205,10 @@ pub unsafe fn w_module_get_name(obj: PyObjectRef) -> &'static str { /// /// # Safety /// `obj` must point to a valid `Module`. -pub unsafe fn w_module_set_name(obj: PyObjectRef, name: &str) { +pub unsafe fn w_module_set_name(obj: PyObjectRef, name: &Wtf8) { let module = &mut *(obj as *mut Module); let old = module.name; - module.name = crate::lltype::malloc_raw(name.to_string()); + module.name = crate::lltype::malloc_raw(name.to_owned()); if !old.is_null() { drop(Box::from_raw(old)); } @@ -270,7 +273,7 @@ mod tests { unsafe { assert!(is_module(obj)); assert!(!is_int(obj)); - assert_eq!(w_module_get_name(obj), "test_mod"); + assert_eq!(w_module_get_name(obj), Wtf8::new("test_mod")); } } } diff --git a/pyre/pyrex/src/lib.rs b/pyre/pyrex/src/lib.rs index 0025c32f1fc..37dd161593c 100644 --- a/pyre/pyrex/src/lib.rs +++ b/pyre/pyrex/src/lib.rs @@ -1195,8 +1195,12 @@ fn release_frees_nothing(value: pyre_object::PyObjectRef) -> bool { // seeds it. An anonymous module proves nothing about reachability, // so it takes the collecting path rather than a `sys.modules[""]` // lookup that only an adversarial program could satisfy. + // A name carrying a lone surrogate cannot key the `&str` lookup, so + // it answers `false` and takes the collecting path. let name = pyre_object::w_module_get_name(value); - return !name.is_empty() && importing::get_sys_module(name).is_some_and(|m| m == value); + return name.as_str().is_ok_and(|name| { + !name.is_empty() && importing::get_sys_module(name).is_some_and(|m| m == value) + }); } } false @@ -1217,8 +1221,25 @@ fn clear_shutdown_module_name(dict: pyre_object::PyObjectRef, name: &rustpython_ } } -/// `_PyModule_ClearDict`: clear string-keyed module globals in two name passes. -fn clear_shutdown_module_dict(dict: pyre_object::PyObjectRef) { +/// Which of `_PyModule_ClearDict`'s two name passes to run. +/// +/// They are separated because rebinding a name to `None` frees nothing on its +/// own here: without refcounting a finalizer runs only from a collection, so +/// the passes are ordering-inert unless one is swept between them. The caller +/// sweeps once for the whole walk rather than once per module. +#[derive(Clone, Copy)] +enum ShutdownClearPass { + /// "clear only names starting with a single underscore", so that a + /// finalizer released here still reads its module's public globals. + PrivateNames, + /// "clear all names except for `__builtins__`". + RemainingNames, +} + +/// `_PyModule_ClearDict`: rebind the string-keyed module globals one pass +/// selects. A non-string key is left alone, as upstream leaves it — the value +/// under it is released by the collection that follows the whole walk. +fn clear_shutdown_module_dict(dict: pyre_object::PyObjectRef, pass: ShutdownClearPass) { if dict.is_null() { return; } @@ -1227,12 +1248,11 @@ fn clear_shutdown_module_dict(dict: pyre_object::PyObjectRef) { .map(|(name, _)| name) .collect(); for name in &keys { - if shutdown_module_private_name(name) { - clear_shutdown_module_name(dict, name); - } - } - for name in &keys { - if name.as_bytes() != b"__builtins__" { + let selected = match pass { + ShutdownClearPass::PrivateNames => shutdown_module_private_name(name), + ShutdownClearPass::RemainingNames => name.as_bytes() != b"__builtins__", + }; + if selected { clear_shutdown_module_name(dict, name); } } @@ -1261,28 +1281,33 @@ fn clear_shutdown_modules( pyre_object::gc_roots::pin_root(module); } collect_and_run_finalizers(ec_ptr); - for index in (0..names.len()).rev() { - let module = pyre_object::gc_roots::shadow_stack_get(roots_start + index); - let is_core_module = sys_module_slot.is_some_and(|slot| { - module == pyre_object::gc_roots::shadow_stack_get(roots_start + slot) - }) || builtins_module_slot.is_some_and(|slot| { - module == pyre_object::gc_roots::shadow_stack_get(roots_start + slot) - }); - if is_core_module { - continue; - } - if module.is_null() || !unsafe { pyre_object::is_module(module) } { - continue; + let clear_pass = |pass| { + for index in (0..names.len()).rev() { + let module = pyre_object::gc_roots::shadow_stack_get(roots_start + index); + let is_core_module = sys_module_slot.is_some_and(|slot| { + module == pyre_object::gc_roots::shadow_stack_get(roots_start + slot) + }) || builtins_module_slot.is_some_and(|slot| { + module == pyre_object::gc_roots::shadow_stack_get(roots_start + slot) + }); + if is_core_module { + continue; + } + if module.is_null() || !unsafe { pyre_object::is_module(module) } { + continue; + } + let dict = unsafe { pyre_object::w_module_get_w_dict(module) }; + clear_shutdown_module_dict(dict, pass); } - let dict = unsafe { pyre_object::w_module_get_w_dict(module) }; - clear_shutdown_module_dict(dict); - } - // One collection for the whole walk, not one per module. `finalize_modules` - // clears the module dictionaries and lets refcounting release what they - // held; a sweep per module buys no ordering here, because a finalizer that - // reads a global reaches its own already-cleared namespace either way, and - // it costs a full mark-and-sweep for each of the ~100 modules a bare - // `import unittest` loads. + }; + // Both passes run over the whole walk before either sweeps, rather than + // both passes per module. The sweep between them is what makes the + // private-name pass mean anything: `_obj.__del__` runs while its module's + // public globals still hold their values. A sweep per module would give + // the same ordering and cost a full mark-and-sweep for each of the ~100 + // modules a bare `import unittest` loads. + clear_pass(ShutdownClearPass::PrivateNames); + collect_and_run_finalizers(ec_ptr); + clear_pass(ShutdownClearPass::RemainingNames); collect_and_run_finalizers(ec_ptr); } @@ -1349,6 +1374,12 @@ fn finalize_runtime(canonical: pyre_object::PyObjectRef, ec_ptr: *const PyExecut collect_and_run_finalizers(ec_ptr); let shutdown_modules = pyre_interpreter::importing::release_sys_modules_for_shutdown(); clear_shutdown_modules(shutdown_modules, ec_ptr); + // The walk pins every module for its whole length, so none of its own + // sweeps can reach what a module dict is the last holder of — a value under + // a non-string key, which neither name pass rebinds. Its roots are gone by + // here, so this one does, and teardown stops leaving those finalizers + // unrun. It is the last collection before the process exits. + collect_and_run_finalizers(ec_ptr); } /// Resolve a pending `SystemExit`'s status, then finalize and exit with it. diff --git a/tools/ubuntu24-arm64-repro/Dockerfile b/tools/ubuntu24-arm64-repro/Dockerfile new file mode 100644 index 00000000000..e66ca973f18 --- /dev/null +++ b/tools/ubuntu24-arm64-repro/Dockerfile @@ -0,0 +1,29 @@ +FROM --platform=linux/arm64 ubuntu:24.04 + +ENV DEBIAN_FRONTEND=noninteractive +ENV CARGO_HOME=/cargo +ENV RUSTUP_HOME=/rustup +ENV PYRE_SHARED_BUILD=/workspace/.pyre-build +ENV PATH=/cargo/bin:/workspace/.pyre-build/charon/linux-aarch64:$PATH +ENV CARGO_BUILD_JOBS=2 + +RUN apt-get update \ + && apt-get install -y --no-install-recommends \ + bash \ + build-essential \ + ca-certificates \ + curl \ + file \ + git \ + libffi-dev \ + pkg-config \ + procps \ + python3 \ + && rm -rf /var/lib/apt/lists/* + +RUN curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs \ + | sh -s -- -y --profile minimal --default-toolchain stable + +WORKDIR /workspace/pyre + +CMD ["/bin/bash"] diff --git a/tools/ubuntu24-arm64-repro/README.md b/tools/ubuntu24-arm64-repro/README.md new file mode 100644 index 00000000000..0b5fc63497e --- /dev/null +++ b/tools/ubuntu24-arm64-repro/README.md @@ -0,0 +1,43 @@ +# Ubuntu 24.04 arm64 repro image + +This image fixes the Linux userspace used to reproduce Linux-only failures on +an Apple silicon host without Rosetta, so the build runs at native speed. Use +`tools/ubuntu24-amd64-repro` instead when the failure is arch-specific or when +confirming against the amd64 CI runner. + +Like the amd64 image, it does not copy the repository in; mount the worktree at +runtime so local edits and build artifacts are visible. + +Build with Apple `container`: + +```bash +container build --platform linux/arm64 -m 8G -c 4 --progress plain \ + -t pyre-ubuntu24-arm64-repro \ + tools/ubuntu24-arm64-repro +``` + +Run from the repository root: + +```bash +mkdir -p "$(dirname "$PWD")/.pyre-build" +container run --rm --platform linux/arm64 -m 20G -c 4 \ + --mount type=bind,source="$(pwd)",target=/workspace/pyre \ + --mount type=bind,source="$(dirname "$PWD")/.pyre-build",target=/workspace/.pyre-build \ + pyre-ubuntu24-arm64-repro +``` + +The second mount keeps the shared Charon cache outside the worktree and reuses +it across sibling worktrees and container runs. + +Inside the container: + +```bash +scripts/install-charon.sh +python3 scripts/extract-llbc.py +cargo build --release -p pyrex --bin pyre-dynasm \ + --no-default-features --features dynasm +python3 pyre/cpython_tests/run.py --binary ./target/release/pyre-dynasm +``` + +The extraction refuses to stamp its artefacts when the tree moves mid-build, so +keep other sessions off the worktree while it runs.