diff --git a/majit/majit-metainterp/src/warmstate.rs b/majit/majit-metainterp/src/warmstate.rs index 1afe0b6ba54..59b814de137 100644 --- a/majit/majit-metainterp/src/warmstate.rs +++ b/majit/majit-metainterp/src/warmstate.rs @@ -707,7 +707,7 @@ impl WarmEnterState { self.counter.tick(bucket, self.increment_threshold) } - pub fn maybe_compile(&mut self, cell_key: u64) -> HotResult { + pub fn maybe_compile_decision(&mut self, cell_key: u64) -> HotResult { let mut cleanup_dead_token_cell = false; if let Some(cell) = self.cell_by_key(cell_key) { let has_procedure_token = cell.get_procedure_token().is_some(); @@ -723,7 +723,7 @@ impl WarmEnterState { } if self.should_start_dont_trace_here_trace(cell_key, flags, has_seen_a_procedure_token) { - return self.start_tracing_cell(cell_key); + return HotResult::StartTracing; } // A JC_DONT_TRACE_HERE cell declines here, except when it once saw a // procedure token that has since been invalidated — that dead entry @@ -752,7 +752,25 @@ impl WarmEnterState { return HotResult::NotHot; } - self.start_tracing_cell(cell_key) + HotResult::StartTracing + } + + /// [`Self::maybe_compile_decision`] followed by the mark that + /// `warmstate.py:441` makes once the decision is taken: + /// `cell.flags |= JC_TRACING | JC_TRACING_OCCURRED`, immediately before + /// the `try:` that runs the trace. + /// + /// Callers that go straight from the answer into `setup_tracing` want + /// this form. A caller that instead delegates to a tracing entry point + /// which makes its own decision — `MetaInterp::bound_reached` reaches the + /// same cell through [`Self::force_start_tracing_for_key`] — must take the + /// decision alone, or that second call reads the mark this one left and + /// answers `AlreadyTracing` for the trace it was asked to start. + pub fn maybe_compile(&mut self, cell_key: u64) -> HotResult { + match self.maybe_compile_decision(cell_key) { + HotResult::StartTracing => self.start_tracing_cell(cell_key), + other => other, + } } /// warmstate.py:446-511 `WarmEnterState.maybe_compile_and_run` — diff --git a/pyre/bench/synth/for_iter_direct_store_double.cranelift.jitstats b/pyre/bench/synth/for_iter_direct_store_double.cranelift.jitstats index 6e2effccd01..520ee917560 100644 --- a/pyre/bench/synth/for_iter_direct_store_double.cranelift.jitstats +++ b/pyre/bench/synth/for_iter_direct_store_double.cranelift.jitstats @@ -3,12 +3,13 @@ descr_set_absent=0 descr_set_ambiguous=0 descr_set_stale_absent=0 fbw_blackhole_adopted_multi_frame=0 -fbw_blackhole_adopted_single_frame=4 +fbw_blackhole_adopted_single_frame=8 fbw_rolled_back_with_effects=0 fbw_store_journal_rollback_failed=0 field_pos_attached_misplaced=0 field_pos_spec_misplaced=0 guard_failures=0 internal_compile_panics=0 -loops_aborted=4 +loops_aborted=8 loops_compiled=1 +retraces_compiled=0 diff --git a/pyre/bench/synth/for_iter_direct_store_double.dynasm.jitstats b/pyre/bench/synth/for_iter_direct_store_double.dynasm.jitstats index 6e2effccd01..520ee917560 100644 --- a/pyre/bench/synth/for_iter_direct_store_double.dynasm.jitstats +++ b/pyre/bench/synth/for_iter_direct_store_double.dynasm.jitstats @@ -3,12 +3,13 @@ descr_set_absent=0 descr_set_ambiguous=0 descr_set_stale_absent=0 fbw_blackhole_adopted_multi_frame=0 -fbw_blackhole_adopted_single_frame=4 +fbw_blackhole_adopted_single_frame=8 fbw_rolled_back_with_effects=0 fbw_store_journal_rollback_failed=0 field_pos_attached_misplaced=0 field_pos_spec_misplaced=0 guard_failures=0 internal_compile_panics=0 -loops_aborted=4 +loops_aborted=8 loops_compiled=1 +retraces_compiled=0 diff --git a/pyre/bench/synth/for_iter_direct_store_double.wasm.jitstats b/pyre/bench/synth/for_iter_direct_store_double.wasm.jitstats index a5c33ef7fe5..655fb3b86e4 100644 --- a/pyre/bench/synth/for_iter_direct_store_double.wasm.jitstats +++ b/pyre/bench/synth/for_iter_direct_store_double.wasm.jitstats @@ -3,12 +3,13 @@ descr_set_absent=0 descr_set_ambiguous=0 descr_set_stale_absent=0 fbw_blackhole_adopted_multi_frame=0 -fbw_blackhole_adopted_single_frame=2 +fbw_blackhole_adopted_single_frame=6 fbw_rolled_back_with_effects=0 fbw_store_journal_rollback_failed=0 field_pos_attached_misplaced=0 field_pos_spec_misplaced=0 guard_failures=0 internal_compile_panics=0 -loops_aborted=2 +loops_aborted=6 loops_compiled=0 +retraces_compiled=0 diff --git a/pyre/bench/synth/getframe_force_cancel_journal.dynasm.jitstats b/pyre/bench/synth/getframe_force_cancel_journal.dynasm.jitstats index 77817899ecb..a893551fa0c 100644 --- a/pyre/bench/synth/getframe_force_cancel_journal.dynasm.jitstats +++ b/pyre/bench/synth/getframe_force_cancel_journal.dynasm.jitstats @@ -12,3 +12,4 @@ guard_failures=0 internal_compile_panics=0 loops_aborted=5 loops_compiled=0 +retraces_compiled=0 diff --git a/pyre/bench/synth/getframe_inline_subwalk_multiframe.cranelift.jitstats b/pyre/bench/synth/getframe_inline_subwalk_multiframe.cranelift.jitstats index 41ac424fcc4..35c7c669f1c 100644 --- a/pyre/bench/synth/getframe_inline_subwalk_multiframe.cranelift.jitstats +++ b/pyre/bench/synth/getframe_inline_subwalk_multiframe.cranelift.jitstats @@ -2,13 +2,14 @@ bridges_compiled=0 descr_set_absent=0 descr_set_ambiguous=0 descr_set_stale_absent=0 -fbw_blackhole_adopted_multi_frame=5 -fbw_blackhole_adopted_single_frame=9 +fbw_blackhole_adopted_multi_frame=0 +fbw_blackhole_adopted_single_frame=0 fbw_rolled_back_with_effects=0 fbw_store_journal_rollback_failed=0 field_pos_attached_misplaced=0 field_pos_spec_misplaced=0 -guard_failures=0 +guard_failures=8948 internal_compile_panics=0 -loops_aborted=15 -loops_compiled=0 +loops_aborted=1 +loops_compiled=2 +retraces_compiled=0 diff --git a/pyre/bench/synth/getframe_inline_subwalk_multiframe.dynasm.jitstats b/pyre/bench/synth/getframe_inline_subwalk_multiframe.dynasm.jitstats index 41ac424fcc4..35c7c669f1c 100644 --- a/pyre/bench/synth/getframe_inline_subwalk_multiframe.dynasm.jitstats +++ b/pyre/bench/synth/getframe_inline_subwalk_multiframe.dynasm.jitstats @@ -2,13 +2,14 @@ bridges_compiled=0 descr_set_absent=0 descr_set_ambiguous=0 descr_set_stale_absent=0 -fbw_blackhole_adopted_multi_frame=5 -fbw_blackhole_adopted_single_frame=9 +fbw_blackhole_adopted_multi_frame=0 +fbw_blackhole_adopted_single_frame=0 fbw_rolled_back_with_effects=0 fbw_store_journal_rollback_failed=0 field_pos_attached_misplaced=0 field_pos_spec_misplaced=0 -guard_failures=0 +guard_failures=8948 internal_compile_panics=0 -loops_aborted=15 -loops_compiled=0 +loops_aborted=1 +loops_compiled=2 +retraces_compiled=0 diff --git a/pyre/bench/synth/getframe_inline_subwalk_multiframe.py b/pyre/bench/synth/getframe_inline_subwalk_multiframe.py index e0e106a27bb..ee46c88cdb7 100644 --- a/pyre/bench/synth/getframe_inline_subwalk_multiframe.py +++ b/pyre/bench/synth/getframe_inline_subwalk_multiframe.py @@ -1,61 +1,45 @@ -# No `max-pypy-ratio`: this fixture compiles no loop -- its jitstats record -# `loops_compiled=0` -- so a pypy ratio compares two interpreters' startup -# rather than any generated code, and reads whatever the host's process -# spawn cost happens to be that run. The jitstats baselines gate it. -# Coverage for the multi-frame blackhole build path: an INLINED callee that -# forces an outer frame through `sys._getframe(2)`. +# Frame-identity regression for two `_getframe` operations executed from an +# inlined `leaf` MIFrame. The depth-zero lookup must return `leaf`'s own red +# frame, and its `f_locals` proxy must retain that frame so reading `"x"` +# observes the live callee local. The positive-depth lookup must start from the +# same callee frame and reach `main`, where `"base"` is stored. # -# The walker executes a residual call concretely, so that level gets a real -# frame from the interpreter's own call sequence; an inline push did not run -# that sequence, so its level had none. A force fired from the inlined body -# therefore built a frame chain that mixed the two, rooted at the intermediate -# residual frame rather than the walked frame, and -# `try_adopt_multi_frame_blackhole`'s chain-root identity gate declined it. What -# that decline wanted was the `jit.virtual_ref` emit at the inline push -# (`executioncontext.py:89`); `walker_ec_enter` / `walker_ec_leave` landed it, so -# the chain-root gate no longer fires here, and with every level resumable from -# the concrete frame it owns, the chain is adopted rather than replayed. This fixture pins the -# result across that adopt. +# Collapsing either lookup onto the portal frame changes the namespace: the +# first read loses `x`, while the second lands on the module frame and loses +# `base`. Residualizing the depth-zero lookup instead forces the published +# callee during tracing and prevents this loop from compiling. The specialized +# path therefore records the callee frame and creates its `FrameLocalsProxy` +# without forcing the outer standard virtualizable; positive-depth lookup stays +# on the established virtual-reference walk until its per-level lowering is +# available. # -# The build wants an escape whose forced frame is the one the inline sub-walk -# PUBLISHED, so the chain has a residual level under the walked frame and an -# inlined level under that. `sys._getframe(0).f_locals` in `leaf` is that -# escape: the depth-0 call names `leaf`'s own frame, the arm declines inside a -# sub-walk, and the getset forces the published frame. +# A real PyPy run compiles one loop with no bridges, forcings, virtualizable +# forcings, or aborts. Pyre must print the same value, compile one loop, and +# report no escape abort or frame-blackhole adoption for this fixture. # -# The `sys._getframe(2)` read below is the OUTPUT guard and is not what reaches -# the build -- `getframe` forces only the frame it returns, and two levels up is -# `main`'s, i.e. the portal, which escapes on the ordinary portal path. Without -# the depth-0 line the file still answers correctly and stops building a -# multi-frame image at all (measured: `fbw_blackhole_adopted_multi_frame` 5 -> -# 0). In the historical gate-enabled sweep with `PYRE_FBW_DEBUG_ABORT=1`, 0 of -# 310 fixtures reached `BUILT multi-frame`. -# -# The multi-frame image is built unconditionally when the latch conditions hold, -# so this is both an output guard and build-path coverage: 5 builds, 5 adopts, -# and zero declines of any kind (`PYRE_FBW_DEBUG_ABORT=1` prints every tally; -# the other 5 escapes in the run have `inline_subwalk=false` and take the -# single-frame arm, which adopts too). -# -# What the decline used to hold back, measured by lifting it before the -# execution-context push landed: the resumed chain shifted every -# `sys._getframe(n)` up exactly one level, so the read below landed on the -# module frame and raised `KeyError: 'base'`. Variants of this shape that cannot -# raise returned a wrong number instead, silently -- `f_locals.get("base", -1)` -# scored -1 for 7, `len(f_locals)` scored the module globals' 12 for this -# frame's 3, `len(f_code.co_name)` scored ``'s 8 for a 9-character -# caller name. That is the failure mode this fixture still guards: every outcome -# arm was wrong, not only the one carrying a resume coordinate. -# -# Carries only a generous `# pyre-check: max-pypy-ratio=` tripwire (top of -# file), not a tight perf gate: this guards an output, and the forcing read -# makes it a poor perf subject, so the ratio is a gross-regression guard. +# The positive-depth clause has a measured cost, and the recorded counters +# encode it rather than endorse it. `leaf`'s `_getframe(2)` reaches `main`'s +# frame, which is the virtualizable of `main`'s compiled loop, and materializes +# it. `mid(i)` stays a residual `CallMayForce` in that loop, so the +# `GuardNotForced` behind it fails on every machine-code entry: `MAJIT_STATS=1` +# reports `mc_entered` equal to `guard_failures`, with `back_edge_polls=0` and +# `bridges_compiled=0`, so the compiled loop leaves for the blackhole before it +# ever crosses its back edge. The attribution is a one-run check -- a `leaf` +# holding only the depth-zero read records `guard_failures=1`, one holding only +# the `_getframe(2)` read reproduces the full count. The recorded +# `guard_failures` is therefore a known shortfall of the positive-depth walk, +# not a number to preserve; it is expected to fall once that walk gets its +# per-level lowering. `loops_compiled` counts two traces here: `main`'s loop, +# and a linear `Finish` trace for the inlined `mid` -> `leaf` chain, whose own +# four `GuardNotForced`s never fail. The wasm baseline records +# `guard_failures=1` instead because that backend always materializes the +# virtualizable, which makes `GuardNotForced` a no-op there. import sys def leaf(x): - sys._getframe(0).f_locals # noqa: B018 — the read itself is the force under test - return sys._getframe(2).f_locals["base"] + x + own_x = sys._getframe(0).f_locals["x"] + return sys._getframe(2).f_locals["base"] + own_x def mid(x): diff --git a/pyre/bench/synth/getframe_inline_subwalk_multiframe.wasm.jitstats b/pyre/bench/synth/getframe_inline_subwalk_multiframe.wasm.jitstats index 41ac424fcc4..910ee008dba 100644 --- a/pyre/bench/synth/getframe_inline_subwalk_multiframe.wasm.jitstats +++ b/pyre/bench/synth/getframe_inline_subwalk_multiframe.wasm.jitstats @@ -2,13 +2,14 @@ bridges_compiled=0 descr_set_absent=0 descr_set_ambiguous=0 descr_set_stale_absent=0 -fbw_blackhole_adopted_multi_frame=5 -fbw_blackhole_adopted_single_frame=9 +fbw_blackhole_adopted_multi_frame=0 +fbw_blackhole_adopted_single_frame=0 fbw_rolled_back_with_effects=0 fbw_store_journal_rollback_failed=0 field_pos_attached_misplaced=0 field_pos_spec_misplaced=0 -guard_failures=0 +guard_failures=1 internal_compile_panics=0 -loops_aborted=15 -loops_compiled=0 +loops_aborted=1 +loops_compiled=2 +retraces_compiled=0 diff --git a/pyre/bench/synth/getframe_inlined_callee_own_frame.dynasm.jitstats b/pyre/bench/synth/getframe_inlined_callee_own_frame.dynasm.jitstats index afd3c3c6425..910ee008dba 100644 --- a/pyre/bench/synth/getframe_inlined_callee_own_frame.dynasm.jitstats +++ b/pyre/bench/synth/getframe_inlined_callee_own_frame.dynasm.jitstats @@ -12,3 +12,4 @@ guard_failures=1 internal_compile_panics=0 loops_aborted=1 loops_compiled=2 +retraces_compiled=0 diff --git a/pyre/bench/synth/getframe_root_loop_force_blackhole_crn_declined.dynasm.jitstats b/pyre/bench/synth/getframe_root_loop_force_blackhole_crn_declined.dynasm.jitstats index 77817899ecb..a893551fa0c 100644 --- a/pyre/bench/synth/getframe_root_loop_force_blackhole_crn_declined.dynasm.jitstats +++ b/pyre/bench/synth/getframe_root_loop_force_blackhole_crn_declined.dynasm.jitstats @@ -12,3 +12,4 @@ guard_failures=0 internal_compile_panics=0 loops_aborted=5 loops_compiled=0 +retraces_compiled=0 diff --git a/pyre/bench/synth/getframe_root_loop_force_blackhole_crn_nonidempotent_declined.dynasm.jitstats b/pyre/bench/synth/getframe_root_loop_force_blackhole_crn_nonidempotent_declined.dynasm.jitstats index 77817899ecb..a893551fa0c 100644 --- a/pyre/bench/synth/getframe_root_loop_force_blackhole_crn_nonidempotent_declined.dynasm.jitstats +++ b/pyre/bench/synth/getframe_root_loop_force_blackhole_crn_nonidempotent_declined.dynasm.jitstats @@ -12,3 +12,4 @@ guard_failures=0 internal_compile_panics=0 loops_aborted=5 loops_compiled=0 +retraces_compiled=0 diff --git a/pyre/bench/synth/getframe_root_loop_force_while_merge_declined.dynasm.jitstats b/pyre/bench/synth/getframe_root_loop_force_while_merge_declined.dynasm.jitstats index 77817899ecb..a893551fa0c 100644 --- a/pyre/bench/synth/getframe_root_loop_force_while_merge_declined.dynasm.jitstats +++ b/pyre/bench/synth/getframe_root_loop_force_while_merge_declined.dynasm.jitstats @@ -12,3 +12,4 @@ guard_failures=0 internal_compile_panics=0 loops_aborted=5 loops_compiled=0 +retraces_compiled=0 diff --git a/pyre/bench/synth/getframe_stored_fback_walk.dynasm.jitstats b/pyre/bench/synth/getframe_stored_fback_walk.dynasm.jitstats index e2c79ca9c9e..d5fcc1a2e97 100644 --- a/pyre/bench/synth/getframe_stored_fback_walk.dynasm.jitstats +++ b/pyre/bench/synth/getframe_stored_fback_walk.dynasm.jitstats @@ -12,3 +12,4 @@ guard_failures=0 internal_compile_panics=0 loops_aborted=5 loops_compiled=1 +retraces_compiled=0 diff --git a/pyre/bench/synth/getframe_while_caller_locals_across_subwalk.cranelift.jitstats b/pyre/bench/synth/getframe_while_caller_locals_across_subwalk.cranelift.jitstats index 2215bb8e77b..651a3eaf3e9 100644 --- a/pyre/bench/synth/getframe_while_caller_locals_across_subwalk.cranelift.jitstats +++ b/pyre/bench/synth/getframe_while_caller_locals_across_subwalk.cranelift.jitstats @@ -2,13 +2,14 @@ bridges_compiled=0 descr_set_absent=0 descr_set_ambiguous=0 descr_set_stale_absent=0 -fbw_blackhole_adopted_multi_frame=5 +fbw_blackhole_adopted_multi_frame=0 fbw_blackhole_adopted_single_frame=0 fbw_rolled_back_with_effects=0 fbw_store_journal_rollback_failed=0 field_pos_attached_misplaced=0 field_pos_spec_misplaced=0 -guard_failures=0 +guard_failures=1 internal_compile_panics=0 -loops_aborted=5 +loops_aborted=0 loops_compiled=1 +retraces_compiled=0 diff --git a/pyre/bench/synth/getframe_while_caller_locals_across_subwalk.dynasm.jitstats b/pyre/bench/synth/getframe_while_caller_locals_across_subwalk.dynasm.jitstats index 2215bb8e77b..651a3eaf3e9 100644 --- a/pyre/bench/synth/getframe_while_caller_locals_across_subwalk.dynasm.jitstats +++ b/pyre/bench/synth/getframe_while_caller_locals_across_subwalk.dynasm.jitstats @@ -2,13 +2,14 @@ bridges_compiled=0 descr_set_absent=0 descr_set_ambiguous=0 descr_set_stale_absent=0 -fbw_blackhole_adopted_multi_frame=5 +fbw_blackhole_adopted_multi_frame=0 fbw_blackhole_adopted_single_frame=0 fbw_rolled_back_with_effects=0 fbw_store_journal_rollback_failed=0 field_pos_attached_misplaced=0 field_pos_spec_misplaced=0 -guard_failures=0 +guard_failures=1 internal_compile_panics=0 -loops_aborted=5 +loops_aborted=0 loops_compiled=1 +retraces_compiled=0 diff --git a/pyre/bench/synth/getframe_while_caller_locals_across_subwalk.wasm.jitstats b/pyre/bench/synth/getframe_while_caller_locals_across_subwalk.wasm.jitstats index 2215bb8e77b..651a3eaf3e9 100644 --- a/pyre/bench/synth/getframe_while_caller_locals_across_subwalk.wasm.jitstats +++ b/pyre/bench/synth/getframe_while_caller_locals_across_subwalk.wasm.jitstats @@ -2,13 +2,14 @@ bridges_compiled=0 descr_set_absent=0 descr_set_ambiguous=0 descr_set_stale_absent=0 -fbw_blackhole_adopted_multi_frame=5 +fbw_blackhole_adopted_multi_frame=0 fbw_blackhole_adopted_single_frame=0 fbw_rolled_back_with_effects=0 fbw_store_journal_rollback_failed=0 field_pos_attached_misplaced=0 field_pos_spec_misplaced=0 -guard_failures=0 +guard_failures=1 internal_compile_panics=0 -loops_aborted=5 +loops_aborted=0 loops_compiled=1 +retraces_compiled=0 diff --git a/pyre/bench/synth/getframe_while_captured_frame_outlives_call.cranelift.jitstats b/pyre/bench/synth/getframe_while_captured_frame_outlives_call.cranelift.jitstats index 2215bb8e77b..651a3eaf3e9 100644 --- a/pyre/bench/synth/getframe_while_captured_frame_outlives_call.cranelift.jitstats +++ b/pyre/bench/synth/getframe_while_captured_frame_outlives_call.cranelift.jitstats @@ -2,13 +2,14 @@ bridges_compiled=0 descr_set_absent=0 descr_set_ambiguous=0 descr_set_stale_absent=0 -fbw_blackhole_adopted_multi_frame=5 +fbw_blackhole_adopted_multi_frame=0 fbw_blackhole_adopted_single_frame=0 fbw_rolled_back_with_effects=0 fbw_store_journal_rollback_failed=0 field_pos_attached_misplaced=0 field_pos_spec_misplaced=0 -guard_failures=0 +guard_failures=1 internal_compile_panics=0 -loops_aborted=5 +loops_aborted=0 loops_compiled=1 +retraces_compiled=0 diff --git a/pyre/bench/synth/getframe_while_captured_frame_outlives_call.dynasm.jitstats b/pyre/bench/synth/getframe_while_captured_frame_outlives_call.dynasm.jitstats index 2215bb8e77b..651a3eaf3e9 100644 --- a/pyre/bench/synth/getframe_while_captured_frame_outlives_call.dynasm.jitstats +++ b/pyre/bench/synth/getframe_while_captured_frame_outlives_call.dynasm.jitstats @@ -2,13 +2,14 @@ bridges_compiled=0 descr_set_absent=0 descr_set_ambiguous=0 descr_set_stale_absent=0 -fbw_blackhole_adopted_multi_frame=5 +fbw_blackhole_adopted_multi_frame=0 fbw_blackhole_adopted_single_frame=0 fbw_rolled_back_with_effects=0 fbw_store_journal_rollback_failed=0 field_pos_attached_misplaced=0 field_pos_spec_misplaced=0 -guard_failures=0 +guard_failures=1 internal_compile_panics=0 -loops_aborted=5 +loops_aborted=0 loops_compiled=1 +retraces_compiled=0 diff --git a/pyre/bench/synth/getframe_while_captured_frame_outlives_call.wasm.jitstats b/pyre/bench/synth/getframe_while_captured_frame_outlives_call.wasm.jitstats index 2215bb8e77b..651a3eaf3e9 100644 --- a/pyre/bench/synth/getframe_while_captured_frame_outlives_call.wasm.jitstats +++ b/pyre/bench/synth/getframe_while_captured_frame_outlives_call.wasm.jitstats @@ -2,13 +2,14 @@ bridges_compiled=0 descr_set_absent=0 descr_set_ambiguous=0 descr_set_stale_absent=0 -fbw_blackhole_adopted_multi_frame=5 +fbw_blackhole_adopted_multi_frame=0 fbw_blackhole_adopted_single_frame=0 fbw_rolled_back_with_effects=0 fbw_store_journal_rollback_failed=0 field_pos_attached_misplaced=0 field_pos_spec_misplaced=0 -guard_failures=0 +guard_failures=1 internal_compile_panics=0 -loops_aborted=5 +loops_aborted=0 loops_compiled=1 +retraces_compiled=0 diff --git a/pyre/bench/synth/getframe_while_escaping_read_frame_identity.cranelift.jitstats b/pyre/bench/synth/getframe_while_escaping_read_frame_identity.cranelift.jitstats index 9758b023e10..15e4b56e6d8 100644 --- a/pyre/bench/synth/getframe_while_escaping_read_frame_identity.cranelift.jitstats +++ b/pyre/bench/synth/getframe_while_escaping_read_frame_identity.cranelift.jitstats @@ -2,14 +2,14 @@ bridges_compiled=0 descr_set_absent=0 descr_set_ambiguous=0 descr_set_stale_absent=0 -fbw_blackhole_adopted_multi_frame=10 +fbw_blackhole_adopted_multi_frame=5 fbw_blackhole_adopted_single_frame=0 fbw_rolled_back_with_effects=0 fbw_store_journal_rollback_failed=0 field_pos_attached_misplaced=0 field_pos_spec_misplaced=0 -guard_failures=0 +guard_failures=1 internal_compile_panics=0 -loops_aborted=10 +loops_aborted=5 loops_compiled=2 retraces_compiled=0 diff --git a/pyre/bench/synth/getframe_while_escaping_read_frame_identity.dynasm.jitstats b/pyre/bench/synth/getframe_while_escaping_read_frame_identity.dynasm.jitstats index 9758b023e10..15e4b56e6d8 100644 --- a/pyre/bench/synth/getframe_while_escaping_read_frame_identity.dynasm.jitstats +++ b/pyre/bench/synth/getframe_while_escaping_read_frame_identity.dynasm.jitstats @@ -2,14 +2,14 @@ bridges_compiled=0 descr_set_absent=0 descr_set_ambiguous=0 descr_set_stale_absent=0 -fbw_blackhole_adopted_multi_frame=10 +fbw_blackhole_adopted_multi_frame=5 fbw_blackhole_adopted_single_frame=0 fbw_rolled_back_with_effects=0 fbw_store_journal_rollback_failed=0 field_pos_attached_misplaced=0 field_pos_spec_misplaced=0 -guard_failures=0 +guard_failures=1 internal_compile_panics=0 -loops_aborted=10 +loops_aborted=5 loops_compiled=2 retraces_compiled=0 diff --git a/pyre/bench/synth/getframe_while_escaping_read_frame_identity.wasm.jitstats b/pyre/bench/synth/getframe_while_escaping_read_frame_identity.wasm.jitstats index 9758b023e10..15e4b56e6d8 100644 --- a/pyre/bench/synth/getframe_while_escaping_read_frame_identity.wasm.jitstats +++ b/pyre/bench/synth/getframe_while_escaping_read_frame_identity.wasm.jitstats @@ -2,14 +2,14 @@ bridges_compiled=0 descr_set_absent=0 descr_set_ambiguous=0 descr_set_stale_absent=0 -fbw_blackhole_adopted_multi_frame=10 +fbw_blackhole_adopted_multi_frame=5 fbw_blackhole_adopted_single_frame=0 fbw_rolled_back_with_effects=0 fbw_store_journal_rollback_failed=0 field_pos_attached_misplaced=0 field_pos_spec_misplaced=0 -guard_failures=0 +guard_failures=1 internal_compile_panics=0 -loops_aborted=10 +loops_aborted=5 loops_compiled=2 retraces_compiled=0 diff --git a/pyre/bench/synth/getframe_while_inlined_callee_subwalk.cranelift.jitstats b/pyre/bench/synth/getframe_while_inlined_callee_subwalk.cranelift.jitstats index 2215bb8e77b..651a3eaf3e9 100644 --- a/pyre/bench/synth/getframe_while_inlined_callee_subwalk.cranelift.jitstats +++ b/pyre/bench/synth/getframe_while_inlined_callee_subwalk.cranelift.jitstats @@ -2,13 +2,14 @@ bridges_compiled=0 descr_set_absent=0 descr_set_ambiguous=0 descr_set_stale_absent=0 -fbw_blackhole_adopted_multi_frame=5 +fbw_blackhole_adopted_multi_frame=0 fbw_blackhole_adopted_single_frame=0 fbw_rolled_back_with_effects=0 fbw_store_journal_rollback_failed=0 field_pos_attached_misplaced=0 field_pos_spec_misplaced=0 -guard_failures=0 +guard_failures=1 internal_compile_panics=0 -loops_aborted=5 +loops_aborted=0 loops_compiled=1 +retraces_compiled=0 diff --git a/pyre/bench/synth/getframe_while_inlined_callee_subwalk.dynasm.jitstats b/pyre/bench/synth/getframe_while_inlined_callee_subwalk.dynasm.jitstats index 2215bb8e77b..651a3eaf3e9 100644 --- a/pyre/bench/synth/getframe_while_inlined_callee_subwalk.dynasm.jitstats +++ b/pyre/bench/synth/getframe_while_inlined_callee_subwalk.dynasm.jitstats @@ -2,13 +2,14 @@ bridges_compiled=0 descr_set_absent=0 descr_set_ambiguous=0 descr_set_stale_absent=0 -fbw_blackhole_adopted_multi_frame=5 +fbw_blackhole_adopted_multi_frame=0 fbw_blackhole_adopted_single_frame=0 fbw_rolled_back_with_effects=0 fbw_store_journal_rollback_failed=0 field_pos_attached_misplaced=0 field_pos_spec_misplaced=0 -guard_failures=0 +guard_failures=1 internal_compile_panics=0 -loops_aborted=5 +loops_aborted=0 loops_compiled=1 +retraces_compiled=0 diff --git a/pyre/bench/synth/getframe_while_inlined_callee_subwalk.wasm.jitstats b/pyre/bench/synth/getframe_while_inlined_callee_subwalk.wasm.jitstats index 2215bb8e77b..651a3eaf3e9 100644 --- a/pyre/bench/synth/getframe_while_inlined_callee_subwalk.wasm.jitstats +++ b/pyre/bench/synth/getframe_while_inlined_callee_subwalk.wasm.jitstats @@ -2,13 +2,14 @@ bridges_compiled=0 descr_set_absent=0 descr_set_ambiguous=0 descr_set_stale_absent=0 -fbw_blackhole_adopted_multi_frame=5 +fbw_blackhole_adopted_multi_frame=0 fbw_blackhole_adopted_single_frame=0 fbw_rolled_back_with_effects=0 fbw_store_journal_rollback_failed=0 field_pos_attached_misplaced=0 field_pos_spec_misplaced=0 -guard_failures=0 +guard_failures=1 internal_compile_panics=0 -loops_aborted=5 +loops_aborted=0 loops_compiled=1 +retraces_compiled=0 diff --git a/pyre/bench/synth/getframe_while_subwalk_decline_shapes.cranelift.jitstats b/pyre/bench/synth/getframe_while_subwalk_decline_shapes.cranelift.jitstats index b5a8d73904a..c3bab7dbb4a 100644 --- a/pyre/bench/synth/getframe_while_subwalk_decline_shapes.cranelift.jitstats +++ b/pyre/bench/synth/getframe_while_subwalk_decline_shapes.cranelift.jitstats @@ -2,13 +2,14 @@ bridges_compiled=0 descr_set_absent=0 descr_set_ambiguous=0 descr_set_stale_absent=0 -fbw_blackhole_adopted_multi_frame=15 +fbw_blackhole_adopted_multi_frame=0 fbw_blackhole_adopted_single_frame=0 fbw_rolled_back_with_effects=0 fbw_store_journal_rollback_failed=0 field_pos_attached_misplaced=0 field_pos_spec_misplaced=0 -guard_failures=0 +guard_failures=2 internal_compile_panics=0 -loops_aborted=15 +loops_aborted=0 loops_compiled=2 +retraces_compiled=0 diff --git a/pyre/bench/synth/getframe_while_subwalk_decline_shapes.dynasm.jitstats b/pyre/bench/synth/getframe_while_subwalk_decline_shapes.dynasm.jitstats index b5a8d73904a..c3bab7dbb4a 100644 --- a/pyre/bench/synth/getframe_while_subwalk_decline_shapes.dynasm.jitstats +++ b/pyre/bench/synth/getframe_while_subwalk_decline_shapes.dynasm.jitstats @@ -2,13 +2,14 @@ bridges_compiled=0 descr_set_absent=0 descr_set_ambiguous=0 descr_set_stale_absent=0 -fbw_blackhole_adopted_multi_frame=15 +fbw_blackhole_adopted_multi_frame=0 fbw_blackhole_adopted_single_frame=0 fbw_rolled_back_with_effects=0 fbw_store_journal_rollback_failed=0 field_pos_attached_misplaced=0 field_pos_spec_misplaced=0 -guard_failures=0 +guard_failures=2 internal_compile_panics=0 -loops_aborted=15 +loops_aborted=0 loops_compiled=2 +retraces_compiled=0 diff --git a/pyre/bench/synth/getframe_while_subwalk_decline_shapes.wasm.jitstats b/pyre/bench/synth/getframe_while_subwalk_decline_shapes.wasm.jitstats index b5a8d73904a..c3bab7dbb4a 100644 --- a/pyre/bench/synth/getframe_while_subwalk_decline_shapes.wasm.jitstats +++ b/pyre/bench/synth/getframe_while_subwalk_decline_shapes.wasm.jitstats @@ -2,13 +2,14 @@ bridges_compiled=0 descr_set_absent=0 descr_set_ambiguous=0 descr_set_stale_absent=0 -fbw_blackhole_adopted_multi_frame=15 +fbw_blackhole_adopted_multi_frame=0 fbw_blackhole_adopted_single_frame=0 fbw_rolled_back_with_effects=0 fbw_store_journal_rollback_failed=0 field_pos_attached_misplaced=0 field_pos_spec_misplaced=0 -guard_failures=0 +guard_failures=2 internal_compile_panics=0 -loops_aborted=15 +loops_aborted=0 loops_compiled=2 +retraces_compiled=0 diff --git a/pyre/bench/synth/trace_too_long_effect_replay.cranelift.jitstats b/pyre/bench/synth/trace_too_long_effect_replay.cranelift.jitstats index 7acac3a89b7..0bd99ccd39e 100644 --- a/pyre/bench/synth/trace_too_long_effect_replay.cranelift.jitstats +++ b/pyre/bench/synth/trace_too_long_effect_replay.cranelift.jitstats @@ -3,12 +3,13 @@ descr_set_absent=0 descr_set_ambiguous=0 descr_set_stale_absent=0 fbw_blackhole_adopted_multi_frame=0 -fbw_blackhole_adopted_single_frame=5 +fbw_blackhole_adopted_single_frame=17 fbw_rolled_back_with_effects=0 fbw_store_journal_rollback_failed=0 field_pos_attached_misplaced=0 field_pos_spec_misplaced=0 guard_failures=0 internal_compile_panics=0 -loops_aborted=6 +loops_aborted=18 loops_compiled=3 +retraces_compiled=0 diff --git a/pyre/bench/synth/trace_too_long_effect_replay.dynasm.jitstats b/pyre/bench/synth/trace_too_long_effect_replay.dynasm.jitstats index 7acac3a89b7..0bd99ccd39e 100644 --- a/pyre/bench/synth/trace_too_long_effect_replay.dynasm.jitstats +++ b/pyre/bench/synth/trace_too_long_effect_replay.dynasm.jitstats @@ -3,12 +3,13 @@ descr_set_absent=0 descr_set_ambiguous=0 descr_set_stale_absent=0 fbw_blackhole_adopted_multi_frame=0 -fbw_blackhole_adopted_single_frame=5 +fbw_blackhole_adopted_single_frame=17 fbw_rolled_back_with_effects=0 fbw_store_journal_rollback_failed=0 field_pos_attached_misplaced=0 field_pos_spec_misplaced=0 guard_failures=0 internal_compile_panics=0 -loops_aborted=6 +loops_aborted=18 loops_compiled=3 +retraces_compiled=0 diff --git a/pyre/bench/synth/trace_too_long_effect_replay.wasm.jitstats b/pyre/bench/synth/trace_too_long_effect_replay.wasm.jitstats index e2c79ca9c9e..fc1e16c5e9d 100644 --- a/pyre/bench/synth/trace_too_long_effect_replay.wasm.jitstats +++ b/pyre/bench/synth/trace_too_long_effect_replay.wasm.jitstats @@ -3,12 +3,13 @@ descr_set_absent=0 descr_set_ambiguous=0 descr_set_stale_absent=0 fbw_blackhole_adopted_multi_frame=0 -fbw_blackhole_adopted_single_frame=5 +fbw_blackhole_adopted_single_frame=17 fbw_rolled_back_with_effects=0 fbw_store_journal_rollback_failed=0 field_pos_attached_misplaced=0 field_pos_spec_misplaced=0 guard_failures=0 internal_compile_panics=0 -loops_aborted=5 +loops_aborted=17 loops_compiled=1 +retraces_compiled=0 diff --git a/pyre/bench/synth/trace_too_long_inline_multiframe.cranelift.jitstats b/pyre/bench/synth/trace_too_long_inline_multiframe.cranelift.jitstats index 53062486126..88de6dd70fb 100644 --- a/pyre/bench/synth/trace_too_long_inline_multiframe.cranelift.jitstats +++ b/pyre/bench/synth/trace_too_long_inline_multiframe.cranelift.jitstats @@ -3,13 +3,13 @@ descr_set_absent=0 descr_set_ambiguous=0 descr_set_stale_absent=0 fbw_blackhole_adopted_multi_frame=10 -fbw_blackhole_adopted_single_frame=13 +fbw_blackhole_adopted_single_frame=21 fbw_rolled_back_with_effects=0 fbw_store_journal_rollback_failed=0 field_pos_attached_misplaced=0 field_pos_spec_misplaced=0 guard_failures=0 internal_compile_panics=0 -loops_aborted=23 +loops_aborted=31 loops_compiled=2 retraces_compiled=0 diff --git a/pyre/bench/synth/trace_too_long_inline_multiframe.dynasm.jitstats b/pyre/bench/synth/trace_too_long_inline_multiframe.dynasm.jitstats index 53062486126..88de6dd70fb 100644 --- a/pyre/bench/synth/trace_too_long_inline_multiframe.dynasm.jitstats +++ b/pyre/bench/synth/trace_too_long_inline_multiframe.dynasm.jitstats @@ -3,13 +3,13 @@ descr_set_absent=0 descr_set_ambiguous=0 descr_set_stale_absent=0 fbw_blackhole_adopted_multi_frame=10 -fbw_blackhole_adopted_single_frame=13 +fbw_blackhole_adopted_single_frame=21 fbw_rolled_back_with_effects=0 fbw_store_journal_rollback_failed=0 field_pos_attached_misplaced=0 field_pos_spec_misplaced=0 guard_failures=0 internal_compile_panics=0 -loops_aborted=23 +loops_aborted=31 loops_compiled=2 retraces_compiled=0 diff --git a/pyre/bench/synth/trace_too_long_inline_multiframe.wasm.jitstats b/pyre/bench/synth/trace_too_long_inline_multiframe.wasm.jitstats index da0c413c8c3..6f5caa75783 100644 --- a/pyre/bench/synth/trace_too_long_inline_multiframe.wasm.jitstats +++ b/pyre/bench/synth/trace_too_long_inline_multiframe.wasm.jitstats @@ -3,13 +3,13 @@ descr_set_absent=0 descr_set_ambiguous=0 descr_set_stale_absent=0 fbw_blackhole_adopted_multi_frame=10 -fbw_blackhole_adopted_single_frame=12 +fbw_blackhole_adopted_single_frame=20 fbw_rolled_back_with_effects=0 fbw_store_journal_rollback_failed=0 field_pos_attached_misplaced=0 field_pos_spec_misplaced=0 guard_failures=0 internal_compile_panics=0 -loops_aborted=22 +loops_aborted=30 loops_compiled=0 retraces_compiled=0 diff --git a/pyre/extra_tests/snippets/cli_dash_c_dedent.py b/pyre/extra_tests/snippets/cli_dash_c_dedent.py new file mode 100644 index 00000000000..b5933aae819 --- /dev/null +++ b/pyre/extra_tests/snippets/cli_dash_c_dedent.py @@ -0,0 +1,73 @@ +import subprocess +import sys + +# `-c` strips common leading whitespace before compiling an indented block. +indented = '\n import sys\n print("dedented")\n' +done = subprocess.run( + [sys.executable, "-c", indented], capture_output=True, text=True +) +assert done.returncode == 0, (done.returncode, done.stderr) +assert done.stdout.strip() == "dedented", done.stdout + +# A single line carries its own prefix, with no leading newline to start from. +done = subprocess.run( + [sys.executable, "-c", ' print("one line")'], capture_output=True, text=True +) +assert done.returncode == 0, (done.returncode, done.stderr) +assert done.stdout.strip() == "one line", done.stdout + +# Dedenting uses the raw text, including text inside string literals; a +# nonblank line at column zero therefore prevents all stripping. +done = subprocess.run( + [sys.executable, "-c", ' x = 1\ny = 2\n'], capture_output=True, text=True +) +assert done.returncode != 0, done.stdout +assert "IndentationError" in done.stderr, done.stderr + +# A tab and a space are different characters and never cancel, so a block +# indented with tabs is stripped by its tab. +done = subprocess.run( + [sys.executable, "-c", '\n\tprint("tabbed")\n'], capture_output=True, text=True +) +assert done.returncode == 0, (done.returncode, done.stderr) +assert done.stdout.strip() == "tabbed", done.stdout + +# Space/tab-only lines are emptied and do not narrow the common prefix. +done = subprocess.run( + [sys.executable, "-c", ' s = """A\n \n B"""\n print(repr(s))'], + capture_output=True, + text=True, +) +assert done.returncode == 0, (done.returncode, done.stderr) +assert done.stdout.strip() == repr("A\n\nB"), done.stdout + +# Without a common prefix, the source is unchanged, including blank lines. +done = subprocess.run( + [sys.executable, "-c", 'x = """a\n \nb"""\nprint(repr(x))'], + capture_output=True, + text=True, +) +assert done.returncode == 0, (done.returncode, done.stderr) +assert done.stdout.strip() == repr("a\n \nb"), done.stdout + +# Whitespace other than space or tab is content: at column zero it prevents +# dedenting, and after two spaces it narrows the common prefix to two spaces. +# +# Non-printable vertical tabs and information separators are omitted because +# their tokenizer error category is unrelated to dedenting. +for content in ("\x0c", "\r", " \r", "\xa0"): + done = subprocess.run( + [sys.executable, "-c", ' a = 1\n' + content + '\n print("x")'], + capture_output=True, + text=True, + ) + assert done.returncode != 0, (content, done.stdout) + assert "IndentationError" in done.stderr, (content, done.stderr) + +# A script file and code on stdin keep their indentation, which is still an +# error — the stripping belongs to `-c` alone. +done = subprocess.run( + [sys.executable, "-"], input=' print("stdin")\n', capture_output=True, text=True +) +assert done.returncode != 0, done.stdout +assert "IndentationError" in done.stderr, done.stderr diff --git a/pyre/extra_tests/snippets/stdlib_posix.py b/pyre/extra_tests/snippets/stdlib_posix.py index b371b2c8b61..7f70c7dbcae 100644 --- a/pyre/extra_tests/snippets/stdlib_posix.py +++ b/pyre/extra_tests/snippets/stdlib_posix.py @@ -1,3 +1,8 @@ +import gc +import shutil +import sys +import tempfile +import warnings import os import posix @@ -33,3 +38,297 @@ def __int__(self): finally: os.close(fd) os.remove(path) + + +# `__fspath__` is looked up on the type and its descriptor is resolved against +# the instance, so a property runs and what it answers is what gets called. A +# `None` left on the type switches the protocol off the way `__hash__ = None` +# does, and the object is reported as not path-like rather than as something +# that failed to be called. +class FsPathOk: + def __fspath__(self): + return "fspath-ok" + + +class FsPathNone: + __fspath__ = None + + +class FsPathNoneOverride(FsPathOk): + __fspath__ = None + + +class FsPathProperty: + __fspath__ = property(lambda self: (lambda: "fspath-property")) + + +class FsPathPropertyNone: + __fspath__ = property(lambda self: None) + + +class FsPathInt: + def __fspath__(self): + return 1 + + +class FsPathBytes: + def __fspath__(self): + return b"fspath-bytes" + + +assert os.fspath(FsPathOk()) == "fspath-ok" +assert os.fspath(FsPathProperty()) == "fspath-property" +assert os.fspath(FsPathBytes()) == b"fspath-bytes" + +try: + os.fspath(FsPathInt()) +except TypeError as exc: + assert str(exc) == ( + "expected FsPathInt.__fspath__() to return str or bytes, not int" + ), exc +else: + raise AssertionError("os.fspath(FsPathInt()) did not raise") + +for cls in (FsPathNone, FsPathNoneOverride, FsPathPropertyNone): + try: + os.fspath(cls()) + except TypeError as exc: + # A disabled `__fspath__` reports the original object's type. + assert cls.__name__ in str(exc), (cls, exc) + assert "NoneType" not in str(exc), (cls, exc) + else: + raise AssertionError("os.fspath(%s()) did not raise" % cls.__name__) + + # Builtins using the shared path converter also reject a disabled + # `__fspath__` as a property of the original object. + try: + open(cls()) + except TypeError as exc: + assert cls.__name__ in str(exc), (cls, exc) + assert "NoneType" not in str(exc), (cls, exc) + else: + raise AssertionError("open(%s()) did not raise" % cls.__name__) + + try: + os.rename(cls(), "unused") + except TypeError as exc: + assert cls.__name__ in str(exc), (cls, exc) + assert "NoneType" not in str(exc), (cls, exc) + else: + raise AssertionError("os.rename(%s(), ...) did not raise" % cls.__name__) + + +# Every exercised user/group ID setter rejects these values during `uid_t` +# conversion, before a privilege-changing syscall can run. +class _IndexId: + def __init__(self, value): + self.value = value + + def __index__(self): + return self.value + + +class _RaisingIndexId: + def __index__(self): + raise ValueError("index failed") + + +class _IntOnlyId: + def __int__(self): + return 0 + + +for _name in ("setuid", "seteuid", "setgid", "setegid"): + _setter = getattr(os, _name, None) + if _setter is None: + continue + assert_raises(TypeError, lambda f=_setter: f("not an int")) + assert_raises(OverflowError, lambda f=_setter: f(1 << 32)) + assert_raises(OverflowError, lambda f=_setter: f(_IndexId(-2))) + assert_raises(OverflowError, lambda f=_setter: f(_IndexId(1 << 32))) + with assert_raises(TypeError) as _exc: + _setter(_RaisingIndexId()) + assert "_RaisingIndexId" in str(_exc.exception), (_name, _exc.exception) + assert_raises(TypeError, lambda f=_setter: f(_IntOnlyId())) + +for _name in ("setreuid", "setregid"): + _setter = getattr(os, _name, None) + if _setter is None: + continue + assert_raises(TypeError, lambda f=_setter: f("not an int", 0)) + assert_raises(TypeError, lambda f=_setter: f(0, "not an int")) + assert_raises(OverflowError, lambda f=_setter: f(1 << 32, 0)) + assert_raises(OverflowError, lambda f=_setter: f(0, 1 << 32)) + + +# An object that is not an integer at all is refused by the `uid_t` conversion, +# and the refusal names the object's own class — `_typed_unwrap_error` formats +# `%T`, not the tag every instance of a Python-level class shares. Only the +# class name is asserted, because that is what both runtimes agree on: the +# wording around it differs. +class _NotAnId: + pass + + +for _name in ("setuid", "seteuid", "setgid", "setegid"): + _setter = getattr(os, _name, None) + if _setter is None: + continue + with assert_raises(TypeError) as _exc: + _setter(_NotAnId()) + assert "_NotAnId" in str(_exc.exception), (_name, _exc.exception) + + +# An iterator from `os.scandir` that was neither closed nor run to the end is +# an unclosed one, and says so when it is collected. Closing it, or reaching +# the end of the enumeration, is what makes it silent. +def _scandir_warnings(use): + directory = tempfile.mkdtemp() + try: + for name in ("a", "b"): + with open(os.path.join(directory, name), "w"): + pass + with warnings.catch_warnings(record=True) as caught: + warnings.simplefilter("always") + use(directory) + gc.collect() + return [w for w in caught if issubclass(w.category, ResourceWarning)] + finally: + shutil.rmtree(directory) + + +def _abandon_midway(directory): + iterator = os.scandir(directory) + next(iterator) + del iterator + + +def _exhaust(directory): + iterator = os.scandir(directory) + list(iterator) + del iterator + + +def _close_midway(directory): + with os.scandir(directory) as iterator: + next(iterator) + del iterator + + +# Closing partway through ends the enumeration, so the entries that were never +# read are not handed out afterwards. +def _closed_iterator_stops(): + directory = tempfile.mkdtemp() + try: + for name in ("a", "b", "c"): + with open(os.path.join(directory, name), "w"): + pass + iterator = os.scandir(directory) + next(iterator) + iterator.close() + assert_raises(StopIteration, lambda: next(iterator)) + finally: + shutil.rmtree(directory) + + +_closed_iterator_stops() + + +assert _scandir_warnings(_abandon_midway), "an abandoned scandir iterator is unclosed" +assert not _scandir_warnings(_exhaust), "an exhausted scandir iterator is closed" +assert not _scandir_warnings(_close_midway), "a closed scandir iterator is closed" + + +# The number of variables an exec takes from its environment is the mapping's +# own length, and `keys()` and `values()` are read by position under it, so a +# snapshot that cannot cover that length is an error rather than a quietly +# shorter environment. `__getitem__` must exist on the type but is never +# called for any of it. +class _Env(dict): + def __init__(self, keys, values, size): + self._keys, self._values, self._size = keys, values, size + + def __len__(self): + return self._size + + def keys(self): + return self._keys + + def values(self): + return self._values + + def __getitem__(self, key): + raise AssertionError("__getitem__ must not be called") + + +class _NonMappingEnv: + def __len__(self): + return 0 + + def keys(self): + return () + + def values(self): + return () + + +_MISSING = "/nonexistent-program-" + str(os.getpid()) + +assert_raises(TypeError, lambda: os.execve(_MISSING, ["x"], _NonMappingEnv())) +assert_raises(TypeError, lambda: os.execve(_MISSING, ["x"], None)) +# `posix_spawn` takes the same environment conversion but also accepts None, so +# it is absent where the platform has no spawn at all. +if hasattr(os, "posix_spawn"): + assert_raises( + TypeError, lambda: os.posix_spawn(_MISSING, ["x"], _NonMappingEnv()) + ) + +for _short in (_Env([b"A"], [b"1"], 3), _Env([b"A", b"B"], [b"1"], 2)): + assert_raises(IndexError, lambda e=_short: os.execve(_MISSING, ["x"], e)) + +# The snapshots need only be iterable, and one longer than that length has its +# tail ignored. Reaching the exec is what says the conversion finished, so the +# program that is not there is the error that arrives. +for _ok in ( + _Env((b"A",), (b"1",), 1), + _Env({b"A"}, [b"1"], 1), + _Env([b"A", b"B"], [b"1", b"2"], 1), +): + assert_raises(FileNotFoundError, lambda e=_ok: os.execve(_MISSING, ["x"], e)) + +# Empty names and names with an interior `=` cannot be represented, while an +# initial `=` is the permitted drive-current-directory spelling. +for _bad_name in (b"", b"A=B"): + assert_raises( + ValueError, + lambda n=_bad_name: os.execve(_MISSING, ["x"], _Env([n], [b"1"], 1)), + ) + +assert_raises( + FileNotFoundError, + lambda: os.execve(_MISSING, ["x"], _Env([b"=C:"], [b"1"], 1)), +) + + +# `sendfile`'s header and trailer vectors are read by index, so each has to be +# a sequence; a generator is refused rather than consumed. Only the BSD-shaped +# call takes them at all. +if sys.platform == "darwin": + _read_fd, _write_fd = os.pipe() + try: + for _name in ("headers", "trailers"): + assert_raises( + TypeError, + lambda n=_name: os.sendfile( + _write_fd, _read_fd, 0, 1, **{n: (b"x" for _ in range(1))} + ), + ) + assert_raises( + TypeError, + lambda n=_name: os.sendfile( + _write_fd, _read_fd, 0, 1, **{n: [b"x", 1]} + ), + ) + finally: + os.close(_read_fd) + os.close(_write_fd) diff --git a/pyre/pyre-interpreter/src/argument.rs b/pyre/pyre-interpreter/src/argument.rs index 9c808fa0fa5..a31bd078877 100644 --- a/pyre/pyre-interpreter/src/argument.rs +++ b/pyre/pyre-interpreter/src/argument.rs @@ -456,10 +456,9 @@ pub fn combine_starargs_wrapped( /// ...merge into self.keyword_names_w / self.keywords_w... /// ``` /// -/// TODO: pyre's `view_as_kwargs` always returns -/// `(None, [])` (kwargsdict variant unported), so the fast-path arm -/// (lines 110-120) is unreachable until the dict-strategy port lands. -/// The slow `keys()` iteration arm runs unconditionally for now. +/// `baseobjspace::view_as_kwargs` supplies a direct names-and-values view for +/// exact dicts whose keys are all strings. Other mappings use the `keys()` and +/// item-lookup path below, preserving mapping overrides and their exceptions. pub fn combine_starstarargs_wrapped( keyword_names_out: &mut Vec, keywords_out: &mut Vec, diff --git a/pyre/pyre-interpreter/src/baseobjspace.rs b/pyre/pyre-interpreter/src/baseobjspace.rs index 87b3ebd68be..34ddb312329 100644 --- a/pyre/pyre-interpreter/src/baseobjspace.rs +++ b/pyre/pyre-interpreter/src/baseobjspace.rs @@ -3045,20 +3045,18 @@ pub(crate) fn enumerate_reduce_method(args: &[PyObjectRef]) -> PyResult { let i64_index = pyre_object::functional::w_enumerate_get_index(self_); let raw = pyre_object::functional::w_enumerate_get_iter_or_list(self_); let w_iter = if raw.is_null() { - // Exhausted enumerate (`:294-295` set `w_iter_or_list` to - // null); substitute an empty seq-iter so the reduce stays - // round-trippable. - pyre_object::w_seq_iter_new(w_list_new(vec![]), 0) + // `W_Enumerate.descr_next` clears the source after exhaustion. + // An empty list iterator preserves that exhausted state and the + // concrete iterator type exposed by CPython's reduce protocol. + pyre_object::w_list_iter_new(w_list_new(vec![])) } else if pyre_object::is_list(raw) { - // List fast path (`:289-294`): `w_iter_or_list` is the source - // list itself and `index` is the cursor into it. Materialise - // a seq-iterator positioned at the cursor so the reconstructed - // enumerate resumes from the right element rather than the - // list head. + // The exact-list constructor fast path stores the source list and + // uses `index` as its cursor. Materialise the corresponding list + // iterator at that cursor so unpickling resumes at the same item. let len = pyre_object::w_list_len(raw); - let it = pyre_object::w_seq_iter_new(raw, len); + let it = pyre_object::w_list_iter_new(raw); let pos = i64_index.clamp(0, len as i64); - pyre_object::w_seq_iter_set_index(it, pos); + pyre_object::w_list_iter_set_index(it, pos); it } else { raw @@ -8440,10 +8438,14 @@ pub fn uint_w(obj: PyObjectRef) -> Result { "int too large to convert to unsigned int", )); } - // W_Root.uint_w → _typed_unwrap_error(space, "integer"). - let tp_name = unsafe { (*(*obj).ob_type).name }; + // W_Root.uint_w → _typed_unwrap_error(space, "integer"), whose body + // (baseobjspace.py:316-318) is `"expected %s, got %T object"`. `%T` + // formats `space.type(w_obj).getname(space)`, the user-visible class — + // not the `ob_type` tag, which every instance of a Python-level class + // shares and which would name them all `object`. Err(PyError::type_error(format!( - "expected integer, got {tp_name} object" + "expected integer, got {} object", + object_functionstr_type_name(obj) ))) } @@ -8500,12 +8502,14 @@ pub fn c_ushort_w(obj: PyObjectRef) -> Result { Ok(value as u16) } -/// pypy/interpreter/baseobjspace.py c_uid_t_w. Equivalent to c_uint_w, -/// except -1 maps to UINT_MAX ((uid_t)-1) and values below -1 raise -/// OverflowError rather than ValueError. `uint_w` does not run any -/// __index__ conversion, so the `int_w` retry on the negative branch sees -/// only the real int and is side-effect free. +/// `baseobjspace.py:2110` c_uid_t_w. Equivalent to c_uint_w, except -1 maps to +/// UINT_MAX ((uid_t)-1) and values below -1 raise OverflowError rather than +/// ValueError. `posixmodule.c:823` applies the index protocol before the range +/// checks, but reports the original object's type if conversion fails. A +/// successful conversion happens once up front, so `c_uint_w` and the `int_w` +/// retry see the same integer and the retry is side-effect free. pub fn c_uid_t_w(obj: PyObjectRef) -> Result { + let obj = space_index(obj).unwrap_or(obj); match c_uint_w(obj) { Ok(value) => Ok(value), Err(e) if e.kind == PyErrorKind::ValueError => { @@ -14374,6 +14378,37 @@ pub fn ismapping_w(w_obj: PyObjectRef) -> bool { } } +/// Port of `ObjSpace.issequence_w` from PyPy's `baseobjspace`. +/// The `is_dict` arm mirrors `ismapping_w`'s, for the same reason: the builtin +/// mapping's flag may not be reachable through `typedef::r#type`, while a dict +/// subclass carries the inherited flag. +pub fn issequence_w(w_obj: PyObjectRef) -> bool { + unsafe { + if is_dict(w_obj) { + return false; + } + let w_type = crate::typedef::r#type(w_obj).map_or(std::ptr::null_mut(), |p| p.as_ptr()); + let flag = pyre_object::typeobject::w_type_get_flag_map_or_seq(w_type); + if flag == b'M' { + return false; + } + if flag == b'S' { + return true; + } + lookup(w_obj, "__getitem__").is_some() + } +} + +/// `PyMapping_Check`'s rule: the object's type defines `__getitem__`. +/// +/// This deliberately does not use `ismapping_w` or `issequence_w`: both first +/// consult `flag_map_or_seq`, which makes a list non-mapping and a dict +/// non-sequence before their shared type-level `__getitem__` fallback. The C +/// API predicate is only that fallback, and ignores instance attributes. +pub fn py_mapping_check(w_obj: PyObjectRef) -> bool { + unsafe { lookup(w_obj, "__getitem__").is_some() } +} + pub fn is_iterable(obj: PyObjectRef) -> bool { if obj.is_null() { return false; diff --git a/pyre/pyre-interpreter/src/builtins.rs b/pyre/pyre-interpreter/src/builtins.rs index a64a0e2cb19..a2b142ec89f 100644 --- a/pyre/pyre-interpreter/src/builtins.rs +++ b/pyre/pyre-interpreter/src/builtins.rs @@ -9394,6 +9394,32 @@ pub fn lookup_exc_class(name: &str) -> Option { registry.get(name).copied().map(|cls| cls as PyObjectRef) } +/// The `PythonFinalizationError` for an operation refused at interpreter +/// shutdown, carrying `message` as its argument when one is given. +/// +/// Every caller reaches this only while `thread::is_finalizing()` holds, which +/// is exactly when the registry [`lookup_exc_class`] reads may already be torn +/// down. Naming the class with `expect` there ends the process instead of +/// refusing the operation, so a missing class falls back to the base the +/// registration gives it — `PythonFinalizationError` is made on +/// `runtime_error` — and the `except RuntimeError` that would have caught it +/// still does. +pub fn finalization_error(message: Option<&str>) -> crate::PyError { + let Some(cls) = lookup_exc_class("PythonFinalizationError") else { + return crate::PyError::runtime_error( + message.unwrap_or("Operation blocked during Python finalization."), + ); + }; + let args = match message { + Some(message) => vec![cls, pyre_object::w_str_new(message)], + None => vec![cls], + }; + match exc_exception_new(&args) { + Ok(exc) => unsafe { crate::PyError::from_exc_object(exc) }, + Err(err) => err, + } +} + /// Look up the reusable prebuilt instance for a builtin exception /// class, addressed by `ExcKind` name. Mirrors RPython's /// `rpython/rtyper/exceptiondata.py:34-45 get_standard_ll_exc_instance` diff --git a/pyre/pyre-interpreter/src/function.rs b/pyre/pyre-interpreter/src/function.rs index eb41f8f0f49..6fa50570855 100644 --- a/pyre/pyre-interpreter/src/function.rs +++ b/pyre/pyre-interpreter/src/function.rs @@ -3323,15 +3323,14 @@ pub fn funccall(func: PyObjectRef, args: &[PyObjectRef]) -> PyObjectRef { /// fast-path in `funccall_valuestack` can recognize this specific call and /// inline `exc_info_direct` without going through the regular dispatch. /// -/// PyPy stores this on the space; pyre is single-space-per-thread, so a -/// thread-local cell suffices. The paired `direct_fn` returns the same -/// `(type, value, traceback)` tuple as the regular closure but skips the -/// builtin-call setup. +/// PyPy stores this on the shared object space. Pyre's process-global slots +/// preserve the same identity across execution contexts and OS threads. The +/// paired `direct_fn` returns the same `(type, value, traceback)` tuple as the +/// regular closure but skips the builtin-call setup. type ExcInfoDirectFn = fn() -> PyObjectRef; -thread_local! { - static SYS_EXC_INFO_CODE: std::cell::Cell<*const ()> = const { std::cell::Cell::new(std::ptr::null()) }; - static SYS_EXC_INFO_DIRECT_FN: std::cell::Cell> = const { std::cell::Cell::new(None) }; -} +static SYS_EXC_INFO_CODE: std::sync::atomic::AtomicPtr<()> = + std::sync::atomic::AtomicPtr::new(std::ptr::null_mut()); +static SYS_EXC_INFO_DIRECT_FN: std::sync::OnceLock = std::sync::OnceLock::new(); /// Register the BuiltinCode pointer + direct helper for `sys.exc_info`. /// @@ -3340,18 +3339,21 @@ thread_local! { /// builtin function; `direct_fn` is the JIT-direct equivalent of the /// closure body. `funccall_valuestack` consults both to take the fast path. pub fn register_sys_exc_info_path(code: *const (), direct_fn: ExcInfoDirectFn) { - SYS_EXC_INFO_CODE.with(|cell| cell.set(code)); - SYS_EXC_INFO_DIRECT_FN.with(|cell| cell.set(Some(direct_fn))); + SYS_EXC_INFO_CODE.store(code.cast_mut(), std::sync::atomic::Ordering::Release); + // Reinitializing the sys module may replace its BuiltinCode object, but + // every instance is backed by the same `exc_info_direct` symbol. Preserve + // the object-space-wide helper installed by the first initialization. + let _ = SYS_EXC_INFO_DIRECT_FN.set(direct_fn); } #[inline] fn sys_exc_info_code() -> *const () { - SYS_EXC_INFO_CODE.with(|cell| cell.get()) + SYS_EXC_INFO_CODE.load(std::sync::atomic::Ordering::Acquire) } #[inline] fn sys_exc_info_direct_fn() -> Option { - SYS_EXC_INFO_DIRECT_FN.with(|cell| cell.get()) + SYS_EXC_INFO_DIRECT_FN.get().copied() } /// function.py:139-203 `funccall_valuestack` — fast-path call dispatcher. @@ -3388,7 +3390,7 @@ pub fn funccall_valuestack( // function.py:146-150 — JIT direct path for `sys.exc_info()` with no // arguments: skip the builtin call entirely and inline the tuple // construction. PyPy uses `space._code_of_sys_exc_info`; pyre uses the - // thread-local cache populated during sys module init. + // shared cache populated during sys module init. if nargs == 0 && majit_metainterp::jit::we_are_jitted() && std::ptr::eq(code, sys_exc_info_code()) diff --git a/pyre/pyre-interpreter/src/gateway.rs b/pyre/pyre-interpreter/src/gateway.rs index d588f831bc1..e7e7c01e37b 100644 --- a/pyre/pyre-interpreter/src/gateway.rs +++ b/pyre/pyre-interpreter/src/gateway.rs @@ -1807,18 +1807,36 @@ fn path_or_fd_w( } (Vec::new(), obj_slot, fd) } else { - // `type(path).__fspath__(path)` — the descriptor read off the type is - // unbound, so `path` is supplied as the sole argument. - let Some(fspath_fn) = crate::typedef::r#type(obj) + let Some(fspath_descr) = crate::typedef::r#type(obj) .and_then(|pt| crate::baseobjspace::lookup_in_type(pt.as_ptr(), "__fspath__")) else { return Err(reject(obj)); }; let fspath_slot = pyre_object::gc_roots::shadow_stack_len(); - pyre_object::gc_roots::pin_root(fspath_fn); + pyre_object::gc_roots::pin_root(fspath_descr); + let path_type = + crate::typedef::r#type(pyre_object::gc_roots::shadow_stack_get(obj_slot)) + .expect("a path argument has a type"); + // PyPy's `interp_posix._fspath` binds `__fspath__` before calling + // it; a non-descriptor is its own bound value. + let fspath_fn = crate::baseobjspace::get( + pyre_object::gc_roots::shadow_stack_get(fspath_slot), + pyre_object::gc_roots::shadow_stack_get(obj_slot), + path_type.as_ptr(), + )? + .unwrap_or_else(|| pyre_object::gc_roots::shadow_stack_get(fspath_slot)); + let obj = pyre_object::gc_roots::shadow_stack_get(obj_slot); + // A `None` left on the type switches the protocol off the way + // `__hash__ = None` does, so the object is turned away as not + // path-like and named by its own type. `_unwrap_path` instead + // calls what it found, which reports `NoneType` as not callable. + if pyre_object::is_none(fspath_fn) { + return Err(reject(obj)); + } + pyre_object::gc_roots::shadow_stack_set(fspath_slot, fspath_fn); let result = crate::call::call_function_impl_result( pyre_object::gc_roots::shadow_stack_get(fspath_slot), - &[pyre_object::gc_roots::shadow_stack_get(obj_slot)], + &[], )?; let result_slot = pyre_object::gc_roots::shadow_stack_len(); pyre_object::gc_roots::pin_root(result); diff --git a/pyre/pyre-interpreter/src/importing.rs b/pyre/pyre-interpreter/src/importing.rs index a2da83bd9a4..9acb79bc979 100644 --- a/pyre/pyre-interpreter/src/importing.rs +++ b/pyre/pyre-interpreter/src/importing.rs @@ -7,11 +7,10 @@ //! - `check_sys_modules()` — consult the module cache //! - `import_all_from()` — IMPORT_STAR handler -use std::cell::RefCell; use std::collections::HashMap; use std::path::PathBuf; use std::sync::{ - LazyLock, Mutex, OnceLock, + Arc, LazyLock, Mutex, OnceLock, atomic::{AtomicBool, AtomicI64, AtomicUsize, Ordering}, }; // `Path` is used only by the host_env source/package loaders; keep it gated @@ -103,7 +102,7 @@ use host::os as host_os; // embedded stdlib bundle). The import machinery never branches per host; // only the installed provider differs. #[cfg(feature = "host_env")] -pub trait SourceProvider { +pub trait SourceProvider: Send + Sync { /// True when `path` names a readable regular file. fn is_file(&self, path: &Path) -> bool; /// True when `path` names a directory. @@ -114,33 +113,29 @@ pub trait SourceProvider { } #[cfg(feature = "host_env")] -thread_local! { - static SOURCE_PROVIDER: RefCell>> = - const { RefCell::new(None) }; -} +static SOURCE_PROVIDER: LazyLock>>> = + LazyLock::new(|| Mutex::new(None)); /// Install the byte source the import machinery reads through. The wasm /// bootstrap installs a host-import-backed or in-memory-VFS provider before /// the first import; native/pyrex leaves it unset and the default kernel-FS /// provider answers every probe. #[cfg(feature = "host_env")] -pub fn install_source_provider(provider: std::rc::Rc) { - SOURCE_PROVIDER.with(|p| *p.borrow_mut() = Some(provider)); +pub fn install_source_provider(provider: Arc) { + *SOURCE_PROVIDER.lock().unwrap() = Some(provider); } /// Run `f` against the installed provider, lazily defaulting to the platform's -/// kernel-FS provider when none was installed. The `Rc` is cloned out before -/// `f` runs so the thread-local borrow is not held across the call (the import -/// path is re-entrant). +/// kernel-FS provider when none was installed. The `Arc` is cloned out before +/// `f` runs so the process-global lock is not held across the re-entrant import +/// path. PyPy owns this provider through its shared object space, not through +/// an execution context or OS thread. #[cfg(feature = "host_env")] fn with_source_provider(f: impl FnOnce(&dyn SourceProvider) -> R) -> R { - let provider = SOURCE_PROVIDER.with(|p| { - let mut slot = p.borrow_mut(); - if slot.is_none() { - *slot = Some(default_source_provider()); - } - slot.clone().unwrap() - }); + let provider = { + let mut slot = SOURCE_PROVIDER.lock().unwrap(); + slot.get_or_insert_with(default_source_provider).clone() + }; f(&*provider) } @@ -172,18 +167,18 @@ pub fn read_source_to_string(path: &Path) -> std::io::Result { not(target_arch = "wasm32"), not(feature = "sandbox") ))] -fn default_source_provider() -> std::rc::Rc { - std::rc::Rc::new(HostFsProvider) +fn default_source_provider() -> Arc { + Arc::new(HostFsProvider) } #[cfg(all(feature = "host_env", not(target_arch = "wasm32"), feature = "sandbox"))] -fn default_source_provider() -> std::rc::Rc { - std::rc::Rc::new(SeamSourceProvider) +fn default_source_provider() -> Arc { + Arc::new(SeamSourceProvider) } #[cfg(all(feature = "host_env", target_arch = "wasm32"))] -fn default_source_provider() -> std::rc::Rc { - std::rc::Rc::new(NullSourceProvider) +fn default_source_provider() -> Arc { + Arc::new(NullSourceProvider) } /// Kernel-filesystem provider — the default on native and the wasmtime @@ -298,7 +293,7 @@ pub static VFS_BLOB: &[u8] = include_bytes!(concat!(env!("OUT_DIR"), "/stdlib_vf #[cfg(feature = "wasm_vfs")] enum VfsEntry { - File(std::rc::Rc), + File(Arc), Dir, } @@ -348,7 +343,7 @@ impl VfsProvider { map.entry(dir.to_path_buf()).or_insert(VfsEntry::Dir); ancestor = dir.parent(); } - map.insert(full, VfsEntry::File(std::rc::Rc::from(src.as_str()))); + map.insert(full, VfsEntry::File(Arc::from(src.as_str()))); } VfsProvider { map } } @@ -380,7 +375,7 @@ impl SourceProvider for VfsProvider { pub fn mount_embedded_stdlib(mount: &Path) { let provider = VfsProvider::from_blob(VFS_BLOB, mount); add_sys_path(mount); - install_source_provider(std::rc::Rc::new(provider)); + install_source_provider(Arc::new(provider)); } // ── sys.modules cache ──────────────────────────────────────────────── @@ -1214,6 +1209,41 @@ fn init_sysconfigdata(ns: PyObjectRef) { store_int(vars_slot, "Py_DEBUG", 0); store_int(vars_slot, "Py_GIL_DISABLED", 1); store_int(vars_slot, "Py_ENABLE_SHARED", 0); + // Publish an entropy capability only when it names the call this build + // makes. Unsandboxed `host_env` reaches the host through `getrandom::fill`, + // which picks its backend per target: `getrandom(2)` for the first set + // below, `getentropy(2)` for the second. A target in neither set makes + // neither call — the iOS family goes through `CCRandomGenerateBytes`, + // NetBSD resolves `getrandom` at runtime and reads a sysctl when it is + // absent, and Haiku, Redox, NTO and AIX open `/dev/urandom` — so it + // publishes no name, and neither do the sandbox and no-`host_env` routes, + // which open `/dev/urandom`, or Windows, which uses its cryptography + // provider. + #[cfg(all( + any( + target_os = "linux", + target_os = "android", + target_os = "freebsd", + target_os = "dragonfly", + target_os = "illumos", + target_os = "solaris", + target_os = "hurd", + ), + feature = "host_env", + not(feature = "sandbox") + ))] + store_int(vars_slot, "HAVE_GETRANDOM_SYSCALL", 1); + #[cfg(all( + any( + target_os = "macos", + target_os = "openbsd", + target_os = "vita", + target_os = "emscripten", + ), + feature = "host_env", + not(feature = "sandbox") + ))] + store_int(vars_slot, "HAVE_GETENTROPY", 1); // Pyre has no separately linkable runtime library. Keep the build ABI // metadata above for wheel tags, but never invent files that are absent // from the installation. diff --git a/pyre/pyre-interpreter/src/jit_fnaddr.rs b/pyre/pyre-interpreter/src/jit_fnaddr.rs index 3708727f220..0daa3e52f80 100644 --- a/pyre/pyre-interpreter/src/jit_fnaddr.rs +++ b/pyre/pyre-interpreter/src/jit_fnaddr.rs @@ -1114,7 +1114,7 @@ pub fn jit_trace_fnaddrs() -> Vec<(&'static str, i64)> { // thread-local `install_current_frame` moves, the two `display::repr_*` // twins the `REPR_ACTIVE` mid-repr set (the // `note_eval_activation_{enter,exit}` twin shape), and `autoflusher_add` - // the `AUTOFLUSHER` thread-local handle table. + // the process-global `AUTOFLUSHER` handle table owned by the object space. let unsupported_operation_type: fn() -> pyre_object::PyObjectRef = crate::module::_io::unsupported_operation_type; push_alias_pair( diff --git a/pyre/pyre-interpreter/src/lib.rs b/pyre/pyre-interpreter/src/lib.rs index bea7ca5fe68..777881dbae5 100644 --- a/pyre/pyre-interpreter/src/lib.rs +++ b/pyre/pyre-interpreter/src/lib.rs @@ -1171,39 +1171,41 @@ pub fn all_subclass_range_aliases() -> Vec()), subclass_range_alias(177, typed::()), subclass_range_alias(178, typed::()), - // `posix.DirEntry` follows the unconditional native owners. + // Native-only posix aliases 179 and 180 preserve `build_gc`'s rclass + // registration order after the unconditional aliases. #[cfg(not(target_arch = "wasm32"))] subclass_range_alias(179, typed::()), - // rustls-backed `_ssl` native payloads. They are appended after the - // last pre-existing native class in the same order `build_gc` - // registers them, so no established type id moves. + #[cfg(not(target_arch = "wasm32"))] + subclass_range_alias(180, typed::()), + // The rustls-backed `_ssl` aliases preserve `build_gc`'s registration + // order for `W_SSLContext`, `W_MemoryBIO`, and `W_SSLSession`. #[cfg(all(not(target_arch = "wasm32"), not(feature = "sandbox")))] - subclass_range_alias(180, typed::()), + subclass_range_alias(181, typed::()), #[cfg(all(not(target_arch = "wasm32"), not(feature = "sandbox")))] - subclass_range_alias(181, typed::()), + subclass_range_alias(182, typed::()), #[cfg(all(not(target_arch = "wasm32"), not(feature = "sandbox")))] - subclass_range_alias(182, typed::()), + subclass_range_alias(183, typed::()), #[cfg(all(not(target_arch = "wasm32"), not(feature = "sandbox")))] - subclass_range_alias(183, typed::()), + subclass_range_alias(184, typed::()), #[cfg(all(not(target_arch = "wasm32"), not(feature = "sandbox")))] - subclass_range_alias(184, typed::()), + subclass_range_alias(185, typed::()), // `mmap.mmap` follows the optional SSL tail on ordinary Unix builds. // A sandbox build has no `mmap` module at all (`module/mod.rs`), so it // contributes no alias rather than sliding into the vacated SSL slot. #[cfg(all(any(unix, windows), not(feature = "sandbox")))] - subclass_range_alias(185, typed::()), + subclass_range_alias(186, typed::()), // Windows asyncio's Overlapped owner follows mmap at the native tail. // It is a non-subclassable builtin in Python, but still participates // in the rclass hierarchy because its managed header and retained // buffer/result fields are traced by the ordinary object marker. #[cfg(all(windows, feature = "host_env", not(feature = "sandbox")))] - subclass_range_alias(186, typed::()), + subclass_range_alias(187, typed::()), // `_winapi.Overlapped` follows it: a second record of the same kind, // owning its own event and transfer buffer rather than retained // Python objects, so nothing of it is traced beyond the header. #[cfg(all(windows, feature = "host_env", not(feature = "sandbox")))] subclass_range_alias( - 187, + 188, typed::(), ), ] diff --git a/pyre/pyre-interpreter/src/module/_io/mod.rs b/pyre/pyre-interpreter/src/module/_io/mod.rs index a7dae299293..04751c69a49 100644 --- a/pyre/pyre-interpreter/src/module/_io/mod.rs +++ b/pyre/pyre-interpreter/src/module/_io/mod.rs @@ -296,9 +296,10 @@ fn iobase_next(args: &[PyObjectRef]) -> crate::PyResult { // // `rweaklist.py:52 store_handle` holds each stream through `weakref.ref`; the // `GcWeakrefBox` is pyre's rweakref, so the handle list is a `Vec` of boxes. -// A box is collector-managed and this list is its only referent, so the slots -// are walked as roots ([`walk_autoflusher_roots_area`]) — hence the list lives -// with the boxing thread, and shutdown flushes that thread's streams. +// A box is collector-managed and this list is its only referent, so +// [`walk_autoflusher_roots`] walks the slots as roots. The table is shared +// process state because `space.fromcache(AutoFlusher)` owns one instance for +// the object space, independent of the thread that constructs a stream. /// `rweaklist.py:4 INITIAL_SIZE`. const AUTOFLUSHER_INITIAL_SIZE: usize = 4; @@ -307,15 +308,19 @@ const AUTOFLUSHER_INITIAL_SIZE: usize = 4; struct AutoFlusher { /// `rweaklist.py:18 self.handles` — index → `GcWeakrefBox`. A null slot is /// the `dead_ref` placeholder RPython pre-fills the list with. - handles: Vec, + handles: Vec, /// `rweaklist.py:19 self.free_list`. free_list: Vec, + /// Marks each `rweaklist.py:17-20 initialize` of the handle table so an + /// allocation can detect that shutdown discarded its reserved slot. + generation: u64, } impl AutoFlusher { /// `rweaklist.py:17-20 initialize`. fn initialize(&mut self) { - self.handles = vec![std::ptr::null_mut(); AUTOFLUSHER_INITIAL_SIZE]; + self.generation = self.generation.wrapping_add(1); + self.handles = vec![0; AUTOFLUSHER_INITIAL_SIZE]; self.free_list = (0..AUTOFLUSHER_INITIAL_SIZE).collect(); } @@ -328,14 +333,16 @@ impl AutoFlusher { return index; } for (index, &handle) in self.handles.iter().enumerate() { - if unsafe { pyre_object::weakref::w_gc_weakref_box_deref(handle) }.is_null() { + if unsafe { pyre_object::weakref::w_gc_weakref_box_deref(handle as PyObjectRef) } + .is_null() + { self.free_list.push(index); } } if self.free_list.len() * 3 < self.handles.len() * 2 { let length = self.handles.len(); self.free_list.extend(length..length * 2); - self.handles.resize(length * 2, std::ptr::null_mut()); + self.handles.resize(length * 2, 0); } self.free_list .pop() @@ -343,15 +350,9 @@ impl AutoFlusher { } } -thread_local! { - /// `interp_iobase.py:475-476 get_autoflusher` — `space.fromcache(AutoFlusher)`. - static AUTOFLUSHER: std::cell::RefCell = - std::cell::RefCell::new(AutoFlusher::default()); -} - -pub fn capture_autoflusher_root_area() -> *const () { - AUTOFLUSHER.with(|flusher| flusher as *const _ as *const ()) -} +/// `interp_iobase.py get_autoflusher` — `space.fromcache(AutoFlusher)`. +static AUTOFLUSHER: std::sync::LazyLock> = + std::sync::LazyLock::new(|| std::sync::Mutex::new(AutoFlusher::default())); /// Visit each stored `GcWeakrefBox` as a strong root. /// @@ -364,21 +365,15 @@ pub fn capture_autoflusher_root_area() -> *const () { /// `invalidate_young_weakrefs` / `invalidate_old_weakrefs` when the stream /// itself dies, which is what turns the slot into `rweaklist.py`'s `dead_ref`. /// -/// # Safety -/// `data` must come from [`capture_autoflusher_root_area`], and the owning -/// thread must be quiesced. -pub unsafe fn walk_autoflusher_roots_area( - data: *const (), - mut visitor: impl FnMut(&mut PyObjectRef), -) { - // Read through `as_ptr`: the walk runs at a safepoint the borrowing - // stream-construction path may already be inside. - let flusher = unsafe { &mut *(*(data as *const std::cell::RefCell)).as_ptr() }; +pub fn walk_autoflusher_roots(mut visitor: impl FnMut(&mut PyObjectRef)) { + let mut flusher = AUTOFLUSHER.lock().unwrap(); for slot in flusher.handles.iter_mut() { - if slot.is_null() { + if *slot == 0 { continue; } - visitor(slot); + let mut handle = *slot as PyObjectRef; + visitor(&mut handle); + *slot = handle as usize; } } @@ -388,7 +383,7 @@ pub unsafe fn walk_autoflusher_roots_area( /// /// Returns `w_iobase`, which the rweakref allocation may have relocated. /// -/// Reads and rewrites the runtime-mutable `AUTOFLUSHER` thread-local handle +/// Reads and rewrites the runtime-mutable process-global `AUTOFLUSHER` handle /// table, not a build-time constant, so the JIT residualises the call instead /// of tracing into it (`@dont_look_inside`, the `importing::sys_modules_dict` /// shape). @@ -397,20 +392,39 @@ pub(crate) fn autoflusher_add(w_iobase: PyObjectRef) -> PyObjectRef { if w_iobase.is_null() { return w_iobase; } - let index = AUTOFLUSHER.with(|flusher| flusher.borrow_mut().reserve_next_handle_index()); let _roots = pyre_object::gc_roots::push_roots(); let target_root = pyre_object::gc_roots::shadow_stack_len(); pyre_object::gc_roots::pin_root(w_iobase); - // `rweaklist.py:51-52 store_handle` — reuse the slot's box when it already - // holds one, so a long-lived process bounds the immortal boxes by its peak - // number of open streams. - let handle = AUTOFLUSHER.with(|flusher| flusher.borrow().handles[index]); - let stored = unsafe { pyre_object::weakref::w_gc_weakref_box_retarget(handle, w_iobase) }; - if !stored { - let boxed = pyre_object::weakref::w_gc_weakref_box_new( - pyre_object::gc_roots::shadow_stack_get(target_root), - ); - AUTOFLUSHER.with(|flusher| flusher.borrow_mut().handles[index] = boxed); + loop { + let (index, generation, handle) = { + let mut flusher = AUTOFLUSHER.lock().unwrap(); + let index = flusher.reserve_next_handle_index(); + (index, flusher.generation, flusher.handles[index]) + }; + // `rweaklist.py:51-52 store_handle` — reuse the slot's box when it already + // holds one, so a long-lived process bounds the immortal boxes by its peak + // number of open streams. + let target = pyre_object::gc_roots::shadow_stack_get(target_root); + let stored = unsafe { + pyre_object::weakref::w_gc_weakref_box_retarget(handle as PyObjectRef, target) + }; + let boxed = if stored { + std::ptr::null_mut() + } else { + pyre_object::weakref::w_gc_weakref_box_new(pyre_object::gc_roots::shadow_stack_get( + target_root, + )) + }; + let mut flusher = AUTOFLUSHER.lock().unwrap(); + // `rweaklist.py:17-20 initialize` replaces the table, invalidating an + // index reserved from the previous generation. + if flusher.generation != generation { + continue; + } + if !stored { + flusher.handles[index] = boxed as usize; + } + break; } pyre_object::gc_roots::shadow_stack_get(target_root) } @@ -420,14 +434,15 @@ pub(crate) fn autoflusher_add(w_iobase: PyObjectRef) -> PyObjectRef { /// Ignore I/O errors." pub fn flush_all_streams() { loop { - let handles = AUTOFLUSHER.with(|flusher| { - let mut flusher = flusher.borrow_mut(); + let handles = { + let mut flusher = AUTOFLUSHER.lock().unwrap(); flusher.free_list.clear(); - // `self.initialize()` — reset the state here, so a stream created - // while flushing is picked up by the next round instead of being - // flushed twice. + // `rweaklist.py:17-20 initialize` resets the state here, so a + // stream created while flushing is picked up by the next round + // instead of being flushed twice. + flusher.generation = flusher.generation.wrapping_add(1); std::mem::take(&mut flusher.handles) - }); + }; // `initialize` has rebound the list the walker reads, so the detached // boxes have no root left — in RPython the local `handles` list is // itself traced. A `flush` runs Python code, so a collection between @@ -435,8 +450,11 @@ pub fn flush_all_streams() { // Pin them for the round and read each one back, the way // `autoflusher_add` reads its pinned stream back. The `dead_ref` // placeholder slots carry nothing to keep alive. - let handles: Vec = - handles.into_iter().filter(|slot| !slot.is_null()).collect(); + let handles: Vec = handles + .into_iter() + .filter(|slot| *slot != 0) + .map(|slot| slot as PyObjectRef) + .collect(); let _handle_roots = pyre_object::gc_roots::push_roots(); let handles_root = pyre_object::gc_roots::shadow_stack_len(); for &handle in &handles { diff --git a/pyre/pyre-interpreter/src/module/posix/interp_posix.rs b/pyre/pyre-interpreter/src/module/posix/interp_posix.rs index bee77c65ecf..0b0e4deb2ef 100644 --- a/pyre/pyre-interpreter/src/module/posix/interp_posix.rs +++ b/pyre/pyre-interpreter/src/module/posix/interp_posix.rs @@ -54,12 +54,117 @@ pub struct W_DirEntry { pub enum_type: i32, } +/// Native owner for `posix.ScandirIterator` entries and enumeration state. +/// PyPy's `interp_scandir.W_ScandirIterator` keeps the equivalent state on +/// `dirp`; its typedef exposes operations rather than these fields. +#[crate::pyre_class("posix.ScandirIterator")] +#[derive(Default)] +pub struct W_ScandirIterator { + pub entries: PyObjectRef, + pub index: i64, + pub open: bool, + /// `W_ScandirIterator._in_next` (interp_scandir.py:86). + pub in_next: bool, +} + static APPLEVEL_FORK_CALLBACKS: LazyLock> = LazyLock::new(|| Mutex::new(ApplevelForkCallbacks::default())); // PyPy's GIL serializes concurrent fork entry. Pyre is free-threaded, so the // corresponding process operation has its own narrow serializer. static FORK_SERIALIZER: Mutex<()> = Mutex::new(()); +// `_in_next`'s test-and-set is indivisible under PyPy's GIL. Pyre is +// free-threaded, so every borrow of the native scandir iterator takes this +// narrow serializer. Claiming, taking, and releasing are separate serialized +// accesses, so a second thread arriving during a claimed step observes +// `_in_next` and is refused as interp_scandir.py:133-135 requires. +static SCANDIR_IN_NEXT_SERIALIZER: Mutex<()> = Mutex::new(()); + +fn require_env_mapping( + mapping: PyObjectRef, + function: &str, + accepts_none: bool, +) -> Result<(), crate::PyError> { + if crate::baseobjspace::py_mapping_check(mapping) { + return Ok(()); + } + let none_tail = if accepts_none { " or None" } else { "" }; + Err(crate::PyError::type_error(format!( + "{function}: environment must be a mapping object{none_tail}" + ))) +} + +/// The `key=value` byte entries an exec takes from `mapping`, in the order its +/// `keys()` and `values()` hold them. +/// +/// Both sequences are snapshotted before any element is encoded, so a +/// `__fspath__` running during the encoding cannot make a later read observe a +/// mutation it performed. How many variables there are is the mapping's own +/// `len()`, so a snapshot too short to cover it is an error rather than a +/// quietly shorter environment. +/// +/// `function` names the caller in the errors, and `accepts_none` spells the +/// message for an entry point that also takes `None` — what `None` means is +/// decided before the call, never here. +fn collect_env_entries( + mapping: PyObjectRef, + function: &str, + accepts_none: bool, +) -> Result>, crate::PyError> { + let _env_roots = pyre_object::gc_roots::push_roots(); + let mapping_slot = pyre_object::gc_roots::pin_roots(&[mapping]); + require_env_mapping( + pyre_object::gc_roots::shadow_stack_get(mapping_slot), + function, + accepts_none, + )?; + let pair_count = + crate::baseobjspace::len_w(pyre_object::gc_roots::shadow_stack_get(mapping_slot))? as usize; + let mut bases = [0usize; 2]; + let mut lengths = [0usize; 2]; + for (i, method) in ["keys", "values"].into_iter().enumerate() { + let sequence = crate::baseobjspace::call_method( + pyre_object::gc_roots::shadow_stack_get(mapping_slot), + method, + &[], + ); + if sequence.is_null() { + return Err(crate::call::take_call_error().unwrap_or_else(|| { + crate::PyError::type_error(format!("{function}: env must be a mapping")) + })); + } + let items = crate::baseobjspace::unpackiterable(sequence, -1)?; + bases[i] = pyre_object::gc_roots::pin_roots(&items); + lengths[i] = items.len(); + } + // Capacity follows the available snapshots, while iteration still uses the + // mapping's reported length and rejects a snapshot too short to cover it. + let mut env = Vec::with_capacity(pair_count.min(lengths[0]).min(lengths[1])); + for i in 0..pair_count { + if i >= lengths[0] || i >= lengths[1] { + return Err(crate::PyError::index_error("list index out of range")); + } + let key = crate::gateway::fsencode_bytes_w(pyre_object::gc_roots::shadow_stack_get( + bases[0] + i, + ))?; + let value = crate::gateway::fsencode_bytes_w(pyre_object::gc_roots::shadow_stack_get( + bases[1] + i, + ))?; + // PyPy's `_env2interp` permits the Windows `=C:` form and rejects `=` + // only after the first byte. + if key.is_empty() || key.get(1..).is_some_and(|tail| tail.contains(&b'=')) { + return Err(crate::PyError::value_error( + "illegal environment variable name", + )); + } + let mut entry = key; + entry.push(b'='); + entry.extend_from_slice(&value); + env.push(entry); + } + Ok(env) +} + #[cfg(all(unix, feature = "host_env", not(target_os = "redox")))] fn sysconf_names() -> &'static [(&'static str, i32)] { &[ @@ -952,14 +1057,12 @@ fn create_environ() -> pyre_object::PyObjectRef { pyre_object::gc_roots::shadow_stack_get(dict_slot) } -/// posix stub — PyPy: pypy/module/posix/ interp_posix.py -/// -/// Provides the minimal surface that os.py module init needs to succeed. -/// Real posix calls are not implemented — they raise or return defaults. /// `posix_fspath` / `PyOS_FSPath` — `str` and `bytes` pass through unchanged -/// (the protocol's identity case); any other object is resolved through -/// `type(path).__fspath__(path)`. -pub(crate) fn fspath(arg: pyre_object::PyObjectRef) -> Result { +/// (the protocol's identity case); any other object has `type(path).__fspath__` +/// bound before it is called. +pub(crate) fn fspath( + arg: pyre_object::PyObjectRef, +) -> Result { // `str` and `bytes` only — a `bytearray` is a readable buffer and not a // path, so it goes on to be rejected below. unsafe { @@ -967,14 +1070,46 @@ pub(crate) fn fspath(arg: pyre_object::PyObjectRef) -> Result Result Result { Ok(args[0]) } - fn scandir_iter_close(_args: &[PyObjectRef]) -> Result { + + /// Every mutable borrow of the native iterator is derived, used and dropped + /// inside the serializer, so no two callers ever hold overlapping + /// references to the same `W_ScandirIterator`. + fn with_scandir_iter( + self_obj: PyObjectRef, + body: impl FnOnce(&mut W_ScandirIterator) -> R, + ) -> Option { + let _serialized = SCANDIR_IN_NEXT_SERIALIZER + .lock() + .unwrap_or_else(|poisoned| poisoned.into_inner()); + W_ScandirIterator::from_obj(self_obj).map(body) + } + + fn scandir_iter_mark_closed(self_obj: PyObjectRef) { + // `W_ScandirIterator._close` clears the state inspected by + // `_finalize_`, whether closure is explicit or due to exhaustion. + let _ = with_scandir_iter(self_obj, |iterator| { + iterator.open = false; + }); + } + fn scandir_iter_close(args: &[PyObjectRef]) -> Result { + scandir_iter_mark_closed(args[0]); Ok(pyre_object::w_none()) } + /// What a `next()` may do, decided in one serialized read of the + /// iterator's flags. + enum ScandirStep { + /// Enumeration is over, either by `close()` or by exhaustion. + Ended, + /// Another step holds `_in_next` (interp_scandir.py:133-135). + InProgress, + /// This call owns the step and must release it. + Claimed, + } + + /// `_in_next` around one enumeration step: `true` on the way in, `false` on + /// every way out (interp_scandir.py:136,158). The open flag is read in the + /// same serialized region, so the answer names a state no concurrent + /// `close()` can be halfway through. + fn scandir_iter_claim_next(self_obj: PyObjectRef) -> Option { + with_scandir_iter(self_obj, |iterator| { + // `W_ScandirIterator.next_w` ends enumeration after `close()`, without + // yielding entries already buffered in the native owner. + if !iterator.open { + return ScandirStep::Ended; + } + if iterator.in_next { + return ScandirStep::InProgress; + } + iterator.in_next = true; + ScandirStep::Claimed + }) + } + + fn scandir_iter_release_next(self_obj: PyObjectRef) { + let _ = with_scandir_iter(self_obj, |iterator| { + iterator.in_next = false; + }); + } + + /// One enumeration step, with the step already claimed. + fn scandir_iter_next_entry(self_obj: PyObjectRef) -> Result { + with_scandir_iter(self_obj, |iterator| { + let idx = iterator.index; + let entries = iterator.entries; + let len = unsafe { pyre_object::w_list_len(entries) } as i64; + if idx >= len { + iterator.open = false; + return Err(crate::PyError::stop_iteration()); + } + let Some(item) = (unsafe { pyre_object::w_list_getitem(entries, idx) }) else { + iterator.open = false; + return Err(crate::PyError::stop_iteration()); + }; + iterator.index = idx + 1; + Ok(item) + }) + .unwrap_or_else(|| { + Err(crate::PyError::type_error( + "expected a 'posix.ScandirIterator' object", + )) + }) + } + fn scandir_iter_next(args: &[PyObjectRef]) -> Result { let self_obj = args[0]; - // The type carries an instance dict, so `_index` is writable from - // Python and cannot be assumed to still hold the int this iterator - // stored. - let idx = - crate::baseobjspace::int_w(crate::baseobjspace::getattr_str(self_obj, "_index")?)?; - let entries = crate::baseobjspace::getattr_str(self_obj, "_entries")?; - let len = unsafe { pyre_object::w_list_len(entries) } as i64; - if idx >= len { - return Err(crate::PyError::stop_iteration()); - } - let item = unsafe { pyre_object::w_list_getitem(entries, idx) } - .ok_or_else(crate::PyError::stop_iteration)?; - let _ = - crate::baseobjspace::setattr_str(self_obj, "_index", pyre_object::w_int_new(idx + 1)); - Ok(item) + let step = scandir_iter_claim_next(self_obj).ok_or_else(|| { + crate::PyError::type_error("expected a 'posix.ScandirIterator' object") + })?; + match step { + ScandirStep::Ended => return Err(crate::PyError::stop_iteration()), + // interp_scandir.py:133-135 refuses a step taken while another is + // in progress, and refuses it through `fail`, which closes the + // iterator before raising. Without this two steps read one `index` + // and hand out the same entry twice. + ScandirStep::InProgress => { + scandir_iter_mark_closed(self_obj); + return Err(crate::PyError::runtime_error( + "cannot use ScandirIterator from multiple threads concurrently", + )); + } + ScandirStep::Claimed => {} + } + let result = scandir_iter_next_entry(self_obj); + scandir_iter_release_next(self_obj); + result + } + fn scandir_iter_is_open(self_obj: PyObjectRef) -> bool { + with_scandir_iter(self_obj, |iterator| iterator.open).unwrap_or(false) + } + fn scandir_iter_del(args: &[PyObjectRef]) -> Result { + let self_obj = args[0]; + if !scandir_iter_is_open(self_obj) { + return Ok(pyre_object::w_none()); + } + + let message = match unsafe { crate::display::py_repr_wtf8(self_obj) } { + Ok(repr) => format!( + "unclosed scandir iterator {}", + repr.to_string_lossy() + ), + Err(_) => "unclosed scandir iterator".to_string(), + }; + if let Err(mut error) = crate::warn::warn_category(&message, "ResourceWarning", 1) { + // `W_ScandirIterator._finalize_` reports a warning promoted to an + // error as unraisable because finalization cannot propagate it. + error.write_unraisable( + pyre_object::w_none(), + rustpython_wtf8::Wtf8::new(""), + self_obj, + ); + } + scandir_iter_mark_closed(self_obj); + Ok(pyre_object::w_none()) } fn scandir_iter_type() -> PyObjectRef { static CELL: std::sync::OnceLock = std::sync::OnceLock::new(); *CELL.get_or_init(|| { // `interp_scandir.py:173` names the typedef `'posix.ScandirIterator'`. - let tp = crate::typedef::make_builtin_type("posix.ScandirIterator", |ns| { - for (name, f) in [ - ( - "__iter__", - scandir_iter_self as crate::gateway::BuiltinCodeFn, - ), - ("__next__", scandir_iter_next), - ("__enter__", scandir_iter_self), - ("__exit__", scandir_iter_close), - ("close", scandir_iter_close), - ] { - unsafe { - pyre_object::dictmultiobject::w_dict_setitem_str_no_proxy( - ns, - name, - crate::make_builtin_function(name, f), - ) - }; - } - }); - unsafe { pyre_object::typeobject::w_type_set_hasdict(tp, true) }; - // `interp_scandir.py:172-180` declares no `__new__` on the typedef - // and `:180` sets `acceptable_as_base_class = False`. The iterator - // is produced only by `scandir_fn` below, through - // `pyre_object::w_instance_new`. + let tp = crate::typedef::make_builtin_type_with_layout( + "posix.ScandirIterator", + |ns| { + for (name, f) in [ + ( + "__iter__", + scandir_iter_self as crate::gateway::BuiltinCodeFn, + ), + ("__next__", scandir_iter_next), + ("__enter__", scandir_iter_self), + ("__exit__", scandir_iter_close), + ("close", scandir_iter_close), + // `interp_scandir.py:172-180` keeps finalization on the + // RPython-internal `_finalize_` and publishes no + // `__del__`. 3.14 makes `__del__` a real entry in + // `posix.ScandirIterator`'s type dict, so it is + // published here, with the arity-1 binding below that + // makes it callable as an ordinary method. + ("__del__", scandir_iter_del), + ] { + let function = if name == "__del__" { + crate::make_builtin_function_with_arity(name, f, 1) + } else { + crate::make_builtin_function(name, f) + }; + unsafe { + pyre_object::dictmultiobject::w_dict_setitem_str_no_proxy( + ns, name, function, + ) + }; + } + }, + crate::typedef::w_object(), + ::PYTYPE, + ); + pyre_object::pyobject::set_instantiate( + unsafe { + &*::PYTYPE + }, + tp, + ); + unsafe { pyre_object::w_type_set_hasuserdel(tp, true) }; + // PyPy's `W_ScandirIterator.typedef` has no `__new__` and disallows + // subclassing; `scandir_fn` creates instances with + // `W_ScandirIterator::allocate_stable`. unsafe { pyre_object::typeobject::w_type_set_disallow_instantiation(tp); pyre_object::typeobject::w_type_set_acceptable_as_base_class(tp, false); @@ -4998,15 +5272,24 @@ pub fn register_module(ns: pyre_object::PyObjectRef) { } } } - // Pin the iterator so the `_entries`/`_index` setattr allocations cannot - // strand it, re-reading `it`/`list` from their slots after each. - pyre_object::gc_roots::pin_root(pyre_object::w_instance_new(scandir_iter_type())); + // Initialise the iterator type before allocating its native owner, then + // pin that stable owner while connecting it to the entries list. + let _ = scandir_iter_type(); + pyre_object::gc_roots::pin_root(W_ScandirIterator::allocate_stable( + W_ScandirIterator::default(), + )); let it_slot = pyre_object::gc_roots::shadow_stack_len() - 1; let it = pyre_object::gc_roots::shadow_stack_get(it_slot); let list = pyre_object::gc_roots::shadow_stack_get(list_slot); - let _ = crate::baseobjspace::setattr_str(it, "_entries", list); - let it = pyre_object::gc_roots::shadow_stack_get(it_slot); - let _ = crate::baseobjspace::setattr_str(it, "_index", pyre_object::w_int_new(0)); + let iterator = W_ScandirIterator::from_obj(it) + .expect("freshly allocated posix.ScandirIterator"); + iterator.entries = list; + iterator.open = true; + unsafe { pyre_object::gc_hook::try_gc_write_barrier(it as *mut u8) }; + // `StdObjSpace.allocate_instance` immediately queues instances whose + // type has `hasuserdel`. This native allocation bypasses that helper, + // so it must register the new iterator explicitly. + pyre_object::gc_hook::maybe_register_finalizer(it); let it = pyre_object::gc_roots::shadow_stack_get(it_slot); drop(_list_scope); Ok(it) @@ -5474,32 +5757,16 @@ pub fn register_module(ns: pyre_object::PyObjectRef) { let argv = exec_argv(args[1], "execve")?; let argv_ptrs = exec_pointer_array(&argv); - let keys_obj = crate::baseobjspace::call_method(args[2], "keys", &[]); - if keys_obj.is_null() { - return Err(crate::call::take_call_error().unwrap_or_else(|| { - crate::PyError::type_error("execve: env must be a mapping") - })); - } - let keys = crate::baseobjspace::unpackiterable(keys_obj, -1)?; - let mut env = Vec::with_capacity(keys.len()); - for key_obj in keys { - let value_obj = crate::baseobjspace::getitem(args[2], key_obj)?; - let key = extract_path(key_obj)?; - let value = extract_path(value_obj)?; - if key.is_empty() || key.get(1..).is_some_and(|tail| tail.contains(&b'=')) { - return Err(crate::PyError::value_error( - "illegal environment variable name", - )); - } - let mut entry = key; - entry.push(b'='); - entry.extend_from_slice(&value); - env.push(std::ffi::CString::new(entry).map_err(|_| { - crate::PyError::value_error( - "execve() environment contains an embedded null byte", - ) - })?); - } + let env = collect_env_entries(args[2], "execve", false)? + .into_iter() + .map(|entry| { + std::ffi::CString::new(entry).map_err(|_| { + crate::PyError::value_error( + "execve() environment contains an embedded null byte", + ) + }) + }) + .collect::, _>>()?; let env_ptrs = exec_pointer_array(&env); let errno = host_posix::exec_replace( &[command_c], @@ -6129,6 +6396,18 @@ pub fn register_module(ns: pyre_object::PyObjectRef) { ), ); + // PyPy's `_run_forking_function` enters the callback lifecycle + // immediately. CPython 3.14 checks finalization first, so a refused + // fork takes no callback lock and signals no thread. + fn guard_fork_finalization() -> Result<(), crate::PyError> { + if !crate::module::thread::is_finalizing() { + return Ok(()); + } + Err(crate::builtins::finalization_error(Some( + "can't fork at interpreter shutdown", + ))) + } + // os.fork() -> child pid in parent, 0 in child crate::module_ns_store( ns, @@ -6136,6 +6415,7 @@ pub fn register_module(ns: pyre_object::PyObjectRef) { crate::make_builtin_function_with_arity( "fork", |_| { + guard_fork_finalization()?; if majit_gc::gc_sync::registered_threads() > 1 { crate::warn::warn_deprecation( "This process is multi-threaded, use of fork() may lead to deadlocks", @@ -6212,6 +6492,7 @@ pub fn register_module(ns: pyre_object::PyObjectRef) { crate::make_builtin_function_with_arity( "forkpty", |_| { + guard_fork_finalization()?; if majit_gc::gc_sync::registered_threads() > 1 { crate::warn::warn_deprecation( "This process is multi-threaded, use of forkpty() may lead to deadlocks", @@ -6347,6 +6628,105 @@ pub fn register_module(ns: pyre_object::PyObjectRef) { ), ); + // PyPy's six user/group ID setters share the `c_uid_t` conversion and + // report syscall errors without retrying. + #[cfg(not(feature = "sandbox"))] + fn set_one_id( + args: &[PyObjectRef], + name: &str, + setter: fn(u32) -> libc::c_int, + ) -> Result { + let id = match args.first() { + Some(&obj) => crate::baseobjspace::c_uid_t_w(obj)?, + None => { + return Err(crate::PyError::type_error(format!( + "{name}() requires 1 argument" + ))); + } + }; + if setter(id) == -1 { + return Err(io_err(std::io::Error::last_os_error(), "")); + } + Ok(pyre_object::w_none()) + } + + #[cfg(not(feature = "sandbox"))] + fn set_two_ids( + args: &[PyObjectRef], + name: &str, + setter: fn(u32, u32) -> libc::c_int, + ) -> Result { + if args.len() < 2 { + return Err(crate::PyError::type_error(format!( + "{name}() requires 2 arguments" + ))); + } + let first = crate::baseobjspace::c_uid_t_w(args[0])?; + let second = crate::baseobjspace::c_uid_t_w(args[1])?; + if setter(first, second) == -1 { + return Err(io_err(std::io::Error::last_os_error(), "")); + } + Ok(pyre_object::w_none()) + } + + #[cfg(not(feature = "sandbox"))] + fn setuid(args: &[PyObjectRef]) -> Result { + set_one_id(args, "setuid", |uid| unsafe { + libc::setuid(uid as libc::uid_t) + }) + } + + #[cfg(not(feature = "sandbox"))] + fn seteuid(args: &[PyObjectRef]) -> Result { + set_one_id(args, "seteuid", |euid| unsafe { + libc::seteuid(euid as libc::uid_t) + }) + } + + #[cfg(not(feature = "sandbox"))] + fn setgid(args: &[PyObjectRef]) -> Result { + set_one_id(args, "setgid", |gid| unsafe { + libc::setgid(gid as libc::gid_t) + }) + } + + #[cfg(not(feature = "sandbox"))] + fn setegid(args: &[PyObjectRef]) -> Result { + set_one_id(args, "setegid", |egid| unsafe { + libc::setegid(egid as libc::gid_t) + }) + } + + #[cfg(not(feature = "sandbox"))] + fn setreuid(args: &[PyObjectRef]) -> Result { + set_two_ids(args, "setreuid", |ruid, euid| unsafe { + libc::setreuid(ruid as libc::uid_t, euid as libc::uid_t) + }) + } + + #[cfg(not(feature = "sandbox"))] + fn setregid(args: &[PyObjectRef]) -> Result { + set_two_ids(args, "setregid", |rgid, egid| unsafe { + libc::setregid(rgid as libc::gid_t, egid as libc::gid_t) + }) + } + + #[cfg(not(feature = "sandbox"))] + for (name, function, arity) in [ + ("setuid", setuid as crate::gateway::BuiltinCodeFn, 1), + ("seteuid", seteuid as crate::gateway::BuiltinCodeFn, 1), + ("setgid", setgid as crate::gateway::BuiltinCodeFn, 1), + ("setegid", setegid as crate::gateway::BuiltinCodeFn, 1), + ("setreuid", setreuid as crate::gateway::BuiltinCodeFn, 2), + ("setregid", setregid as crate::gateway::BuiltinCodeFn, 2), + ] { + crate::module_ns_store( + ns, + name, + crate::make_builtin_function_with_arity(name, function, arity), + ); + } + // `interp_posix.py:2603-2608` — the controlling terminal's name, which // `rposix.py:1724-1728` reads by handing the call a null pointer and // taking the static buffer it answers with. It is a filename, so it is @@ -8149,10 +8529,11 @@ pub fn register_module(ns: pyre_object::PyObjectRef) { crate::make_builtin_function("sendfile", |args| { use std::os::fd::BorrowedFd; // Every parameter is positional-or-keyword. `headers`, - // `trailers` and `flags` are the BSD `sendfile(2)` tail, which - // neither arm below passes on; they are named here so a - // caller that supplies them is bound rather than truncated, - // and so an unknown keyword is an error. + // `trailers` and `flags` are the BSD `sendfile(2)` tail. The + // macOS arm forwards both vectors; `flags` alone remains + // unused because the host wrapper exposes no flags parameter. + // Listing the BSD-only parameters makes unknown keywords fail + // during argument binding on every platform. let (bound, _kwargs) = bind_path_args( args, "sendfile", @@ -8228,6 +8609,97 @@ pub fn register_module(ns: pyre_object::PyObjectRef) { } #[cfg(target_os = "macos")] { + // Both Python sequences and all of their buffer exports are + // consumed before entering the EINTR retry loop. The retry + // therefore reuses only Rust-owned bytes. + let (header_buffers, trailer_buffers) = { + let _roots = pyre_object::gc_roots::push_roots(); + let header_slot = bound[4].map(|value| { + let slot = pyre_object::gc_roots::shadow_stack_len(); + pyre_object::gc_roots::pin_root(value); + slot + }); + let trailer_slot = bound[5].map(|value| { + let slot = pyre_object::gc_roots::shadow_stack_len(); + pyre_object::gc_roots::pin_root(value); + slot + }); + let collect_buffers = |slot: Option, name: &str| { + let Some(slot) = slot else { + return Ok(None); + }; + let value = pyre_object::gc_roots::shadow_stack_get(slot); + if unsafe { pyre_object::is_none(value) } { + return Ok(None); + } + // Indexed header/trailer vectors require a sequence; + // consuming an iterator or mapping keys would change + // the accepted `sendfile` argument protocol. + if !crate::baseobjspace::issequence_w(value) { + return Err(crate::PyError::type_error(format!( + "sendfile() {name} must be a sequence" + ))); + } + let items = crate::baseobjspace::unpackiterable(value, -1)?; + let items_base = pyre_object::gc_roots::pin_roots(&items); + let mut buffers = Vec::with_capacity(items.len()); + for index in 0..items.len() { + let item = + pyre_object::gc_roots::shadow_stack_get(items_base + index); + let Some(buffer) = + crate::baseobjspace::simple_buffer_bytes(item)? + else { + return Err(crate::PyError::type_error(format!( + "sendfile() {name} items must be bytes-like" + ))); + }; + buffers.push(buffer.as_bytes().to_vec()); + buffer.release(); + } + if buffers.is_empty() { + Ok(None) + } else { + Ok(Some(buffers)) + } + }; + ( + collect_buffers(header_slot, "headers")?, + collect_buffers(trailer_slot, "trailers")?, + ) + }; + // An empty sequence is indistinguishable from an absent + // one at the syscall boundary, independently for headers + // and trailers. + let header_slices = header_buffers.as_ref().map(|buffers| { + buffers + .iter() + .map(Vec::as_slice) + .collect::>() + }); + let trailer_slices = trailer_buffers.as_ref().map(|buffers| { + buffers + .iter() + .map(Vec::as_slice) + .collect::>() + }); + // `sendfile(2)` on this host spends the length cell on the + // header and the file together — "the value of len argument + // indicates the maximum number of bytes in the header + // and/or file to be sent" — so a caller asking for `count` + // bytes of the file has to be given room for its headers on + // top, or the headers eat into the range it asked for. The + // trailer is outside the budget and is always sent whole. + // A count of 0 already asks for everything and stays 0. + let count = match header_buffers.as_ref() { + Some(buffers) if count_raw != 0 => { + buffers.iter().try_fold(count_raw, |count, buffer| { + count.checked_add(buffer.len() as i64).ok_or_else(|| { + crate::PyError::overflow_error("sendfile() count is too large") + }) + })? + } + _ => count_raw, + }; loop { let (res, written) = { let _blocked = crate::module::thread::before_external_block(); @@ -8235,9 +8707,9 @@ pub fn register_module(ns: pyre_object::PyObjectRef) { in_b, out_b, offset_i64 as rustpython_host_env::crt_fd::Offset, - count_raw, - None, - None, + count, + header_slices.as_deref(), + trailer_slices.as_deref(), ) }; match res { @@ -8318,10 +8790,9 @@ pub fn register_module(ns: pyre_object::PyObjectRef) { crate::PyError::value_error("posix_spawn: embedded null in path") })?; let argv = collect_cstring_seq(positional[1], "posix_spawn", "argv")?; - // posixmodule.c parses `env` as a mapping. This is the same - // owner/shape as PyPy's `_env2interp` path used by execve: - // iterate `keys()`, fetch each value through `getitem`, then - // filesystem-encode both sides into `key=value`. + // posixmodule.c parses `env` through the same keys/values + // snapshot used by execve, then filesystem-encodes paired + // elements into `key=value`. let env = collect_spawn_env(positional[2])?; let file_actions_obj = crate::builtins::kwarg_get(kwargs, "file_actions"); let actions: Vec = @@ -8356,70 +8827,31 @@ pub fn register_module(ns: pyre_object::PyObjectRef) { // CPython's posix_spawn accepts None as "inherit environ"; // subprocess._posix_spawn uses exactly this form when Popen // was called without an explicit env mapping. - if unsafe { pyre_object::is_none(mapping) } { - let mut env = Vec::new(); - for (key, value) in host_os::vars_os() { - let key = key.as_encoded_bytes(); - let value = value.as_encoded_bytes(); - let mut entry = Vec::with_capacity(key.len() + 1 + value.len()); - entry.extend_from_slice(key); - entry.push(b'='); - entry.extend_from_slice(value); - env.push(std::ffi::CString::new(entry).map_err(|_| { + let entries = if unsafe { pyre_object::is_none(mapping) } { + host_os::vars_os() + .map(|(key, value)| { + let key = key.as_encoded_bytes(); + let value = value.as_encoded_bytes(); + let mut entry = Vec::with_capacity(key.len() + 1 + value.len()); + entry.extend_from_slice(key); + entry.push(b'='); + entry.extend_from_slice(value); + entry + }) + .collect() + } else { + collect_env_entries(mapping, "posix_spawn", true)? + }; + entries + .into_iter() + .map(|entry| { + std::ffi::CString::new(entry).map_err(|_| { crate::PyError::value_error( "posix_spawn() environment contains an embedded null byte", ) - })?); - } - return Ok(env); - } - let keys_obj = crate::baseobjspace::call_method(mapping, "keys", &[]); - if keys_obj.is_null() { - return Err(crate::call::take_call_error().unwrap_or_else(|| { - crate::PyError::type_error("posix_spawn: env must be a mapping") - })); - } - let keys = crate::baseobjspace::unpackiterable(keys_obj, -1)?; - // `getitem` runs the mapping's `__getitem__` and both encodes - // allocate, so the mapping and every key are published once and - // read back per iteration rather than kept in plain locals. - let _env_roots = pyre_object::gc_roots::push_roots(); - let mapping_slot = pyre_object::gc_roots::pin_roots(&[mapping]); - let keys_base = pyre_object::gc_roots::pin_roots(&keys); - let mut env = Vec::with_capacity(keys.len()); - for i in 0..keys.len() { - let _entry_roots = pyre_object::gc_roots::push_roots(); - let value_obj = crate::baseobjspace::getitem( - pyre_object::gc_roots::shadow_stack_get(mapping_slot), - pyre_object::gc_roots::shadow_stack_get(keys_base + i), - )?; - // Encoding the key can collect, so the value it was fetched - // beside has to be published before that call. - let value_slot = pyre_object::gc_roots::pin_roots(&[value_obj]); - let key = crate::gateway::fsencode_bytes_w( - pyre_object::gc_roots::shadow_stack_get(keys_base + i), - )?; - let value = crate::gateway::fsencode_bytes_w( - pyre_object::gc_roots::shadow_stack_get(value_slot), - )?; - // interp_posix.py:1762-1769 permits the Windows `=C:` - // spelling and rejects `=` only after the first byte. - if key.is_empty() || key.get(1..).is_some_and(|tail| tail.contains(&b'=')) { - return Err(crate::PyError::value_error( - "illegal environment variable name", - )); - } - let mut entry = Vec::with_capacity(key.len() + 1 + value.len()); - entry.extend_from_slice(&key); - entry.push(b'='); - entry.extend_from_slice(&value); - env.push(std::ffi::CString::new(entry).map_err(|_| { - crate::PyError::value_error( - "posix_spawn() environment contains an embedded null byte", - ) - })?); - } - Ok(env) + }) + }) + .collect() } fn collect_cstring_seq( obj: pyre_object::PyObjectRef, @@ -9537,35 +9969,17 @@ pub fn register_module(ns: pyre_object::PyObjectRef) { let argv = exec_argv_wide(args[1], "execve")?; let argv_ptrs = exec_pointer_array_wide(&argv); - let keys_obj = crate::baseobjspace::call_method(args[2], "keys", &[]); - if keys_obj.is_null() { - return Err(crate::call::take_call_error().unwrap_or_else(|| { - crate::PyError::type_error("execve: env must be a mapping") - })); - } - let keys = crate::baseobjspace::unpackiterable(keys_obj, -1)?; - let mut env = Vec::with_capacity(keys.len()); - for key_obj in keys { - let value_obj = crate::baseobjspace::getitem(args[2], key_obj)?; - let key = extract_path(key_obj)?; - let value = extract_path(value_obj)?; - if key.is_empty() || key.get(1..).is_some_and(|tail| tail.contains(&b'=')) { - return Err(crate::PyError::value_error( - "illegal environment variable name", - )); - } - let mut entry = key; - entry.push(b'='); - entry.extend_from_slice(&value); - env.push( + let env = collect_env_entries(args[2], "execve", false)? + .into_iter() + .map(|entry| { widestring::WideCString::from_os_str(&*os_str_from_bytes(&entry)) .map_err(|_| { crate::PyError::value_error( "execve() environment contains an embedded null byte", ) - })?, - ); - } + }) + }) + .collect::, _>>()?; let env_ptrs = exec_pointer_array_wide(&env); unsafe { libc::wexecve(command_w.as_ptr(), argv_ptrs.as_ptr(), env_ptrs.as_ptr()) @@ -10264,7 +10678,9 @@ pub fn register_module(ns: pyre_object::PyObjectRef) { "mknod", // privilege / scheduling "setuid", + "seteuid", "setgid", + "setegid", "setreuid", "setregid", "setresuid", diff --git a/pyre/pyre-interpreter/src/module/posix/mod.rs b/pyre/pyre-interpreter/src/module/posix/mod.rs index 1b7f3d7ba94..a21d59d7e22 100644 --- a/pyre/pyre-interpreter/src/module/posix/mod.rs +++ b/pyre/pyre-interpreter/src/module/posix/mod.rs @@ -6,4 +6,4 @@ crate::pyre_module_init!(interp_posix); -pub use interp_posix::W_DirEntry; +pub use interp_posix::{W_DirEntry, W_ScandirIterator}; diff --git a/pyre/pyre-interpreter/src/module/thread/mod.rs b/pyre/pyre-interpreter/src/module/thread/mod.rs index e68a1b9ffa5..46495490e38 100644 --- a/pyre/pyre-interpreter/src/module/thread/mod.rs +++ b/pyre/pyre-interpreter/src/module/thread/mod.rs @@ -1142,10 +1142,7 @@ mod handle_class { return Ok(()); } if state.daemon && is_finalizing() { - let cls = crate::builtins::lookup_exc_class("PythonFinalizationError") - .expect("PythonFinalizationError must be installed"); - let exc = crate::builtins::exc_exception_new(&[cls])?; - return Err(unsafe { crate::PyError::from_exc_object(exc) }); + return Err(crate::builtins::finalization_error(None)); } let _blocked = before_external_block(); match duration { diff --git a/pyre/pyre-jit-trace/build.rs b/pyre/pyre-jit-trace/build.rs index e8e2eef976a..a5ca64e3531 100644 --- a/pyre/pyre-jit-trace/build.rs +++ b/pyre/pyre-jit-trace/build.rs @@ -40,10 +40,6 @@ const CODEGEN_OUTPUTS: &[&str] = &[ "static_ref_bindings.bin", ]; -/// Lowering switches read by `majit-translate` while this build script runs. -/// They affect generated graphs, so Cargo and the content cache must both see -/// their values; otherwise an A/B can silently restore the opposite setting. - /// Outputs carrying this build-script process's own addresses, by /// construction. ASLR moves them on every process, so two processes always /// disagree and that disagreement is not a defect. diff --git a/pyre/pyre-jit-trace/src/jitcode_dispatch/residual_call.rs b/pyre/pyre-jit-trace/src/jitcode_dispatch/residual_call.rs index 9a842c4c679..80c4d809108 100644 --- a/pyre/pyre-jit-trace/src/jitcode_dispatch/residual_call.rs +++ b/pyre/pyre-jit-trace/src/jitcode_dispatch/residual_call.rs @@ -3616,9 +3616,7 @@ pub(crate) fn try_execute_residual_call_via_executor( // the inlined-call push, which closed the gap. // `synth/getframe_while_escaping_read_frame_identity` is the // regression guard. - } else if writes_live_heap - && odometer_unchanged - && ctx.fbw_mode.inline_subwalk + } else if ctx.fbw_mode.inline_subwalk && let Some(framestack) = build_multi_frame_miframe( ctx, resume_pc, @@ -4148,7 +4146,7 @@ fn live_py_pc_from_snapshot( /// Kept as a thin pass-through so the dispatcher call sites stay /// readable; collapses to direct `walker_*` once the dispatchers /// inline. -fn maybe_record_inline_callee_last_instr( +pub(crate) fn maybe_record_inline_callee_last_instr( ctx: &mut WalkContext<'_, '_, Sym>, jit_pc: usize, ) { diff --git a/pyre/pyre-jit-trace/src/jitcode_dispatch/specialize.rs b/pyre/pyre-jit-trace/src/jitcode_dispatch/specialize.rs index 5cc4259df32..7e87180c0cf 100644 --- a/pyre/pyre-jit-trace/src/jitcode_dispatch/specialize.rs +++ b/pyre/pyre-jit-trace/src/jitcode_dispatch/specialize.rs @@ -2289,6 +2289,13 @@ fn traceback_walk_field( None } +/// Runtime half of the optimized-frame `f_locals` getter. The proxy owns the +/// exact frame passed to it; reading or mutating the proxy later goes through +/// that frame's existing synchronization path. +extern "C" fn jit_inline_frame_locals_proxy_new(frame: i64) -> i64 { + pyre_interpreter::pyframe::frame_locals_proxy::new(frame as pyre_object::PyObjectRef) as i64 +} + /// Prove the receiving code object still owns its host `CodeObject`, the /// `require_code` check every code-field getter runs before reading a slot. fn walker_guard_code_ptr_present( @@ -2563,6 +2570,52 @@ pub(crate) fn try_walker_specialize_load_attr( let Some(name) = walker_load_name_from_code(w_code_ptr, name_idx) else { return Ok(None); }; + // CPython 3.14 exposes an optimized frame's locals as a fresh + // `FrameLocalsProxy`. Constructing that proxy does not read fast locals; + // its operations synchronize through the frame when they are actually + // used. Keep the inline callee's own red frame as the proxy owner instead + // of residualizing the getter, whose explicit read barrier would force the + // outer standard virtualizable while this MIFrame is still active. + let inline_frame = current_inline_concrete_frame(); + if name == "f_locals" + && inline_frame != 0 + && concrete_obj as usize == inline_frame + && unsafe { (*concrete_obj).ob_type } == &pyre_interpreter::pyframe::FRAME_TYPE + && unsafe { + (*(concrete_obj as *const pyre_interpreter::PyFrame)) + .code() + .flags + .contains(pyre_interpreter::CodeFlags::OPTIMIZED) + } + && ctx + .callee_shadow + .as_ref() + .is_some_and(|shadow| shadow.concrete_frame == inline_frame && shadow.frame_box == obj) + { + let w_type = + pyre_interpreter::typedef::gettypeobject(&pyre_interpreter::pyframe::FRAME_TYPE); + let version_tag = unsafe { pyre_object::typeobject::w_type_get_version_tag(w_type) }; + if version_tag == 0 || unsafe { (*concrete_obj).w_class } != w_type { + return Ok(None); + } + let concrete_proxy = pyre_interpreter::pyframe::frame_locals_proxy::new(concrete_obj); + walker_guard_exception_attr_slot(ctx, op_pc, obj, concrete_obj, w_type, version_tag)?; + let proxy = ctx.trace_ctx.call_ref_typed_with_effect( + jit_inline_frame_locals_proxy_new as *const (), + &[obj], + &[majit_ir::Type::Ref], + majit_ir::EffectInfo::new( + majit_ir::ExtraEffect::CannotRaise, + majit_ir::OopSpecIndex::None, + ), + ); + ctx.trace_ctx.set_opref_concrete( + proxy, + majit_ir::Value::Ref(majit_ir::GcRef(concrete_proxy as usize)), + ); + write_residual_call_result_to_dst(ctx, op_pc, dst, dst_bank, proxy)?; + return Ok(Some(())); + } // `mapdict.py` resolution, returning the fold ingredients (the // read is left to the caller so it can be folded to a guarded inline read). if let Some((w_type, version_tag, map, storageindex)) = unsafe { @@ -8321,22 +8374,13 @@ pub(crate) fn try_walker_specialize_builtin_locals( /// land on the same frame. Any other chain declines, and at runtime a /// `topframeref` that stops matching side-exits. /// -/// Returns `None` (fall through to the generic residual, SAFE — exactly -/// today's behaviour) for every other shape: a rebound `sys._getframe`, a -/// bound receiver, a negative / non-int / inexact / non-constant depth, an -/// inline sub-walk, a walk with no standard virtualizable, armed audit hooks, -/// a `topframeref` / `gettopframe_nohidden()` mismatch with the portal frame, -/// a hop whose forced `f_backref` is null, or a hop whose result is hidden. +/// Returns `None` (fall through to the generic residual) for every other +/// shape: a rebound `sys._getframe`, a bound receiver, a negative / non-int / +/// inexact / non-constant depth, a walk with no frame identity, armed audit +/// hooks, a top-level `topframeref` mismatch with the portal frame, a hop whose +/// forced `f_backref` is null, or a hop whose result is hidden. /// Declines after emission rewind to the pre-specialization trace position and /// reset the heap cache before falling through. -/// -/// ⛔ The TOP walk level is the only level this may take, at any depth. Inside -/// an inline sub-walk depth 0 names the callee's virtual frame, whose -/// `last_instr` is still the `-1` its constructor wrote and which nothing -/// updates through the inlined body (`jitcode_dispatch/mod.rs` says so in the -/// tree); depth > 0 would start the hop chain from that same frame. The -/// sub-walk gate is what makes "depth 0 == the portal" true rather than -/// assumed. pub(crate) fn try_walker_specialize_sys_getframe( ctx: &mut WalkContext<'_, '_, Sym>, code: &[u8], @@ -8385,12 +8429,6 @@ pub(crate) fn try_walker_specialize_sys_getframe( } else { (None, 0) }; - // `virtualizable_boxes` describe the PORTAL frame only. An inline sub-walk - // publishes a different concrete frame, so depth 0 there names the callee — - // the level this arm must not take. - if ctx.fbw_mode.inline_subwalk || current_inline_concrete_frame() != 0 { - return Ok(None); - } // `vm.py:51 audit(space, "sys._getframe", [f])`. With no hook installed // `audit` takes its `holder.hooks_w is None` early-out (`vm.py:481`) and the // event costs nothing; the emission below pins that read so a later @@ -8402,29 +8440,62 @@ pub(crate) fn try_walker_specialize_sys_getframe( if audit_holder.is_null() || pyre_interpreter::module::sys::vm::audit_hooks_armed() { return Ok(None); } - let (Some(vable_op), Some(vable_ptr)) = ( - ctx.trace_ctx.standard_virtualizable_box(), - ctx.trace_ctx.standard_virtualizable_ptr(), - ) else { + // Every MIFrame owns one red frame. At the root that is the standard + // virtualizable; inside an inline sub-walk it is the callee frame seeded in + // `dispatch_inline_call_dr_kind` and carried by `CalleeLocalsShadow`. + // Starting the constant-depth walk from that per-level frame is the direct + // counterpart of `ec.gettopframe_nohidden()` returning the live MIFrame's + // virtual frame upstream. + let inline_ptr = current_inline_concrete_frame(); + let inline_level = ctx.fbw_mode.inline_subwalk || inline_ptr != 0; + // A depth-zero lookup returns this MIFrame's own red frame directly. A + // positive depth has to force each intervening virtual reference through + // `_do_jit_force_virtual`; until the inline walker carries that operation + // with the same per-level resume state, keep the established residual path + // instead of treating the concrete recording-time chain as its substitute. + if inline_level && depth_value != 0 { return Ok(None); + } + let (vable_op, vable_ptr) = if inline_level { + let Some(shadow) = ctx.callee_shadow.as_ref() else { + return Ok(None); + }; + if inline_ptr == 0 || shadow.concrete_frame != inline_ptr || shadow.frame_box == OpRef::NONE + { + return Ok(None); + } + (shadow.frame_box, inline_ptr) + } else { + let (Some(op), Some(ptr)) = ( + ctx.trace_ctx.standard_virtualizable_box(), + ctx.trace_ctx.standard_virtualizable_ptr(), + ) else { + return Ok(None); + }; + (op, ptr) }; let ec = pyre_interpreter::call::getexecutioncontext() as *mut pyre_interpreter::PyExecutionContext; if ec.is_null() { return Ok(None); } - // The emitted guard compares the RAW `topframeref` against the portal, so - // require the record-time chain to make that comparison equivalent to - // `getframe`'s own resolution: the slot holds the frame pointer itself - // (an inlined callee's `JitVirtualRef` would decline here, and so would a - // deeper portal), and the hidden-frame walk lands on that same frame. - if unsafe { (*ec).topframeref } as usize != vable_ptr { - return Ok(None); - } - let frame = unsafe { (*ec).gettopframe_nohidden() }; - if frame.is_null() || frame as usize != vable_ptr { - return Ok(None); - } + // At the portal, prove that the raw execution-context chain still names + // the standard frame and emit the equivalent runtime guard below. An + // inline level already has the stronger per-MIFrame identity witness: + // `frame_box` and `concrete_frame` were seeded together when that level was + // pushed, and the compiled trace carries the same box directly. + let frame = if inline_level { + inline_ptr as *mut pyre_interpreter::PyFrame + } else { + if unsafe { (*ec).topframeref } as usize != vable_ptr { + return Ok(None); + } + let frame = unsafe { (*ec).gettopframe_nohidden() }; + if frame.is_null() || frame as usize != vable_ptr { + return Ok(None); + } + frame + }; // --- emit the specialized IR (walker-native) --- let pre_emit_pos = ctx.trace_ctx.get_trace_position(); @@ -8462,34 +8533,34 @@ pub(crate) fn try_walker_specialize_sys_getframe( return Ok(None); } } - // `ec = space.getexecutioncontext()` — recovered off the portal frame, the - // same route `walker_ec_enter` takes (`inline_call.rs`), since the outer - // frame's `execution_context` is always the true one. - let ec_op = ctx.trace_ctx.record_op_with_descr( - OpCode::GetfieldGcR, - &[vable_op], - crate::descr::pyframe_execution_context_descr(), - ); - ctx.trace_ctx - .set_opref_concrete(ec_op, majit_ir::Value::Ref(majit_ir::GcRef(ec as usize))); - // `f = ec.gettopframe_nohidden()` followed by `pyjitpl.py:2166-2168`'s - // `ptr_eq(vref_box, standard_box)` + `implement_guard_value`: the identity - // this arm resolved at record time, re-checked every compiled iteration. - let topframeref_op = ctx.trace_ctx.record_op_with_descr( - OpCode::GetfieldGcR, - &[ec_op], - crate::descr::ec_topframeref_descr(), - ); - ctx.trace_ctx.set_opref_concrete( - topframeref_op, - majit_ir::Value::Ref(majit_ir::GcRef(vable_ptr)), - ); - let is_standard = ctx - .trace_ctx - .record_op(OpCode::PtrEq, &[topframeref_op, vable_op]); - ctx.trace_ctx - .set_opref_concrete(is_standard, majit_ir::Value::Int(1)); - walker_emit_fold_guard_with_snapshot(ctx, op.pc, OpCode::GuardTrue, &[is_standard])?; + if !inline_level { + // `ec = space.getexecutioncontext()` — recovered off the portal frame, + // the same route `walker_ec_enter` takes (`inline_call.rs`). + let ec_op = ctx.trace_ctx.record_op_with_descr( + OpCode::GetfieldGcR, + &[vable_op], + crate::descr::pyframe_execution_context_descr(), + ); + ctx.trace_ctx + .set_opref_concrete(ec_op, majit_ir::Value::Ref(majit_ir::GcRef(ec as usize))); + // `f = ec.gettopframe_nohidden()` followed by + // `_do_jit_force_virtual`'s standard-box identity guard. + let topframeref_op = ctx.trace_ctx.record_op_with_descr( + OpCode::GetfieldGcR, + &[ec_op], + crate::descr::ec_topframeref_descr(), + ); + ctx.trace_ctx.set_opref_concrete( + topframeref_op, + majit_ir::Value::Ref(majit_ir::GcRef(vable_ptr)), + ); + let is_standard = ctx + .trace_ctx + .record_op(OpCode::PtrEq, &[topframeref_op, vable_op]); + ctx.trace_ctx + .set_opref_concrete(is_standard, majit_ir::Value::Int(1)); + walker_emit_fold_guard_with_snapshot(ctx, op.pc, OpCode::GuardTrue, &[is_standard])?; + } let mut cur_op = vable_op; let mut cur_ptr = frame; @@ -8582,6 +8653,16 @@ pub(crate) fn try_walker_specialize_sys_getframe( cur_ptr = next_ptr; } + // A depth-zero inline result exposes this callee frame. Publish its current + // coordinate and any locals that were still held in the strict-fold shadow + // before the frame becomes observable. This is the same per-frame state + // the ordinary residual force path flushes, without forcing the outer + // portal virtualizable or aborting its trace. + if inline_level && depth_value == 0 { + maybe_record_inline_callee_last_instr(ctx, op.pc); + disarm_folded_inline_callee_after_escape(ctx, op.pc)?; + } + // `f.mark_as_escaped()` — vm.py:54. `escaped` is not one of the six fields // `interp_jit.py:25-30` declares, so the store cannot force; it is // load-bearing at `executioncontext.py:99-106 leave`, which forces the diff --git a/pyre/pyre-jit/src/eval.rs b/pyre/pyre-jit/src/eval.rs index 760afc4797c..d0d84af3fed 100644 --- a/pyre/pyre-jit/src/eval.rs +++ b/pyre/pyre-jit/src/eval.rs @@ -3850,15 +3850,11 @@ fn build_gc() -> Box { as pyre_object::lltype::PyreClassPyTypeOf>::DESCRIPTOR, ); - // `posix.DirEntry`: four inline GC edges (`w_name`/`w_path` and the cached - // `w_stat`/`w_lstat`, the latter two NULL until first requested). Appended - // after the last unconditional vtable-bearing object (`W_GcStats`) so only - // the trailing bare-`with_gc_ptrs` ids (twister, leaf storage boxes) shift, - // and those carry no census alias. `posix` is - // compiled out on wasm32, so this registration and its census aliases are - // gated to match — which is why it stays last among the rclass - // registrations: an unconditional type after it would take a different id - // on wasm32 than on a native target. + // Register `posix.DirEntry`'s four inline GC edges and + // `posix.ScandirIterator`'s entries-list edge. The entries in + // `SUBCLASS_RANGE_HIERARCHY` and `all_subclass_range_aliases` are + // native-only because `posix` is absent on wasm32; registering these after + // unconditional rclasses keeps their type IDs stable across targets. #[cfg(not(target_arch = "wasm32"))] register_pyre_class( &mut gc, @@ -3866,6 +3862,13 @@ fn build_gc() -> Box { ::DESCRIPTOR, ); + #[cfg(not(target_arch = "wasm32"))] + register_pyre_class( + &mut gc, + &mut pytype_to_tid, + ::DESCRIPTOR, + ); // `_ssl` keeps rustls objects behind opaque native pointers. Context and // MemoryBIO are subclassable native layouts, so their marker walks the // mapdict prefix; Context additionally owns Python callbacks/path values. @@ -4554,6 +4557,9 @@ fn walk_parked_exception_roots(visitor: &mut dyn FnMut(&mut majit_ir::GcRef)) { /// root at that thread's `unregister_mutator` while the holder stayed live. fn walk_immortal_store_roots(visitor: &mut dyn FnMut(&mut majit_ir::GcRef)) { walk_rbigint_parts_cache(visitor); + pyre_interpreter::module::_io::walk_autoflusher_roots(|slot| { + visit_pyobject_root(slot, visitor) + }); sre_pattern_root_walker(visitor); w_globals_stamped_code_root_walker(visitor); mapdict_method_cache_root_walker(visitor); @@ -4631,10 +4637,6 @@ fn register_thread_root_areas() { signal_handler_root_walker_area, pyre_interpreter::module::signal::interp_signal::capture_signal_handler_root_area(), ); - register( - autoflusher_root_walker_area, - pyre_interpreter::module::_io::capture_autoflusher_root_area(), - ); register( jit_callee_frame_root_walker_area, crate::call_jit::capture_jit_callee_frame_root_area(), @@ -5451,17 +5453,6 @@ unsafe fn signal_handler_root_walker_area( } } -unsafe fn autoflusher_root_walker_area( - data: *const (), - visitor: &mut dyn FnMut(&mut majit_ir::GcRef), -) { - unsafe { - pyre_interpreter::module::_io::walk_autoflusher_roots_area(data, |slot| { - visit_pyobject_root(slot, visitor); - }); - } -} - unsafe fn jit_callee_frame_root_walker_area( data: *const (), visitor: &mut dyn FnMut(&mut majit_ir::GcRef), @@ -9109,44 +9100,40 @@ fn maybe_compile_and_run( if driver.has_runnable_compiled_loop(green_key) { return execute_assembler(frame, green_key, loop_header_pc, driver, info, env); } - // Pyre-local deviation: this short-circuit treats `DONT_TRACE_HERE` as a - // permanent never-trace blacklist and returns before the counter tick. - // Upstream `warmstate.py:485-495` uses the identically named flag to force - // a separate trace when no procedure token has been seen; the ported - // behavior exists in `warmstate.rs::should_start_dont_trace_here_trace` - // and `warmstate.rs::can_inline_callable`, but this eval-layer guard - // shadows it. - if driver - .meta_interp() - .warm_state_ref() - .is_dont_trace_here(green_key) - { - return None; - } - // warmstate.py:496-511: counter.tick → threshold reached → bound_reached - // TODO(parity): warmstate.py:473-496 funnels every back-edge through - // `maybe_compile_and_run`, which checks JC_TRACING, compiled-loop - // presence, DONT_TRACE_HERE, has_seen_a_procedure_token, and - // counter.tick in one linear sequence. Pyre splits the checks - // across this function and `counter_tick_checked` (warmstate.rs:559). - // The flag-based DONT_TRACE_HERE path above duplicates part of the - // warmstate logic; verify that `counter_tick_checked` still covers - // the `has_seen_a_procedure_token` guard and the full `bound_reached` - // flow identically to warmstate.py:496-511. - if driver + // `WarmEnterState::maybe_compile_decision` is the port of warmstate.py's + // complete JitCell decision: token lookup, DONT_TRACE_HERE retry, + // dead-token cleanup, and counter tick. Keeping that policy in one symbol + // prevents the eval entry point from assigning different semantics to its + // flags. + // + // The decision alone, without the `cell.flags |= JC_TRACING` that + // warmstate.py:441 makes with it: upstream marks the cell in the same + // `bound_reached` that runs the trace, and pyre marks it there too — + // `MetaInterp::bound_reached` reaches the cell through + // `force_start_tracing_for_key`. Marking it here as well makes that call + // read this door's own mark and decline with `AlreadyTracing`, so the + // trace this door just decided on never starts and the counter it reset + // never re-arms the location. + match driver .meta_interp_mut() .warm_state_mut() - .counter_tick_checked(green_key) + .maybe_compile_decision(green_key) { - if driver - .meta_interp() - .is_tracing_key((frame.pycode as usize, loop_header_pc)) - { - return None; + majit_metainterp::warmstate::HotResult::StartTracing => { + if driver + .meta_interp() + .is_tracing_key((frame.pycode as usize, loop_header_pc)) + { + return None; + } + bound_reached(frame, green_key, loop_header_pc, driver, info, env) } - return bound_reached(frame, green_key, loop_header_pc, driver, info, env); + majit_metainterp::warmstate::HotResult::RunCompiled => { + execute_assembler(frame, green_key, loop_header_pc, driver, info, env) + } + majit_metainterp::warmstate::HotResult::NotHot + | majit_metainterp::warmstate::HotResult::AlreadyTracing => None, } - None } /// Panic-safe RAII pairing for `FailDescr::start_compiling` / diff --git a/pyre/pyre-object/src/functional.rs b/pyre/pyre-object/src/functional.rs index 49daaf9fbf5..22e06dd2483 100644 --- a/pyre/pyre-object/src/functional.rs +++ b/pyre/pyre-object/src/functional.rs @@ -18,10 +18,9 @@ use pyre_macros::pyre_class; // ``` // // `w_iter_or_list` is either the source iterator (general case) OR -// the source list itself (start == 0 + exact-list source, line 268-269). -// Pyre takes the simpler "always store the iterator" subset for now — -// the list fast-path is a layered optimisation pyre does not need -// today (covered by the `is_list` fast path at the call site). +// the source list itself when the start is zero and the source is an exact +// list. `w_enumerate_new` preserves that representation; reduce and iteration +// code must therefore handle both the stored-list and stored-iterator forms. // // `index: i64` is the fast counter; once it overflows i64, `w_index` // carries the bigint value (PyPy line 297-303 diff --git a/pyre/pyre-object/src/pyobject.rs b/pyre/pyre-object/src/pyobject.rs index f3796454c26..080773af713 100644 --- a/pyre/pyre-object/src/pyobject.rs +++ b/pyre/pyre-object/src/pyobject.rs @@ -677,17 +677,17 @@ pub const SUBCLASS_RANGE_HIERARCHY: &[(u32, Option)] = &[ (176, Some(0)), (177, Some(0)), (178, Some(0)), - // `posix.DirEntry` follows the unconditional native owners on native - // builds. + // Native-only type IDs 179 and 180 represent `posix.DirEntry` and + // `posix.ScandirIterator`, matching `build_gc`'s registration order. #[cfg(not(target_arch = "wasm32"))] (179, Some(0)), + #[cfg(not(target_arch = "wasm32"))] + (180, Some(0)), // rustls `_ssl` context, MemoryBIO, and session native payloads. These // extend the append-only native rclass tail; wasm omits the host TLS // module and therefore the hierarchy entries as well. Sandbox filtering // belongs to pyre-interpreter, which owns that module configuration. #[cfg(not(target_arch = "wasm32"))] - (180, Some(0)), - #[cfg(not(target_arch = "wasm32"))] (181, Some(0)), #[cfg(not(target_arch = "wasm32"))] (182, Some(0)), @@ -695,6 +695,8 @@ pub const SUBCLASS_RANGE_HIERARCHY: &[(u32, Option)] = &[ (183, Some(0)), #[cfg(not(target_arch = "wasm32"))] (184, Some(0)), + #[cfg(not(target_arch = "wasm32"))] + (185, Some(0)), // `mmap.mmap` owns its native mapping payload — the duplicated fd on POSIX // and the file handle on Windows — and follows the optional SSL tail // wherever the module is compiled. The gate must match the alias gate in @@ -703,16 +705,16 @@ pub const SUBCLASS_RANGE_HIERARCHY: &[(u32, Option)] = &[ // A sandbox build has no `mmap` module either, so // `active_subclass_range_hierarchy` drops this entry along with SSL's. #[cfg(any(unix, windows))] - (185, Some(0)), + (186, Some(0)), // `_overlapped.Overlapped` owns the Windows OVERLAPPED record and its // retained Python buffers. pyre-interpreter supplies the vtable alias; // the object layer owns only the append-only hierarchy slot. #[cfg(windows)] - (186, Some(0)), + (187, Some(0)), // `_winapi.Overlapped` owns a second Windows OVERLAPPED record, the one // waited on through an event of its own rather than a completion port. #[cfg(windows)] - (187, Some(0)), + (188, Some(0)), ]; /// Compute subclass IDs from [`SUBCLASS_RANGE_HIERARCHY`] and write every diff --git a/pyre/pyre-wasm/src/lib.rs b/pyre/pyre-wasm/src/lib.rs index 18bd5e1a79a..36ba78e4c0c 100644 --- a/pyre/pyre-wasm/src/lib.rs +++ b/pyre/pyre-wasm/src/lib.rs @@ -307,7 +307,7 @@ mod host_fs_provider { pyre_interpreter::importing::add_sys_path(Path::new(root)); } } - pyre_interpreter::importing::install_source_provider(std::rc::Rc::new(HostFsProvider)); + pyre_interpreter::importing::install_source_provider(std::sync::Arc::new(HostFsProvider)); } } diff --git a/pyre/pyrex/src/lib.rs b/pyre/pyrex/src/lib.rs index 2cb1ad75d75..93ad46ba3ec 100644 --- a/pyre/pyrex/src/lib.rs +++ b/pyre/pyrex/src/lib.rs @@ -87,6 +87,71 @@ fn drain_args(parser: &mut lexopt::Parser) -> Result, le Ok(parser.raw_args()?.collect()) } +/// Remove the common space/tab prefix from the `-c` source's nonblank lines and +/// empty its blank ones. +/// +/// A line is what sits between two `\n`, so a `\r` is content like any other +/// character: `" \r"` holds something, narrows the margin to its two spaces +/// and keeps its carriage return, where `" "` alone is blank. That is a +/// narrower notion of blank than `textwrap.dedent`'s, which empties every line +/// `str.isspace()` answers for. +fn dedent_command(source: &str) -> std::borrow::Cow<'_, str> { + fn split_newline(line: &str) -> (&str, &str) { + match line.strip_suffix('\n') { + Some(content) => (content, "\n"), + None => (line, ""), + } + } + fn is_blank(content: &str) -> bool { + content.bytes().all(|byte| matches!(byte, b' ' | b'\t')) + } + + // The margin is the common leading run of spaces and tabs over the lines + // that hold something. A blank line never narrows it, and a run stops at + // the first character that is neither, so an indent built from any other + // character is not a margin at all. + let mut margin: Option<&str> = None; + for line in source.split_inclusive('\n') { + let (content, _) = split_newline(line); + if is_blank(content) { + continue; + } + let indent_len = content + .find(|c| !matches!(c, ' ' | '\t')) + .unwrap_or(content.len()); + let indent = &content[..indent_len]; + margin = Some(match margin { + None => indent, + Some(current) => { + let common_len = current + .bytes() + .zip(indent.bytes()) + .take_while(|(left, right)| left == right) + .count(); + ¤t[..common_len] + } + }); + } + + // No margin, no rewriting: a source with a line at column zero keeps its + // blank lines as they were written, rather than having them emptied on + // their own. + let margin = margin.unwrap_or(""); + if margin.is_empty() { + return std::borrow::Cow::Borrowed(source); + } + let mut dedented = String::with_capacity(source.len()); + for line in source.split_inclusive('\n') { + let (content, newline) = split_newline(line); + if !is_blank(content) { + // Every nonblank line contributed to `margin`, so it is a prefix. + dedented.push_str(&content[margin.len()..]); + } + dedented.push_str(newline); + } + std::borrow::Cow::Owned(dedented) +} + /// Emit the `preconfig_init_utf8_mode` fatal error for an invalid PYTHONUTF8 / /// `-X utf8` value and exit; the value is validated during pre-init config. fn fatal_utf8_config_error(detail: &str) -> ! { @@ -655,7 +720,8 @@ fn real_main(binary_name: &str) { let mut argv = vec![std::ffi::OsString::from("-c")]; argv.extend(args); importing::set_sys_argv(&argv); - run_source(&cmd, Mode::Exec, "", no_site); + let cmd = dedent_command(&cmd); + run_source(cmd.as_ref(), Mode::Exec, "", no_site); if inspect { repl::run_repl(true, no_site); } @@ -1970,7 +2036,104 @@ fn run_source(source: &str, mode: Mode, filename: &str, no_site: bool) { #[cfg(test)] mod tests { - use super::{parse_heapsize, set_last_exec_ctx, setup_exec_context}; + use super::{dedent_command, parse_heapsize, set_last_exec_ctx, setup_exec_context}; + + #[test] + fn command_dedent_removes_shared_space_prefix() { + let source = "\n import sys\n print(\"ok\")\n"; + assert_eq!( + dedent_command(source).as_ref(), + "\nimport sys\nprint(\"ok\")\n" + ); + } + + #[test] + fn command_dedent_leaves_extra_indentation() { + let source = " print(\"first\")\n print(\"second\")\n"; + assert_eq!( + dedent_command(source).as_ref(), + "print(\"first\")\n print(\"second\")\n" + ); + } + + #[test] + fn command_dedent_treats_tabs_as_distinct_prefix_bytes() { + let source = "\tprint(\"first\")\n\tprint(\"second\")\n"; + assert_eq!( + dedent_command(source).as_ref(), + "print(\"first\")\nprint(\"second\")\n" + ); + } + + #[test] + fn command_dedent_handles_one_line_without_leading_newline() { + assert_eq!(dedent_command(" print(\"B\")").as_ref(), "print(\"B\")"); + } + + #[test] + fn command_dedent_empties_whitespace_only_line_shallower_than_prefix() { + let source = " print(\"first\")\n \n print(\"second\")\n"; + assert_eq!( + dedent_command(source).as_ref(), + "print(\"first\")\n\nprint(\"second\")\n" + ); + } + + #[test] + fn command_dedent_empties_whitespace_only_line_deeper_than_prefix() { + let source = " print(\"first\")\n \n print(\"second\")\n"; + assert_eq!( + dedent_command(source).as_ref(), + "print(\"first\")\n\nprint(\"second\")\n" + ); + } + + #[test] + fn command_dedent_empties_whitespace_only_line_equal_to_prefix() { + let source = " print(\"first\")\n \n print(\"second\")\n"; + assert_eq!( + dedent_command(source).as_ref(), + "print(\"first\")\n\nprint(\"second\")\n" + ); + } + + #[test] + fn command_dedent_is_raw_text_and_stops_at_column_zero() { + let source = " text = \"\"\"\n inside\ninside\n\"\"\"\n"; + assert!(matches!( + dedent_command(source), + std::borrow::Cow::Borrowed(value) if value == source + )); + } + + #[test] + fn command_dedent_leaves_a_blank_line_alone_when_there_is_no_margin() { + let source = "x = \"\"\"a\n \nb\"\"\"\n"; + assert!(matches!( + dedent_command(source), + std::borrow::Cow::Borrowed(value) if value == source + )); + } + + #[test] + fn command_dedent_counts_a_carriage_return_as_content() { + // Python tokenization normalizes the retained carriage return. + let source = " a\r\n \r\n b\r\n"; + assert_eq!(dedent_command(source).as_ref(), " a\r\n\r\n b\r\n"); + } + + #[test] + fn command_dedent_counts_whitespace_that_is_not_space_or_tab_as_content() { + // A form feed on its own line is content at column zero, which leaves + // no margin for anything to lose. + for content in ["\u{c}", "\u{b}", "\u{85}"] { + let source = format!(" a\n{content}\n b\n"); + assert!( + matches!(dedent_command(&source), std::borrow::Cow::Borrowed(value) if value == source), + "{content:?}" + ); + } + } #[test] fn heapsize_suffixes_and_validation() {