posix, signal: posix_spawn's keyword arguments, setgroups, sched_param, and the signal mask the interpreter thread was clearing

pyre/extra_tests/parity_tests/lsprof_enable_frees_tool_id_on_failure.py

@@ -0,0 +1,51 @@
+# CPython-suite gap: test_cprofile and test_profile only ever pass real bools
+# to `enable`, so neither exercises the failing-conversion path, and the leak
+# they would expose is process-wide rather than per-test.
+# parity-tests reason: the tool id is a process singleton, so the damage shows
+# up on a *later, unrelated* profiler; that cross-object effect is what this
+# checks, on both backends, on every leg.
+
+"""A failed `Profiler.enable` must not keep the profiler tool id.
+
+`enable` takes `subcalls` and `builtins` through the index/bool protocol, so
+an argument whose `__bool__` raises makes the call fail. The tool id is a
+single process-wide slot: if the failed call has already claimed it and
+nothing releases it, `disable` cannot help -- it is a no-op while the
+profiler never became enabled -- and every later `enable`, on any profiler
+object, reports that another tool is active.
+
+The discriminator is therefore a *second, independent* profiler enabling
+successfully after the first one's `enable` raised.
+"""
+
+import _lsprof
+
+
+class Raises:
+    def __bool__(self):
+        raise ValueError("no truth value")
+
+
+def main():
+    first = _lsprof.Profiler()
+    try:
+        first.enable(Raises())
+    except ValueError:
+        pass
+    else:
+        raise AssertionError("enable() accepted an argument whose __bool__ raises")
+
+    # The failed call must have left the tool id free.
+    second = _lsprof.Profiler()
+    second.enable()
+    second.disable()
+
+    # And the id must still be reusable after a clean enable/disable pair.
+    third = _lsprof.Profiler()
+    third.enable()
+    third.disable()
+
+    print("OK")
+
+
+main()

pyre/extra_tests/parity_tests/simplequeue_put_defaults_replay.py

@@ -0,0 +1,73 @@
+# CPython-suite gap: test_queue reaches this only through
+# `CSimpleQueueTest.test_reentrancy`, which reads as a finalizer-ordering test
+# and reports it as a 10000-element list diff -- it names neither the replayed
+# call nor the argument shape that causes it.
+# parity-tests reason: the defect is a miscompile, so it needs a fixture that
+# runs the loop hot on both backends and checks a count that the interpreter
+# and the compiled trace must agree on.
+
+"""A hot loop must call its producer exactly once per iteration.
+
+`SimpleQueue.put` is declared `put(item, block=True, timeout=None)`; the two
+trailing arguments are accepted and ignored, because an unbounded queue never
+blocks. Writing `put(v)` therefore leaves the call site to fill both defaults
+in, and that is the shape this guards.
+
+When such a call is compiled into a hot loop, the call feeding it must not be
+re-executed. `Counter.next` is deliberately side-effecting, so a replay shows
+up twice over: as a producer count that exceeds the iteration count, and as a
+value that is generated but never queued -- which shifts every later result by
+one rather than reordering a pair.
+
+A plain function call in the producer position does not reproduce it; the call
+has to go through the method path, which is why this uses a bound method.
+
+The count is small enough to stay fast and large enough for the loop to be
+compiled and left at least once.
+"""
+
+import queue
+
+LIMIT = 1500
+
+
+class Counter:
+    def __init__(self):
+        self.n = 0
+
+    def next(self):
+        value = self.n
+        self.n += 1
+        return value
+
+
+def main():
+    q = queue.SimpleQueue()
+    counter = Counter()
+    results = []
+
+    while True:
+        q.put(counter.next())
+        results.append(q.get())
+        if results[-1] >= LIMIT:
+            break
+
+    assert counter.n == len(results), (
+        "the producer ran more often than the loop body",
+        counter.n,
+        len(results),
+    )
+    expected = list(range(LIMIT + 1))
+    assert results == expected, (
+        "a produced value never reached the queue",
+        next(
+            (i, results[i], expected[i])
+            for i in range(len(results))
+            if results[i] != expected[i]
+        ),
+    )
+
+    print("OK")
+
+
+main()

pyre/pyre-interpreter/src/_structseq.rs

@@ -339,7 +339,7 @@ fn structseq_setattr(args: &[PyObjectRef]) -> Result<PyObjectRef, PyError> {
 /// dict])` constructor.  The first `n_sequence_fields` items fill the
 /// tuple body; any surplus positional items, then the optional dict, then
 /// `None` defaults, fill the named-only extra fields.
-fn structseq_descr_new(args: &[PyObjectRef]) -> Result<PyObjectRef, PyError> {
+pub(crate) fn structseq_descr_new(args: &[PyObjectRef]) -> Result<PyObjectRef, PyError> {
     if args.len() < 2 || args[1].is_null() {
         return Err(PyError::type_error("structseq() requires class + sequence"));
     }

pyre/pyre-interpreter/src/cpyext/capsule.rs

@@ -22,7 +22,7 @@ const DESTRUCTOR_KEY: &str = "__pyre_destructor__";
 
 static CAPSULE_TYPE: OnceLock<usize> = OnceLock::new();
 
-fn capsule_type() -> PyObjectRef {
+pub(crate) fn capsule_type() -> PyObjectRef {
     *CAPSULE_TYPE.get_or_init(|| {
         let tp = crate::typedef::make_builtin_type("PyCapsule", |ns| unsafe {
             pyre_object::dictmultiobject::w_dict_setitem_str_no_proxy(

pyre/pyre-interpreter/src/eval.rs

@@ -205,17 +205,17 @@ pub fn install_current_frame_tls_only(frame: &mut PyFrame) -> CurrentFrameGuard
 /// Pushing the frame onto the shadow stack lets the root walker forward it in
 /// place during the collection; `live()` reads the forwarded pointer back.
 /// This mirrors the JIT eval layer's `FrameRoot`.
-pub(crate) struct FrameAnchor {
+pub struct FrameAnchor {
     depth: usize,
 }
 
 impl FrameAnchor {
-    pub(crate) fn new(frame: &mut PyFrame) -> Self {
+    pub fn new(frame: &mut PyFrame) -> Self {
         let depth = majit_gc::shadow_stack::push(majit_ir::GcRef(frame as *mut PyFrame as usize));
         Self { depth }
     }
 
-    pub(crate) fn live(&self) -> *mut PyFrame {
+    pub fn live(&self) -> *mut PyFrame {
         majit_gc::shadow_stack::get(self.depth).0 as *mut PyFrame
     }
 }
@@ -2276,6 +2276,19 @@ pub(crate) fn finalize_failed_attr_receiver_now(obj: PyObjectRef) -> bool {
 impl SharedOpcodeHandler for PyFrame {
     type Value = PyObjectRef;
 
+    type Anchor = FrameAnchor;
+
+    fn anchor(&mut self) -> Self::Anchor {
+        FrameAnchor::new(self)
+    }
+
+    fn push_anchored(anchor: &Self::Anchor, value: Self::Value) -> Result<(), PyError> {
+        // A JIT-created frame lives in the nursery and the allocating step may
+        // have relocated it; push onto the forwarded live frame.
+        unsafe { &mut *anchor.live() }.push(value);
+        Ok(())
+    }
+
     fn push_value(&mut self, value: Self::Value) -> Result<(), PyError> {
         self.push(value);
         Ok(())

pyre/pyre-interpreter/src/host_seam.rs

@@ -101,7 +101,7 @@ pub mod sys {
     #[cfg(all(target_os = "linux", target_env = "gnu"))]
     pub use ::libc::RTLD_DEEPBIND;
     #[cfg(any(target_os = "linux", target_os = "android"))]
-    pub use ::libc::{SCHED_BATCH, SCHED_IDLE};
+    pub use ::libc::{SCHED_BATCH, SCHED_DEADLINE, SCHED_IDLE, SCHED_NORMAL, SCHED_RESET_ON_FORK};
     #[cfg(not(any(target_os = "macos", target_os = "ios")))]
     pub use ::libc::{SCHED_FIFO, SCHED_OTHER, SCHED_RR};
     // `SEEK_HOLE`/`SEEK_DATA` carry the same host list as the table that

pyre/pyre-interpreter/src/importing.rs

@@ -754,6 +754,9 @@ pub fn install_builtin_modules() {
         crate::module::array::startup_array_module,
     );
     register_builtin_module("_csv", crate::module::_csv::init);
+    register_builtin_module("_queue", crate::module::_queue::init);
+    register_builtin_module("_statistics", crate::module::_statistics::init);
+    register_builtin_module("_types", crate::module::_types::init);
     register_builtin_module("_json", crate::module::_json::init);
     register_builtin_module("_tokenize", crate::module::_tokenize::init);
     // `_scproxy` is built only on macOS, and `urllib.request` reaches it only

pyre/pyre-interpreter/src/lib.rs

@@ -1171,41 +1171,44 @@ pub fn all_subclass_range_aliases() -> Vec<pyre_object::pyobject::SubclassRangeA
         subclass_range_alias(176, typed::<crate::module::_lsprof::W_Profiler>()),
         subclass_range_alias(177, typed::<crate::module::_lsprof::W_StatsEntry>()),
         subclass_range_alias(178, typed::<crate::module::_lsprof::W_StatsSubEntry>()),
-        // Native-only posix aliases 179 and 180 preserve `build_gc`'s rclass
+        // `_queue.SimpleQueue` is unconditional and carries a native FIFO, so
+        // it closes the ungated aliases ahead of the target-gated ones.
+        subclass_range_alias(179, typed::<crate::module::_queue::W_SimpleQueue>()),
+        // Native-only posix aliases 180 and 181 preserve `build_gc`'s rclass
         // registration order after the unconditional aliases.
         #[cfg(not(target_arch = "wasm32"))]
-        subclass_range_alias(179, typed::<crate::module::posix::W_DirEntry>()),
+        subclass_range_alias(180, typed::<crate::module::posix::W_DirEntry>()),
         #[cfg(not(target_arch = "wasm32"))]
-        subclass_range_alias(180, typed::<crate::module::posix::W_ScandirIterator>()),
+        subclass_range_alias(181, typed::<crate::module::posix::W_ScandirIterator>()),
         // The rustls-backed `_ssl` aliases preserve `build_gc`'s registration
         // order for `W_SSLContext`, `W_MemoryBIO`, and `W_SSLSession`.
         #[cfg(all(not(target_arch = "wasm32"), not(feature = "sandbox")))]
-        subclass_range_alias(181, typed::<crate::module::_ssl::W_SSLContext>()),
+        subclass_range_alias(182, typed::<crate::module::_ssl::W_SSLContext>()),
         #[cfg(all(not(target_arch = "wasm32"), not(feature = "sandbox")))]
-        subclass_range_alias(182, typed::<crate::module::_ssl::W_MemoryBIO>()),
+        subclass_range_alias(183, typed::<crate::module::_ssl::W_MemoryBIO>()),
         #[cfg(all(not(target_arch = "wasm32"), not(feature = "sandbox")))]
-        subclass_range_alias(183, typed::<crate::module::_ssl::W_SSLSession>()),
+        subclass_range_alias(184, typed::<crate::module::_ssl::W_SSLSession>()),
         #[cfg(all(not(target_arch = "wasm32"), not(feature = "sandbox")))]
-        subclass_range_alias(184, typed::<crate::module::_ssl::W_SSLSocket>()),
+        subclass_range_alias(185, typed::<crate::module::_ssl::W_SSLSocket>()),
         #[cfg(all(not(target_arch = "wasm32"), not(feature = "sandbox")))]
-        subclass_range_alias(185, typed::<crate::module::_ssl::W_Certificate>()),
+        subclass_range_alias(186, typed::<crate::module::_ssl::W_Certificate>()),
         // `mmap.mmap` follows the optional SSL tail on ordinary Unix builds.
         // A sandbox build has no `mmap` module at all (`module/mod.rs`), so it
         // contributes no alias rather than sliding into the vacated SSL slot.
         #[cfg(all(any(unix, windows), not(feature = "sandbox")))]
-        subclass_range_alias(186, typed::<crate::module::mmap::W_MMap>()),
+        subclass_range_alias(187, typed::<crate::module::mmap::W_MMap>()),
         // Windows asyncio's Overlapped owner follows mmap at the native tail.
         // It is a non-subclassable builtin in Python, but still participates
         // in the rclass hierarchy because its managed header and retained
         // buffer/result fields are traced by the ordinary object marker.
         #[cfg(all(windows, feature = "host_env", not(feature = "sandbox")))]
-        subclass_range_alias(187, typed::<crate::module::_overlapped::W_Overlapped>()),
+        subclass_range_alias(188, typed::<crate::module::_overlapped::W_Overlapped>()),
         // `_winapi.Overlapped` follows it: a second record of the same kind,
         // owning its own event and transfer buffer rather than retained
         // Python objects, so nothing of it is traced beyond the header.
         #[cfg(all(windows, feature = "host_env", not(feature = "sandbox")))]
         subclass_range_alias(
-            188,
+            189,
             typed::<crate::module::_winapi::overlapped::W_Overlapped>(),
         ),
     ]

pyre/pyre-interpreter/src/module/_bz2/mod.rs

@@ -158,8 +158,17 @@ mod decompressor_methods {
                     "Decompressor is unusable after a previous error",
                 ));
             }
+            // A cap too large for the platform's index type is an error, not
+            // silently unlimited -- only a negative value means unlimited.
+            let max_length = if max_length < 0 {
+                None
+            } else {
+                Some(usize::try_from(max_length).map_err(|_| {
+                    crate::PyError::overflow_error("Python int too large to convert to C ssize_t")
+                })?)
+            };
             decompressor
-                .decompress(&data, usize::try_from(max_length).ok())
+                .decompress(&data, max_length)
                 .map_err(bz2_error)
         }
 

pyre/pyre-interpreter/src/module/_lsprof/mod.rs

@@ -721,18 +721,33 @@ mod profiler_methods {
             if self.is_enabled {
                 return Ok(());
             }
-            // `_lsprof.c profiler_enable` claims the profiler tool id before
-            // touching any of its own state, so a second profiler's `enable`
-            // reports the conflict and leaves the first one installed.
+            // `subcalls` and `builtins` are declared `bool`, so their truth
+            // value is taken while the arguments are parsed -- before
+            // `profiler_enable` reaches `use_tool_id`. Converting them after
+            // the claim instead would leave the tool id held when `__bool__`
+            // raises, and nothing could release it: `disable` returns early
+            // while `is_enabled` is still false.
+            let flag = |w: PyObjectRef| -> Result<Option<bool>, crate::PyError> {
+                if unsafe { pyre_object::is_none(w) } {
+                    Ok(None)
+                } else {
+                    Ok(Some(crate::baseobjspace::is_true(w)?))
+                }
+            };
+            let subcalls = flag(w_subcalls)?;
+            let builtins = flag(w_builtins)?;
+            // The tool id is claimed before any of the profiler's own state is
+            // touched, so a second profiler's `enable` reports the conflict and
+            // leaves the first one installed.
             crate::module::sys::vm::monitoring_use_tool_id(
                 crate::module::sys::vm::MONITORING_PROFILER_ID,
                 w_str_new("cProfile"),
             )?;
-            if !unsafe { pyre_object::is_none(w_subcalls) } {
-                self.subcalls = crate::baseobjspace::is_true(w_subcalls)?;
+            if let Some(value) = subcalls {
+                self.subcalls = value;
             }
-            if !unsafe { pyre_object::is_none(w_builtins) } {
-                self.builtins = crate::baseobjspace::is_true(w_builtins)?;
+            if let Some(value) = builtins {
+                self.builtins = value;
             }
             self.is_enabled = true;
             self.total_real_time -= read_real_time();

pyre/pyre-interpreter/src/module/_lzma/mod.rs

@@ -455,8 +455,17 @@ mod decompressor_methods {
                     "Already at end of stream",
                 ));
             }
+            // A cap too large for the platform's index type is an error, not
+            // silently unlimited -- only a negative value means unlimited.
+            let max_length = if max_length < 0 {
+                None
+            } else {
+                Some(usize::try_from(max_length).map_err(|_| {
+                    crate::PyError::overflow_error("Python int too large to convert to C ssize_t")
+                })?)
+            };
             decompressor
-                .decompress(&data, usize::try_from(max_length).ok())
+                .decompress(&data, max_length)
                 .map_err(lzma_error)
         }