sysconfig, sizeof: advertise a build without the lock, and report its layout

dist-workspace.toml

@@ -29,7 +29,7 @@ install-updater = false
 # and it is included beside the executable in every archive and installer.
 # `include` keeps each item's own name and has no per-target form, so the one
 # entry names the stdlib directory on every platform: `lib` holding
-# `pyre3.14`, except on Windows where the stdlib is `Lib` itself and the two
+# `pyre3.14t`, except on Windows where the stdlib is `Lib` itself and the two
 # spellings are the same path.
 include = ["dist-assets/lib"]
 github-build-setup = "../pyre-dist-build-setup.yml"

pyre/extra_tests/parity_tests/dict_set_sizeof_python314.py

@@ -3,19 +3,30 @@
 
 """Python 3.14 ``__sizeof__`` surface for dict and set-like types."""
 
+import struct
+import sysconfig
+
 for typ in (dict, set, frozenset):
     assert "__sizeof__" in typ.__dict__
     assert typ.__sizeof__.__text_signature__ == "($self, /)"
 
-assert dict().__sizeof__() == 48
-assert {0: None}.__sizeof__() == 208
-assert {str(i): None for i in range(6)}.__sizeof__() == 256
-assert dict.fromkeys(range(11)).__sizeof__() == 616
+# The numbers below are the layout of a build that has a global interpreter
+# lock. Without one the object header carries the thread id, flags, mutex, gc
+# bits and the two refcount halves rather than one refcount, which is two words
+# wider, and every container answers that much more. `test.support.calcobjsize`
+# derives its struct sizes from the same config var, so the oracle and pyre can
+# disagree about the build and still agree about the layout.
+HEADER = 2 * struct.calcsize("P") if sysconfig.get_config_var("Py_GIL_DISABLED") else 0
+
+assert dict().__sizeof__() == 48 + HEADER
+assert {0: None}.__sizeof__() == 208 + HEADER
+assert {str(i): None for i in range(6)}.__sizeof__() == 256 + HEADER
+assert dict.fromkeys(range(11)).__sizeof__() == 616 + HEADER
 
 for typ in (set, frozenset):
-    assert typ().__sizeof__() == 200
-    assert typ(range(4)).__sizeof__() == 200
-    assert typ(range(5)).__sizeof__() == 712
-    assert typ(range(19)).__sizeof__() == 2248
+    assert typ().__sizeof__() == 200 + HEADER
+    assert typ(range(4)).__sizeof__() == 200 + HEADER
+    assert typ(range(5)).__sizeof__() == 712 + HEADER
+    assert typ(range(19)).__sizeof__() == 2248 + HEADER
 
 print("OK")

pyre/pyre-interpreter/src/eval.rs

@@ -8420,7 +8420,7 @@ result = (
                            24, 24, 24, 24]
     and [allocation(item) for item in sources] == [6, 6, 6, 8, 8, 8, 8, 8, 8]
     and allocation(list(HintTwenty())) == 8
-    and sort_seen == [(0, 32), (0, 32), (0, 32)]
+    and sort_seen == [(0, 56), (0, 56), (0, 56)]
     and sorted_value == [1, 2, 3]
     and allocation(sorted_value) == 4
     and noop_sorted == [1, 2, 3]

pyre/pyre-interpreter/src/importing.rs

@@ -849,9 +849,8 @@ fn init_string_module(ns: PyObjectRef) {
 /// `sysconfig._init_non_posix` SUBSCRIPTS `Py_GIL_DISABLED` and `Py_DEBUG` to
 /// spell `ABIFLAGS`, so on Windows a missing key is a `KeyError` out of the
 /// first `get_config_var` call rather than the `None` the `.get()` readers
-/// take. Pyre runs its mutators under a global interpreter lock
-/// (`majit-gc/src/rgil.rs`, the `thread_gil.c` port), so `Py_GIL_DISABLED` is
-/// 0 and the derived `ABIFLAGS` is empty; `Py_DEBUG` is 0.
+/// take. Pyre runs its mutators without a global interpreter lock, so
+/// `Py_GIL_DISABLED` is 1 and the derived `ABIFLAGS` is `t`; `Py_DEBUG` is 0.
 ///
 /// `EXT_SUFFIX` and `SOABI` name pyre's own cpyext ABI, never CPython's ABI
 /// tag. They are build metadata rather than a claim that an extension loader
@@ -869,7 +868,7 @@ fn init_sysconfig_stub(ns: PyObjectRef) {
             let vars = pyre_object::w_dict_new();
             let so_ext = extension_abi_suffix();
             unsafe {
-                for (name, value) in [("Py_DEBUG", 0), ("Py_GIL_DISABLED", 0)] {
+                for (name, value) in [("Py_DEBUG", 0), ("Py_GIL_DISABLED", 1)] {
                     pyre_object::w_dict_store(
                         vars,
                         pyre_object::w_str_new(name),
@@ -1087,14 +1086,12 @@ fn init_sysconfigdata(ns: PyObjectRef) {
     ));
 
     let vars = pyre_object::w_dict_new();
-    // Empty, matching the `Py_GIL_DISABLED` of 0 that `_init_non_posix`
-    // derives the non-posix spelling from, and matching the lower-case
-    // `abiflags` that forms the include and site-packages directory names —
-    // a separate variable read from `sys.abiflags` (`sysconfig:545`). The
-    // release tree has no `t` suffix on its stdlib directory for
-    // `site.py:409` to find, and `sysconfig`'s `abi_thread` must name the
-    // same directory `site.py` does.
-    store_str(vars, "ABIFLAGS", "");
+    // `_init_non_posix` derives the same `t` from `Py_GIL_DISABLED` below.
+    // The lower-case `abiflags` that forms the include and site-packages
+    // directory names is a separate variable, read from `sys.abiflags`
+    // (`sysconfig:545`), and stays empty: the release tree has no `t` suffix
+    // on its stdlib directory for `site.py:409` to find.
+    store_str(vars, "ABIFLAGS", "t");
     store_str(vars, "SOABI", &soabi);
     // Deprecated in Python 3, kept for backward compatibility.
     store_str(vars, "SO", &so_ext);
@@ -1114,13 +1111,11 @@ fn init_sysconfigdata(ns: PyObjectRef) {
     store_str(vars, "EXE", "");
     store_str(vars, "VERSION", "3.14");
     store_str(vars, "LDVERSION", "3.14");
-    // cpyext never uses Py_DEBUG.  Pyre runs its mutators under a global
-    // interpreter lock (`majit-gc/src/rgil.rs` ports `thread_gil.c`), so it is
-    // not a free-threaded build: `test.support.Py_GIL_DISABLED` reads this key
-    // and skips whole suites on it.  Py_ENABLE_SHARED at 1 would add a python
-    // shared object to link lines as `-lpython3.x`.
+    // cpyext never uses Py_DEBUG.  Pyre runs its mutators without a global
+    // interpreter lock.  Py_ENABLE_SHARED at 1 would add a python shared
+    // object to link lines as `-lpython3.x`.
     store_int(vars, "Py_DEBUG", 0);
-    store_int(vars, "Py_GIL_DISABLED", 0);
+    store_int(vars, "Py_GIL_DISABLED", 1);
     store_int(vars, "Py_ENABLE_SHARED", 0);
     // Pyre currently has neither a CPython-compatible C API nor a separately
     // linkable runtime library.  Keep the build ABI metadata above for wheel
@@ -1781,7 +1776,7 @@ pub(crate) struct StartupPathConfig {
     pub base_prefix: PathBuf,
     /// Bootstrap search entries in PyPy's order.  A source checkout keeps
     /// `lib_pypy` before `lib-python/3`; an installed tree has one merged
-    /// `lib/pyre3.14` entry.
+    /// `lib/pyre3.14t` entry.
     pub stdlib_paths: Vec<PathBuf>,
     /// The language stdlib root (the entry containing `os.py` / `site.py`).
     /// This is exposed as `sys._stdlib_dir` and is deliberately distinct from
@@ -1913,7 +1908,7 @@ fn find_invoked_executable() -> PathBuf {
 }
 
 /// Recognize both PyPy-style source trees (`lib-python/3`) and the packaged
-/// Pyre layout (`lib/pyre3.14`).  The latter is the installation shape pip and
+/// Pyre layout (`lib/pyre3.14t`).  The latter is the installation shape pip and
 /// venv will consume; accepting the former keeps the repository executable as
 /// the untranslated/development oracle, matching PyPy's two
 /// `compute_stdlib_path_*` arms.
@@ -1950,7 +1945,7 @@ fn stdlib_at_prefix(prefix: &Path) -> Option<(Vec<PathBuf>, Option<PathBuf>)> {
     // `compute_stdlib_path`: release packaging merges the two source trees
     // into one implementation-version directory.
     for packaged in [
-        prefix.join("lib").join("pyre3.14"),
+        prefix.join("lib").join("pyre3.14t"),
         // cargo-dist's Homebrew formula installs non-binary archive contents
         // into `pkgshare` (`<keg>/share/pyrex`).  This is still a single
         // interpreter-owned prefix, analogous to PyPy's macOS bundle search
@@ -1959,7 +1954,7 @@ fn stdlib_at_prefix(prefix: &Path) -> Option<(Vec<PathBuf>, Option<PathBuf>)> {
             .join("share")
             .join("pyrex")
             .join("lib")
-            .join("pyre3.14"),
+            .join("pyre3.14t"),
     ] {
         if packaged.join("site.py").is_file() {
             let mut paths = vec![packaged.clone()];
@@ -2081,7 +2076,7 @@ fn prefix_from_stdlib(stdlib: &Path) -> PathBuf {
         .is_some_and(|name| name == "lib")
     {
         // cargo-dist Homebrew data lives at
-        // `<keg>/share/pyrex/lib/pyre3.14`, while the executable still lives
+        // `<keg>/share/pyrex/lib/pyre3.14t`, while the executable still lives
         // at `<keg>/bin/pyre`.  An explicit PYRE_STDLIB pointing at that data
         // must recover the keg, not claim `<keg>/share/pyrex` as sys.prefix.
         if let Some(pyre_share) = parent.and_then(Path::parent)
@@ -5299,12 +5294,12 @@ mod tests {
             PathBuf::from("/src/pyre")
         );
         assert_eq!(
-            prefix_from_stdlib(Path::new("/opt/pyre/lib/pyre3.14")),
+            prefix_from_stdlib(Path::new("/opt/pyre/lib/pyre3.14t")),
             PathBuf::from("/opt/pyre")
         );
         assert_eq!(
             prefix_from_stdlib(Path::new(
-                "/opt/homebrew/Cellar/pyrex/0.0.2/share/pyrex/lib/pyre3.14"
+                "/opt/homebrew/Cellar/pyrex/0.0.2/share/pyrex/lib/pyre3.14t"
             )),
             PathBuf::from("/opt/homebrew/Cellar/pyrex/0.0.2")
         );
@@ -5322,7 +5317,7 @@ mod tests {
     #[test]
     fn stdlib_layout_prefers_the_versioned_zip_without_claiming_a_directory() {
         let tree = tempfile::tempdir().unwrap();
-        let packaged = tree.path().join("lib/pyre3.14");
+        let packaged = tree.path().join("lib/pyre3.14t");
         std::fs::create_dir_all(&packaged).unwrap();
         std::fs::write(packaged.join("site.py"), "").unwrap();
         let zip = tree.path().join("python314.zip");
@@ -5343,7 +5338,7 @@ mod tests {
         let tree = tempfile::tempdir().unwrap();
         let base = tree.path().join("base");
         let base_executable = base.join("bin/pyre");
-        let base_stdlib = base.join("lib/pyre3.14");
+        let base_stdlib = base.join("lib/pyre3.14t");
         std::fs::create_dir_all(base_executable.parent().unwrap()).unwrap();
         std::fs::create_dir_all(&base_stdlib).unwrap();
         std::fs::write(&base_executable, "").unwrap();

pyre/pyre-interpreter/src/module/sys/vm.rs

@@ -71,24 +71,21 @@ fn get_sizeof(w_obj: PyObjectRef) -> crate::PyResult {
         ));
     }
 
-    // `_PyType_PreHeaderSize(Py_TYPE(o))` adds its two components
-    // independently: a two-word GC header for tracked objects, plus a two-word
-    // managed dict/weakref prefix where the instance type requests it.  A
-    // tracked builtin such as list has only the first; a normal heap instance
-    // has both; an untracked heap-derived value may have only the second.
+    // `_PyType_PreHeaderSize(Py_TYPE(o))` — the two-word managed dict/weakref
+    // prefix an instance type requests. Its other term, a `PyGC_Head` ahead of
+    // every tracked object, is compiled out of a build without a global
+    // interpreter lock: the collector keeps its bits in the object header
+    // there. That is why the four-word header those builds carry costs a
+    // tracked type nothing over the two-word one, while an untracked leaf pays
+    // the whole two words.
     //
-    // Both terms are read off the type. Asking instead which heap the instance
+    // The term is read off the type. Asking instead which heap the instance
     // landed in would make the answer depend on the allocation that produced
     // it: a `str` folded into a code constant sits outside the collector's
     // ranges and one built at run time does not, so the same value reported two
     // different sizes.
     let word = std::mem::size_of::<usize>() as u64;
-    let gc_header = if crate::typedef::cpython_object_is_gc(current()) {
-        2 * word
-    } else {
-        0
-    };
-    let managed_prefix = crate::typedef::r#type(current()).map_or(0, |tp| unsafe {
+    let pre_header = crate::typedef::r#type(current()).map_or(0, |tp| unsafe {
         if pyre_object::w_type_is_heaptype(tp.as_ptr())
             && (pyre_object::w_type_get_hasdict(tp.as_ptr())
                 || pyre_object::w_type_get_weakrefable(tp.as_ptr()))
@@ -98,7 +95,6 @@ fn get_sizeof(w_obj: PyObjectRef) -> crate::PyResult {
             0
         }
     });
-    let pre_header = gc_header + managed_prefix;
     let total = (size as u64)
         .checked_add(pre_header)
         .expect("Py_ssize_t plus the fixed pre-header fits in size_t");
@@ -1238,11 +1234,11 @@ pub fn register_module(ns: pyre_object::PyObjectRef) {
     );
     // sys.winver — the "major.minor" tag Windows uses for the per-user site
     // directory and the PythonCore registry keys. site.getusersitepackages
-    // reads it to build USER_SITE.  It carries no `t`: that suffix is how
-    // Windows spells what `sys.abiflags` spells elsewhere, and pyre holds a
-    // global interpreter lock.
+    // reads it to build USER_SITE.  A build without a global interpreter lock
+    // carries the `t` here, which is how Windows spells what `sys.abiflags`
+    // spells elsewhere.
     #[cfg(windows)]
-    module_ns_store(ns, "winver", w_str_new("3.14"));
+    module_ns_store(ns, "winver", w_str_new("3.14t"));
     // sys.dllhandle — the handle of the DLL exporting the Python C API,
     // published beside `winver` because both come from the same `MS_COREDLL`
     // block.  `ctypes/__init__.py:562` builds `pythonapi` out of it with no
@@ -2047,15 +2043,11 @@ pub fn register_module(ns: pyre_object::PyObjectRef) {
             Err(unsafe { crate::PyError::from_exc_object(exc) })
         }),
     );
-    // sys.abiflags — empty, because pyre runs its mutators under a global
-    // interpreter lock (`majit-gc/src/rgil.rs`, the `thread_gil.c` port), so
-    // the `t` that spells a free-threaded ABI does not describe this build.
-    // `site.py:409` reads the flag to name the site-packages directory and
-    // `sysconfig` derives `abi_thread` from `Py_GIL_DISABLED` for the same
-    // name, so the two must agree.  The attribute is absent on Windows, where
-    // the flag is spelled in `sys.winver`; every reader guards with `hasattr`,
-    // so an empty string answers the same.
-    module_ns_store(ns, "abiflags", w_str_new(""));
+    // sys.abiflags — `t` for a build without a global interpreter lock.  The
+    // attribute is absent on Windows, where the flag is spelled in
+    // `sys.winver` and neither the `nt` scheme nor `site._get_path` reads it;
+    // every reader guards with `hasattr`, so an empty string answers the same.
+    module_ns_store(ns, "abiflags", w_str_new(if cfg!(windows) { "" } else { "t" }));
     // sys.argv — pick up pending argv from set_sys_argv if available.
     let pending = crate::importing::take_pending_sys_argv();
     let argv = if pending.is_null() {