From d1a7b790e0dd212c1f5ea2856ef807c7edf0a45d Mon Sep 17 00:00:00 2001 From: "Jeong, YunWon" Date: Wed, 22 Jul 2026 22:09:19 +0900 Subject: [PATCH 1/7] import: run source importlib, set module specs, read Python sys.path - Drop the native importlib / importlib.machinery / importlib.abc module registrations so the on-disk importlib package loads from source: its __init__ binds __import__, and machinery/abc re-export the real finder, loader and ModuleSpec classes from the frozen _bootstrap / _bootstrap_external, which already carry the full surface. The native stubs only injected placeholder object classes that shadowed them. - Give builtin modules __spec__/__loader__/__package__ from BuiltinImporter.find_spec + _bootstrap._init_module_attrs, and source modules their __spec__/__loader__/__file__/__cached__ from _bootstrap_external._fix_up_module. Both are gated on the importlib bootstrap being wired and are best-effort so a module imported while the bootstrap itself is still executing falls back to the previous None seeding; _bootstrap._setup fixes up the pre-wire builtins in bulk. load_part re-reads the builtin from sys.modules after the app-level call, which can relocate it. - find_in_sys_path reads the live Python sys.path first so sys.path mutations and PYTHONPATH are honored, then appends the native SYS_PATH seed (deduplicated) to keep the stdlib and defaults searchable and to cover the pre-sync bootstrap window. init_sys_path seeds PYTHONPATH entries. Assisted-by: Claude --- pyre/pyre-interpreter/src/importing.rs | 289 +++++++++++++++++++++---- 1 file changed, 247 insertions(+), 42 deletions(-) diff --git a/pyre/pyre-interpreter/src/importing.rs b/pyre/pyre-interpreter/src/importing.rs index afa9c576953..2e078d2c041 100644 --- a/pyre/pyre-interpreter/src/importing.rs +++ b/pyre/pyre-interpreter/src/importing.rs @@ -529,24 +529,12 @@ pub fn install_builtin_modules() { pyre_install_module!(_opcode); pyre_install_module!("_imp"(imp)); - // importlib package — four submodules backed by distinct init fns. - pyre_install_module!( - "importlib.machinery" => - crate::module::importlib::interp_importlib::register_machinery - ); - pyre_install_module!( - "importlib" => - crate::module::importlib::interp_importlib::register_pkg - ); - // importlib.util is NOT registered as a builtin: with importlib.__path__ - // pointing at the on-disk package, the real util.py loads from there and - // re-exports the full _bootstrap / _bootstrap_external surface - // (cache_from_source, source_from_cache, source_hash, find_spec, …) that - // a stub could only approximate. - pyre_install_module!( - "importlib.abc" => - crate::module::importlib::interp_importlib::register_abc - ); + // importlib package and its submodules load their real source from disk: + // the package `__init__.py` binds `__import__`/`import_module`/… from the + // frozen `_bootstrap`, and `machinery`/`abc`/`util` re-export the real + // finders/loaders/spec classes out of `_frozen_importlib{,_external}`. The + // frozen bootstrap modules already carry the full surface, so a native stub + // would only inject placeholder `object` classes that shadow them. // __pypy__ package + builders submodule — the PyPy-only surface // pickle.py imports (identity_dict + builders.BytesBuilder). @@ -976,15 +964,170 @@ pub(crate) fn create_builtin_module( Ok(Some(pyre_object::gc_roots::shadow_stack_get(module_slot))) } +/// Set a builtin module's `__spec__`/`__loader__`/`__package__` from the +/// app-level `BuiltinImporter`, matching `BuiltinImporter.exec_module` → +/// `_init_module_attrs`. Reachable only once `importlib._bootstrap` is wired; +/// the handful of builtins imported before that are fixed up in bulk by +/// `_bootstrap._setup`'s sys.modules walk, so a no-op here is correct then. +#[cfg(feature = "host_env")] +fn set_builtin_module_spec( + name: &str, + module: PyObjectRef, +) -> Result<(), crate::PyError> { + use pyre_object::gc_roots::{pin_root, push_roots, shadow_stack_get, shadow_stack_len}; + + let Some(bootstrap) = get_sys_module("importlib._bootstrap") else { + return Ok(()); + }; + + let _roots = push_roots(); + let mod_slot = shadow_stack_len(); + pin_root(module); + let boot_slot = shadow_stack_len(); + pin_root(bootstrap); + + // Best-effort throughout: a builtin imported while `_bootstrap` is still + // executing sees a partially-initialised module whose `BuiltinImporter` / + // `_init_module_attrs` are not defined yet. Skip rather than break the + // import — `_bootstrap._setup` fixes up any builtin missed here. + let Ok(importer) = crate::baseobjspace::getattr_str(shadow_stack_get(boot_slot), "BuiltinImporter") + else { + return Ok(()); + }; + let importer_slot = shadow_stack_len(); + pin_root(importer); + let Ok(find_spec) = crate::baseobjspace::getattr_str(shadow_stack_get(importer_slot), "find_spec") + else { + return Ok(()); + }; + let find_spec_slot = shadow_stack_len(); + pin_root(find_spec); + let w_name = pyre_object::w_str_new(name); + let name_slot = shadow_stack_len(); + pin_root(w_name); + let Ok(spec) = crate::call::call_function_impl_result( + shadow_stack_get(find_spec_slot), + &[shadow_stack_get(name_slot)], + ) else { + return Ok(()); + }; + if unsafe { pyre_object::is_none(spec) } { + return Ok(()); + } + let spec_slot = shadow_stack_len(); + pin_root(spec); + + // _init_module_attrs(spec, module) + let Ok(init) = crate::baseobjspace::getattr_str(shadow_stack_get(boot_slot), "_init_module_attrs") + else { + return Ok(()); + }; + let init_slot = shadow_stack_len(); + pin_root(init); + let _ = crate::call::call_function_impl_result( + shadow_stack_get(init_slot), + &[shadow_stack_get(spec_slot), shadow_stack_get(mod_slot)], + ); + Ok(()) +} + +/// Off-`host_env` builds have no app-level importlib to source specs from. +#[cfg(not(feature = "host_env"))] +fn set_builtin_module_spec(_name: &str, _module: PyObjectRef) -> Result<(), crate::PyError> { + Ok(()) +} + +/// Set a source module's `__spec__`/`__loader__`/`__file__`/`__cached__` from +/// the app-level `_bootstrap_external._fix_up_module` — the helper +/// `PyImport_ExecCodeModuleObject` calls. Returns `false` when the importlib +/// bootstrap is not wired yet, so the caller can seed `None` instead. +/// +/// `ns` (the module dict) is written in place; the caller keeps it pinned. +#[cfg(feature = "host_env")] +fn fix_up_source_module_spec( + ns: PyObjectRef, + pathname: &str, + cpathname: Option<&str>, +) -> Result { + use pyre_object::gc_roots::{pin_root, push_roots, shadow_stack_get, shadow_stack_len}; + + let Some(ext) = get_sys_module("importlib._bootstrap_external") else { + return Ok(false); + }; + let Some(w_name) = (unsafe { pyre_object::w_dict_getitem_str(ns, "__name__") }) else { + return Ok(false); + }; + + let _roots = push_roots(); + let ns_slot = shadow_stack_len(); + pin_root(ns); + let name_slot = shadow_stack_len(); + pin_root(w_name); + let ext_slot = shadow_stack_len(); + pin_root(ext); + let w_path = pyre_object::w_str_new(pathname); + let path_slot = shadow_stack_len(); + pin_root(w_path); + let w_cpath = match cpathname { + Some(c) => pyre_object::w_str_new(c), + None => pyre_object::w_none(), + }; + let cpath_slot = shadow_stack_len(); + pin_root(w_cpath); + + // Best-effort: `_bootstrap_external` itself is a source module, and its + // spec is fixed up (during partial-init) before its body defines + // `_fix_up_module` — the getattr then raises. Fall back to `None` seeding + // for that (and any other partially-initialised) case rather than break + // the import; the module's spec is corrected by later imports / `_setup`. + let Ok(fix) = crate::baseobjspace::getattr_str(shadow_stack_get(ext_slot), "_fix_up_module") + else { + return Ok(false); + }; + let fix_slot = shadow_stack_len(); + pin_root(fix); + if crate::call::call_function_impl_result( + shadow_stack_get(fix_slot), + &[ + shadow_stack_get(ns_slot), + shadow_stack_get(name_slot), + shadow_stack_get(path_slot), + shadow_stack_get(cpath_slot), + ], + ) + .is_err() + { + return Ok(false); + } + Ok(true) +} + +/// Off-`host_env` builds have no app-level importlib to source specs from. +#[cfg(not(feature = "host_env"))] +fn fix_up_source_module_spec( + _ns: PyObjectRef, + _pathname: &str, + _cpathname: Option<&str>, +) -> Result { + Ok(false) +} + fn startup_builtin_module( name: &str, module: PyObjectRef, execution_context: *const PyExecutionContext, ) -> Result<(), crate::PyError> { + use pyre_object::gc_roots::{pin_root, push_roots, shadow_stack_get, shadow_stack_len}; + + let _roots = push_roots(); + let mod_slot = shadow_stack_len(); + pin_root(module); + let startup = BUILTIN_MODULES.with(|m| m.borrow().get(name).and_then(|d| d.startup)); if let Some(startup) = startup { - startup(module, execution_context)?; + startup(shadow_stack_get(mod_slot), execution_context)?; } + set_builtin_module_spec(name, shadow_stack_get(mod_slot))?; Ok(()) } @@ -1021,8 +1164,22 @@ pub fn init_sys_path(script_dir: &Path) { } } } - // CPython stdlib path is detected lazily on first stdlib import - // to avoid spawning python3 subprocess on every startup. + // PYTHONPATH entries follow the script/cwd seed and precede the + // lazily-detected stdlib (pathconfig.c). Honoured regardless of the + // safe-path flag, which only suppresses the script/cwd seed. The + // sandbox interpreter takes its search path from the controller, so it + // does not read the host environment here. + #[cfg(not(feature = "sandbox"))] + if let Ok(pythonpath) = host_os::var("PYTHONPATH") { + for entry in pythonpath.split(':').filter(|e| !e.is_empty()) { + let pb = PathBuf::from(entry); + if !path.contains(&pb) { + path.push(pb); + } + } + } + // The stdlib path is detected lazily on first stdlib import to avoid + // spawning a python3 subprocess on every startup. // See find_module() → ensure_stdlib_path(). }); } @@ -1571,9 +1728,56 @@ fn find_in_dirs(partname: &str, dirs: &[PathBuf]) -> Option { None } +/// Read the live Python `sys.path` list as filesystem directories, or an +/// empty vec before `sys` / its `path` list exists (the pre-`sync` bootstrap +/// window). Only `str` entries are collected — path hooks (zipimporter keys +/// and the like) are not resolvable by the native file search. +#[cfg(feature = "host_env")] +fn python_sys_path_dirs() -> Vec { + let Some(sys_mod) = get_sys_module("sys") else { + return Vec::new(); + }; + let w_dict = unsafe { pyre_object::w_module_get_w_dict(sys_mod) }; + if w_dict.is_null() { + return Vec::new(); + } + let Some(w_path) = (unsafe { pyre_object::w_dict_getitem_str(w_dict, "path") }) else { + return Vec::new(); + }; + if !unsafe { pyre_object::is_list(w_path) } { + return Vec::new(); + } + let n = unsafe { pyre_object::listobject::w_list_len(w_path) }; + let mut dirs = Vec::with_capacity(n); + for i in 0..n { + if let Some(item) = unsafe { pyre_object::listobject::w_list_getitem(w_path, i as i64) } { + if unsafe { pyre_object::is_str(item) } { + dirs.push(PathBuf::from(unsafe { pyre_object::w_str_get_value(item) })); + } + } + } + dirs +} + +/// Search the import path for a top-level `partname`. +/// +/// Python `sys.path` is authoritative — user code (and PYTHONPATH) mutate the +/// Python list and imports must honor it, the same precedence `check_sys_modules` +/// gives the Python `sys.modules` dict. The native `SYS_PATH` is the startup +/// seed: it is appended (deduplicated) so the vendored stdlib and the default +/// entries stay searchable, and it is the sole source in the pre-`sync` +/// bootstrap window where the Python list is still the empty placeholder. #[cfg(feature = "host_env")] fn find_in_sys_path(partname: &str) -> Option { - SYS_PATH.with(|p| find_in_dirs(partname, &p.borrow())) + let mut dirs = python_sys_path_dirs(); + SYS_PATH.with(|p| { + for d in p.borrow().iter() { + if !dirs.contains(d) { + dirs.push(d.clone()); + } + } + }); + find_in_dirs(partname, &dirs) } /// Extract a package module's `__path__` as filesystem directories. @@ -1673,26 +1877,22 @@ fn exec_code_module( pyre_object::w_dict_setitem_str(w_globals, "__cached__", w_cpathname); } // importing.py:286-298 — `_fix_up_module(d, name, pathname, - // cpathname)`. PyPy's `_fix_up_module` - // (`lib-python/3/importlib/_bootstrap_external.py:1728`) sets - // `__spec__`/`__loader__`/`__file__`/`__cached__` from the - // app-level `SourceFileLoader` + `spec_from_file_location` - // helpers. Pyre lacks the importlib bootstrap machinery - // (`SourceFileLoader`, `ModuleSpec`, `spec_from_file_location` - // are not yet ported), so as a TODO we seed - // `__loader__`/`__spec__` with `None` only when missing — - // matching PyPy's `if not loader / if not spec` guards - // (_bootstrap_external.py:1732, 1739). When the importlib - // app-level layer lands, the `None` arms will collapse onto the - // mechanical PyPy port. - if unsafe { pyre_object::w_dict_getitem_str(w_globals, "__loader__") }.is_none() { - unsafe { - pyre_object::w_dict_setitem_str(w_globals, "__loader__", pyre_object::w_none()); + // cpathname)` sets `__spec__`/`__loader__`/`__file__`/`__cached__` + // from the app-level `SourceFileLoader` + `spec_from_file_location` + // helpers. Reachable only once the importlib bootstrap is wired; + // before that, seed `__loader__`/`__spec__` with `None` only when + // missing — the `if not loader / if not spec` guards at + // `_bootstrap_external.py:_fix_up_module`. + if !fix_up_source_module_spec(w_globals, p, cpathname)? { + if unsafe { pyre_object::w_dict_getitem_str(w_globals, "__loader__") }.is_none() { + unsafe { + pyre_object::w_dict_setitem_str(w_globals, "__loader__", pyre_object::w_none()); + } } - } - if unsafe { pyre_object::w_dict_getitem_str(w_globals, "__spec__") }.is_none() { - unsafe { - pyre_object::w_dict_setitem_str(w_globals, "__spec__", pyre_object::w_none()); + if unsafe { pyre_object::w_dict_getitem_str(w_globals, "__spec__") }.is_none() { + unsafe { + pyre_object::w_dict_setitem_str(w_globals, "__spec__", pyre_object::w_none()); + } } } } @@ -2051,6 +2251,9 @@ fn load_part( }; set_sys_module(modulename, m); startup_builtin_module(modulename, m, execution_context)?; + // `startup_builtin_module` runs app-level spec construction that can + // collect and relocate `m`; re-read the live pointer from sys.modules. + let m = check_sys_modules(modulename).unwrap_or(m); return Ok(Some(m)); } @@ -2093,7 +2296,9 @@ fn load_part( // Store builtin modules in cache immediately set_sys_module(modulename, m); startup_builtin_module(partname, m, execution_context)?; - m + // `startup_builtin_module` may collect and relocate `m`; re-read + // the live pointer from sys.modules. + check_sys_modules(modulename).unwrap_or(m) } }; From a4a5f6a6e3a5f6e87507777dbe1487752981c3a3 Mon Sep 17 00:00:00 2001 From: "Jeong, YunWon" Date: Wed, 22 Jul 2026 22:09:19 +0900 Subject: [PATCH 2/7] sys: drop the dot from implementation.cache_tag 'pyre-3.14' put a dot inside the cache tag, so _bootstrap_external's PEP 3147 source_from_cache dot-count parse rejected the resulting __pycache__ names. Use 'pyre-314', matching the dot-free tag convention. Assisted-by: Claude --- pyre/pyre-interpreter/src/module/sys/vm.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyre/pyre-interpreter/src/module/sys/vm.rs b/pyre/pyre-interpreter/src/module/sys/vm.rs index 880a96b9e69..e0318de4571 100644 --- a/pyre/pyre-interpreter/src/module/sys/vm.rs +++ b/pyre/pyre-interpreter/src/module/sys/vm.rs @@ -952,7 +952,7 @@ pub fn register_module(ns: pyre_object::PyObjectRef) { ]), ); crate::baseobjspace::setdictvalue(impl_obj, "hexversion", w_int_new(0x030e06f0)); - crate::baseobjspace::setdictvalue(impl_obj, "cache_tag", w_str_new("pyre-3.14")); + crate::baseobjspace::setdictvalue(impl_obj, "cache_tag", w_str_new("pyre-314")); crate::baseobjspace::setdictvalue(impl_obj, "_multiarch", w_str_new("")); module_ns_store(ns, "implementation", impl_obj); } From e4ea6de17a55b4018a9944b87358a9c5d7c0e8a5 Mon Sep 17 00:00:00 2001 From: "Jeong, YunWon" Date: Wed, 22 Jul 2026 22:09:19 +0900 Subject: [PATCH 3/7] io: implement _io.IncrementalNewlineDecoder It was a stub returning None, so _bootstrap_external.decode_source did None.decode and every app-level SourceLoader.get_source raised. Add the real class (a standalone type, matching the C _io type rather than a codecs.IncrementalDecoder subclass) to _io_app.py and register it. Assisted-by: Claude --- .../src/module/_io/_io_app.py | 92 +++++++++++++++++++ pyre/pyre-interpreter/src/module/_io/mod.rs | 3 +- 2 files changed, 93 insertions(+), 2 deletions(-) diff --git a/pyre/pyre-interpreter/src/module/_io/_io_app.py b/pyre/pyre-interpreter/src/module/_io/_io_app.py index db7fb1021f3..b461fbe5213 100644 --- a/pyre/pyre-interpreter/src/module/_io/_io_app.py +++ b/pyre/pyre-interpreter/src/module/_io/_io_app.py @@ -365,3 +365,95 @@ def __enter__(self): def __exit__(self, *exc): self.close() return False + + +class IncrementalNewlineDecoder: + r"""Codec used when reading a file in universal newlines mode. It wraps + another incremental decoder, translating \r\n and \r into \n. It also + records the types of newlines encountered. When used with + translate=False, it ensures that the newline sequence is returned in + one piece. + + `_io.IncrementalNewlineDecoder` is a standalone type, not a + `codecs.IncrementalDecoder` subclass; `decode_source` and `TextIOWrapper` + construct it with `decoder=None` to translate an already-decoded string. + """ + + _LF = 1 + _CR = 2 + _CRLF = 4 + + def __init__(self, decoder, translate, errors="strict"): + self.errors = errors + self.translate = translate + self.decoder = decoder + self.seennl = 0 + self.pendingcr = False + + def decode(self, input, final=False): + # decode input (with the eventual \r from a previous pass) + if self.decoder is None: + output = input + else: + output = self.decoder.decode(input, final=final) + if self.pendingcr and (output or final): + output = "\r" + output + self.pendingcr = False + + # retain last \r even when not translating data: + # then readline() is sure to get \r\n in one pass + if output.endswith("\r") and not final: + output = output[:-1] + self.pendingcr = True + + # Record which newlines are read + crlf = output.count("\r\n") + cr = output.count("\r") - crlf + lf = output.count("\n") - crlf + self.seennl |= ( + (lf and self._LF) | (cr and self._CR) | (crlf and self._CRLF) + ) + + if self.translate: + if crlf: + output = output.replace("\r\n", "\n") + if cr: + output = output.replace("\r", "\n") + + return output + + def getstate(self): + if self.decoder is None: + buf = b"" + flag = 0 + else: + buf, flag = self.decoder.getstate() + flag <<= 1 + if self.pendingcr: + flag |= 1 + return buf, flag + + def setstate(self, state): + buf, flag = state + self.pendingcr = bool(flag & 1) + if self.decoder is not None: + self.decoder.setstate((buf, flag >> 1)) + + def reset(self): + self.seennl = 0 + self.pendingcr = False + if self.decoder is not None: + self.decoder.reset() + + @property + def newlines(self): + return ( + None, + "\n", + "\r", + ("\r", "\n"), + "\r\n", + ("\n", "\r\n"), + ("\r", "\r\n"), + ("\r", "\n", "\r\n"), + )[self.seennl] diff --git a/pyre/pyre-interpreter/src/module/_io/mod.rs b/pyre/pyre-interpreter/src/module/_io/mod.rs index 6ce837a78c9..58183c9a5e8 100644 --- a/pyre/pyre-interpreter/src/module/_io/mod.rs +++ b/pyre/pyre-interpreter/src/module/_io/mod.rs @@ -856,10 +856,9 @@ crate::py_module! { // BytesIO / StringIO are the pure-Python in-memory streams: pickle's // Pickler/Unpickler use BytesIO; logging / traceback / csv use StringIO. appleveldefs: { - "_io_app.py" => ["BytesIO", "StringIO"], + "_io_app.py" => ["BytesIO", "StringIO", "IncrementalNewlineDecoder"], }, functions: { - "IncrementalNewlineDecoder" / * = |_| Ok(w_none()), "open" / * = crate::builtins::builtin_open, // `io.open_code(path)` — `_PyIO_open_code` opens the path in binary // read mode ("rb"); pyre has no audit hooks so it is just `open`. From 4d3df7d1d9ca5ad71730f2cca572e3954b78422a Mon Sep 17 00:00:00 2001 From: "Jeong, YunWon" Date: Thu, 23 Jul 2026 01:29:40 +0900 Subject: [PATCH 4/7] import: make live sys.path authoritative in find_in_sys_path MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit python_sys_path_dirs now returns Option: None only while the sys module does not exist yet (the pre-sys bootstrap window falls back to the native SYS_PATH seed), otherwise Some(dirs) — a missing, non-list, or empty sys.path searches nothing. Add create_sys_path_list to build the initial Python list from the native seed; not yet wired to sys-module creation. Assisted-by: Claude --- pyre/pyre-interpreter/src/importing.rs | 92 +++++++++++++++++--------- 1 file changed, 60 insertions(+), 32 deletions(-) diff --git a/pyre/pyre-interpreter/src/importing.rs b/pyre/pyre-interpreter/src/importing.rs index 2e078d2c041..3de267b9387 100644 --- a/pyre/pyre-interpreter/src/importing.rs +++ b/pyre/pyre-interpreter/src/importing.rs @@ -1733,51 +1733,79 @@ fn find_in_dirs(partname: &str, dirs: &[PathBuf]) -> Option { /// window). Only `str` entries are collected — path hooks (zipimporter keys /// and the like) are not resolvable by the native file search. #[cfg(feature = "host_env")] -fn python_sys_path_dirs() -> Vec { - let Some(sys_mod) = get_sys_module("sys") else { - return Vec::new(); - }; +fn python_sys_path_dirs() -> Option> { + // `None` means the `sys` module does not exist yet (the pre-`sys` bootstrap + // window) — the caller falls back to the native seed. Once `sys` exists the + // Python list is authoritative even when empty: a missing / non-list / empty + // `sys.path` searches nothing, so `del sys.path` and `sys.path.clear()` break + // imports exactly as they do under CPython, rather than resurrecting the seed. + let sys_mod = get_sys_module("sys")?; let w_dict = unsafe { pyre_object::w_module_get_w_dict(sys_mod) }; if w_dict.is_null() { - return Vec::new(); - } - let Some(w_path) = (unsafe { pyre_object::w_dict_getitem_str(w_dict, "path") }) else { - return Vec::new(); - }; - if !unsafe { pyre_object::is_list(w_path) } { - return Vec::new(); + return None; } - let n = unsafe { pyre_object::listobject::w_list_len(w_path) }; - let mut dirs = Vec::with_capacity(n); - for i in 0..n { - if let Some(item) = unsafe { pyre_object::listobject::w_list_getitem(w_path, i as i64) } { - if unsafe { pyre_object::is_str(item) } { - dirs.push(PathBuf::from(unsafe { pyre_object::w_str_get_value(item) })); + // Copy the entries up front so no Python borrow is held across the + // filesystem probes in `find_in_dirs` (which never invoke user code). + let mut dirs = Vec::new(); + if let Some(w_path) = unsafe { pyre_object::w_dict_getitem_str(w_dict, "path") } { + if unsafe { pyre_object::is_list(w_path) } { + let n = unsafe { pyre_object::listobject::w_list_len(w_path) }; + dirs.reserve(n); + for i in 0..n { + if let Some(item) = + unsafe { pyre_object::listobject::w_list_getitem(w_path, i as i64) } + { + // Non-str entries are skipped: pyre's only path hook is the + // native filesystem probe, and CPython also skips an entry no + // hook accepts. + if unsafe { pyre_object::is_str(item) } { + dirs.push(PathBuf::from(unsafe { pyre_object::w_str_get_value(item) })); + } + } } } } - dirs + Some(dirs) } /// Search the import path for a top-level `partname`. /// -/// Python `sys.path` is authoritative — user code (and PYTHONPATH) mutate the -/// Python list and imports must honor it, the same precedence `check_sys_modules` -/// gives the Python `sys.modules` dict. The native `SYS_PATH` is the startup -/// seed: it is appended (deduplicated) so the vendored stdlib and the default -/// entries stay searchable, and it is the sole source in the pre-`sync` -/// bootstrap window where the Python list is still the empty placeholder. +/// The live Python `sys.path` list is authoritative — user code (and PYTHONPATH) +/// mutate it and imports must honor it, the same precedence `check_sys_modules` +/// gives the Python `sys.modules` dict. The native `SYS_PATH` is only the write +/// side of a pre-`sys` staging seed (flushed into `sys.path` at sys-module +/// creation) and is consulted here solely in that pre-`sys` window. #[cfg(feature = "host_env")] fn find_in_sys_path(partname: &str) -> Option { - let mut dirs = python_sys_path_dirs(); - SYS_PATH.with(|p| { - for d in p.borrow().iter() { - if !dirs.contains(d) { - dirs.push(d.clone()); - } - } + match python_sys_path_dirs() { + Some(dirs) => find_in_dirs(partname, &dirs), + None => SYS_PATH.with(|p| find_in_dirs(partname, &p.borrow())), + } +} + +/// Build the initial Python `sys.path` list from the native seed. Run once at +/// sys-module creation so the Python list is fully populated the instant `sys` +/// exists; from then on it is authoritative and `SYS_PATH` is a spent seed. +#[cfg(feature = "host_env")] +pub(crate) fn create_sys_path_list() -> PyObjectRef { + // Force stdlib detection now (off-wasm) so the stdlib is in the list from + // the start rather than lazily on first miss; the `find_module` retry then + // never has to append to the live list. + #[cfg(not(target_arch = "wasm32"))] + ensure_stdlib_path(); + let items: Vec = SYS_PATH.with(|p| { + p.borrow() + .iter() + .map(|d| pyre_object::w_str_new(&d.to_string_lossy())) + .collect() }); - find_in_dirs(partname, &dirs) + pyre_object::w_list_new(items) +} + +/// Off-`host_env` builds have no native seed; `sys.path` starts empty. +#[cfg(not(feature = "host_env"))] +pub(crate) fn create_sys_path_list() -> PyObjectRef { + pyre_object::w_list_new(vec![]) } /// Extract a package module's `__path__` as filesystem directories. From 42669804fd1962311d32ae33ff0243d45e7e1680 Mon Sep 17 00:00:00 2001 From: "Jeong, YunWon" Date: Thu, 23 Jul 2026 02:50:29 +0900 Subject: [PATCH 5/7] import: wire the sys.path seed flush and live append Complete the authoritative sys.path change. - create_sys_path_list flushes the native SYS_PATH seed into sys.path when the sys module is created (register_module), forcing stdlib detection first so the stdlib is on sys.path even under -S / -S -P, before any user code reads it. - add_sys_path appends to the live list in place once sys exists (GC-pinned, deduplicated) and only stages in the seed before that, so sys.path keeps a single stable list object and mutations are never lost. - Delete the one-way sync_python_sys_path mirror and its -m / -c / REPL call sites; it replaced the list object and is superseded by the flush plus append. Assisted-by: Claude --- pyre/pyre-interpreter/src/importing.rs | 126 ++++++++++++--------- pyre/pyre-interpreter/src/module/sys/vm.rs | 6 +- pyre/pyrex/src/lib.rs | 10 +- pyre/pyrex/src/repl.rs | 4 - 4 files changed, 76 insertions(+), 70 deletions(-) diff --git a/pyre/pyre-interpreter/src/importing.rs b/pyre/pyre-interpreter/src/importing.rs index 3de267b9387..c8e1f97c860 100644 --- a/pyre/pyre-interpreter/src/importing.rs +++ b/pyre/pyre-interpreter/src/importing.rs @@ -970,10 +970,7 @@ pub(crate) fn create_builtin_module( /// the handful of builtins imported before that are fixed up in bulk by /// `_bootstrap._setup`'s sys.modules walk, so a no-op here is correct then. #[cfg(feature = "host_env")] -fn set_builtin_module_spec( - name: &str, - module: PyObjectRef, -) -> Result<(), crate::PyError> { +fn set_builtin_module_spec(name: &str, module: PyObjectRef) -> Result<(), crate::PyError> { use pyre_object::gc_roots::{pin_root, push_roots, shadow_stack_get, shadow_stack_len}; let Some(bootstrap) = get_sys_module("importlib._bootstrap") else { @@ -990,13 +987,15 @@ fn set_builtin_module_spec( // executing sees a partially-initialised module whose `BuiltinImporter` / // `_init_module_attrs` are not defined yet. Skip rather than break the // import — `_bootstrap._setup` fixes up any builtin missed here. - let Ok(importer) = crate::baseobjspace::getattr_str(shadow_stack_get(boot_slot), "BuiltinImporter") + let Ok(importer) = + crate::baseobjspace::getattr_str(shadow_stack_get(boot_slot), "BuiltinImporter") else { return Ok(()); }; let importer_slot = shadow_stack_len(); pin_root(importer); - let Ok(find_spec) = crate::baseobjspace::getattr_str(shadow_stack_get(importer_slot), "find_spec") + let Ok(find_spec) = + crate::baseobjspace::getattr_str(shadow_stack_get(importer_slot), "find_spec") else { return Ok(()); }; @@ -1018,7 +1017,8 @@ fn set_builtin_module_spec( pin_root(spec); // _init_module_attrs(spec, module) - let Ok(init) = crate::baseobjspace::getattr_str(shadow_stack_get(boot_slot), "_init_module_attrs") + let Ok(init) = + crate::baseobjspace::getattr_str(shadow_stack_get(boot_slot), "_init_module_attrs") else { return Ok(()); }; @@ -1258,16 +1258,58 @@ pub(crate) fn detect_stdlib_path() -> Option { } } -/// Add a directory to sys.path. +/// Add a directory to `sys.path`. +/// +/// Before the `sys` module exists this stages the entry in the native +/// `SYS_PATH` seed, which is flushed into `sys.path` when `sys` is created. +/// Once `sys` exists the Python list is authoritative, so the entry is appended +/// to it in place (deduplicated) and the spent seed is left untouched. A +/// missing or non-list `sys.path` (e.g. after `del sys.path`) is respected. #[cfg(feature = "host_env")] pub fn add_sys_path(dir: &Path) { - SYS_PATH.with(|p| { - let mut path = p.borrow_mut(); - let pb = dir.to_path_buf(); - if !path.contains(&pb) { - path.push(pb); + use pyre_object::gc_roots::{pin_root, push_roots, shadow_stack_get, shadow_stack_len}; + + let entry = dir.to_string_lossy(); + if get_sys_module("sys").is_none() { + SYS_PATH.with(|p| { + let mut path = p.borrow_mut(); + let pb = dir.to_path_buf(); + if !path.contains(&pb) { + path.push(pb); + } + }); + return; + } + // Pin the new entry before any further allocation (`get_sys_module` and the + // dict lookup allocate) can relocate it. After the list is fetched below the + // path is allocation-free until the pinned entry reaches `w_list_append`. + let _roots = push_roots(); + let slot = shadow_stack_len(); + pin_root(pyre_object::w_str_new(entry.as_ref())); + let Some(sys_mod) = get_sys_module("sys") else { + return; + }; + let w_dict = unsafe { pyre_object::w_module_get_w_dict(sys_mod) }; + if w_dict.is_null() { + return; + } + let Some(w_path) = (unsafe { pyre_object::w_dict_getitem_str(w_dict, "path") }) else { + return; + }; + if !unsafe { pyre_object::is_list(w_path) } { + return; + } + let n = unsafe { pyre_object::listobject::w_list_len(w_path) }; + for i in 0..n { + if let Some(item) = unsafe { pyre_object::listobject::w_list_getitem(w_path, i as i64) } { + if unsafe { pyre_object::is_str(item) } + && unsafe { pyre_object::w_str_get_value(item) } == entry.as_ref() + { + return; + } } - }); + } + unsafe { pyre_object::listobject::w_list_append(w_path, shadow_stack_get(slot)) }; } // ── check_sys_modules ──────────────────────────────────────────────── @@ -1318,41 +1360,6 @@ pub fn get_sys_module(name: &str) -> Option { check_sys_modules(name) } -/// Mirror the native search path (`SYS_PATH`) into Python `sys.path` so -/// `PathFinder` — reached by `importlib.util.find_spec` for top-level module -/// names — can resolve modules. `runpy._get_module_details` (the `-m` entry) -/// drives that path, which is otherwise left empty. -#[cfg(feature = "host_env")] -pub fn sync_python_sys_path() { - // wasm seeds `sys.path` from its bootstrap and has no current_exe/python3 - // lazy stdlib detection, so `ensure_stdlib_path` exists only off-wasm. - #[cfg(not(target_arch = "wasm32"))] - ensure_stdlib_path(); - let items: Vec = SYS_PATH.with(|p| { - p.borrow() - .iter() - .map(|d| pyre_object::w_str_new(&d.to_string_lossy())) - .collect() - }); - if let Some(sys_mod) = get_sys_module("sys") { - // `sys.path` lives in the sys module's own dict; store it with the - // infallible direct dict write the module `setattr` branch reaches - // (`baseobjspace::object_setattr` module arm), avoiding the - // discarded `Result` of the general `setattr_str`. - unsafe { - let w_dict = pyre_object::w_module_get_w_dict(sys_mod); - if !w_dict.is_null() { - pyre_object::w_dict_setitem_str(w_dict, "path", pyre_object::w_list_new(items)); - } - } - } -} - -/// Off-`host_env` builds keep no native `SYS_PATH`, so there is nothing to -/// mirror into Python `sys.path`. -#[cfg(not(feature = "host_env"))] -pub fn sync_python_sys_path() {} - /// The Python-visible `sys.modules` dict, or `PY_NULL` before it is /// installed. Used by callers that need to iterate every loaded module /// (e.g. pickle's `whichmodule` scan). @@ -1644,7 +1651,9 @@ fn find_module(partname: &str, parent_dirs: Option<&[PathBuf]>) -> Option Option { } } -/// Build the initial Python `sys.path` list from the native seed. Run once at -/// sys-module creation so the Python list is fully populated the instant `sys` -/// exists; from then on it is authoritative and `SYS_PATH` is a spent seed. +/// Build the initial Python `sys.path` list from the native `SYS_PATH` seed. +/// Called once at sys-module creation so the Python list is populated the +/// instant `sys` exists; from then on the Python list is authoritative and the +/// seed is spent. +/// +/// Stdlib detection is forced here (off-wasm) so the vendored stdlib is on +/// `sys.path` before any user code — including `python -S` / `-S -P` runs that +/// never import `site` — reads it, matching the unconditional detection the +/// removed `sync_python_sys_path` performed. `sys` is not yet in `sys.modules` +/// during its own creation, so `ensure_stdlib_path`'s `add_sys_path` stages the +/// stdlib in the seed, and the flush below picks it up. #[cfg(feature = "host_env")] pub(crate) fn create_sys_path_list() -> PyObjectRef { - // Force stdlib detection now (off-wasm) so the stdlib is in the list from - // the start rather than lazily on first miss; the `find_module` retry then - // never has to append to the live list. #[cfg(not(target_arch = "wasm32"))] ensure_stdlib_path(); let items: Vec = SYS_PATH.with(|p| { diff --git a/pyre/pyre-interpreter/src/module/sys/vm.rs b/pyre/pyre-interpreter/src/module/sys/vm.rs index e0318de4571..86845dbf938 100644 --- a/pyre/pyre-interpreter/src/module/sys/vm.rs +++ b/pyre/pyre-interpreter/src/module/sys/vm.rs @@ -550,8 +550,10 @@ pub fn register_module(ns: pyre_object::PyObjectRef) { let modules_dict = w_dict_new(); crate::importing::set_sys_modules_dict(modules_dict); module_ns_store(ns, "modules", modules_dict); - // sys.path — empty list placeholder - module_ns_store(ns, "path", w_list_new(vec![])); + // sys.path — flush the native search-path seed into the authoritative list + // the instant `sys` exists; from here on the Python list is the source of + // truth and `add_sys_path` mutates it in place. + module_ns_store(ns, "path", crate::importing::create_sys_path_list()); // sys.stdout/stderr/stdin — `_io.TextIOWrapper`-typed file-like objects. // Real CPython wires these through io.TextIOWrapper around the std fds; // pyre exposes objects of the same type with the minimum surface so diff --git a/pyre/pyrex/src/lib.rs b/pyre/pyrex/src/lib.rs index 99dd4344018..9ca0a3b9b6d 100644 --- a/pyre/pyrex/src/lib.rs +++ b/pyre/pyrex/src/lib.rs @@ -780,9 +780,6 @@ fn run_module(module: &str, no_site: bool) { let result = (|| -> Result<(), pyre_interpreter::PyError> { init_importlib_bootstrap(canonical, ec_ptr)?; - // Mirror the native search path into Python `sys.path` so `PathFinder` - // (used by `find_spec` for top-level module names) can resolve modules. - importing::sync_python_sys_path(); import_site(no_site, canonical, ec_ptr); let runpy = importing::importhook("runpy", canonical, pyre_object::PY_NULL, 0, ec_ptr)?; let func = pyre_interpreter::getattr(runpy, pyre_object::w_str_new("_run_module_as_main"))?; @@ -863,12 +860,9 @@ fn run_source(source: &str, mode: Mode, filename: &str, no_site: bool) { ); } - // `sys.path` is created as an empty placeholder; mirror the native search - // path into it before `site` and user code read it (run_module does the - // same after its importlib bootstrap). `sync_python_sys_path` needs `sys` - // loaded, so import it first. + // Import `sys` up front so its creation flushes the native search-path seed + // into `sys.path` before `site` and user code read it. let _ = importing::importhook("sys", canonical, pyre_object::PY_NULL, 0, ec_ptr); - importing::sync_python_sys_path(); import_site(no_site, canonical, ec_ptr); diff --git a/pyre/pyrex/src/repl.rs b/pyre/pyrex/src/repl.rs index 736fd7a54ce..256bb6979b8 100644 --- a/pyre/pyrex/src/repl.rs +++ b/pyre/pyrex/src/repl.rs @@ -87,10 +87,6 @@ pub fn run_repl(quiet: bool, no_site: bool) { }; configure_sys_for_repl(sys_module); - // Mirror the native search path into Python `sys.path` (an empty - // placeholder until now) before `site` and interactive input read it. - importing::sync_python_sys_path(); - crate::import_site(no_site, canonical, Rc::as_ptr(&execution_context)); let runtime = ReplRuntime { From 4ca7c98326a4797c114a2011ccb4c5852925bd6c Mon Sep 17 00:00:00 2001 From: "Jeong, YunWon" Date: Thu, 23 Jul 2026 11:36:30 +0900 Subject: [PATCH 6/7] import: fall back to the native seed on Windows sys.path miss Windows registers the posix builtin (never nt), so os.path is posixpath and site.removeduppaths() rewrites drive-letter sys.path entries into /D:... garbage at startup, making every stdlib source import fail. Until nt registration lands, a live-list miss on Windows retries the native seed so the stdlib stays importable. Assisted-by: Claude --- pyre/pyre-interpreter/src/importing.rs | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/pyre/pyre-interpreter/src/importing.rs b/pyre/pyre-interpreter/src/importing.rs index c8e1f97c860..19dd67fc5a6 100644 --- a/pyre/pyre-interpreter/src/importing.rs +++ b/pyre/pyre-interpreter/src/importing.rs @@ -1787,7 +1787,18 @@ fn python_sys_path_dirs() -> Option> { #[cfg(feature = "host_env")] fn find_in_sys_path(partname: &str) -> Option { match python_sys_path_dirs() { - Some(dirs) => find_in_dirs(partname, &dirs), + Some(dirs) => { + let found = find_in_dirs(partname, &dirs); + // Windows: pyre still registers the `posix` builtin (never `nt`), + // so `os.path` is posixpath and `site.removeduppaths()` rewrites + // every drive-letter `sys.path` entry into `/D:\...` garbage + // at startup. Until the `nt` registration lands, a live-list miss + // falls back to the native seed so the stdlib stays importable. + #[cfg(windows)] + let found = + found.or_else(|| SYS_PATH.with(|p| find_in_dirs(partname, &p.borrow()))); + found + } None => SYS_PATH.with(|p| find_in_dirs(partname, &p.borrow())), } } From 4690e798048887cd6f90e9c45f19482f2edb6892 Mon Sep 17 00:00:00 2001 From: "Jeong, YunWon" Date: Thu, 23 Jul 2026 11:36:37 +0900 Subject: [PATCH 7/7] address review: PYTHONPATH env gating, _fix_up_module errors, decoder validation - Skip PYTHONPATH under -E / -I (ignore_environment); split it on the platform path-list separator and keep empty components, extending the seed with the raw split (app_main.setup_and_fix_paths). - Propagate errors raised by _fix_up_module instead of mapping them to the None-seeding fallback; the getattr fallback for the partially initialised bootstrap window stays. - IncrementalNewlineDecoder: accept errors=None as strict, reject non-str errors and unencodable handler names, coerce translate via __index__, and raise TypeError when a wrapped decoder returns a non-str result (interp_textio.py); pass final positionally. Assisted-by: Claude --- pyre/pyre-interpreter/src/importing.rs | 42 +++++++++---------- .../src/module/_io/_io_app.py | 23 +++++++++- 2 files changed, 43 insertions(+), 22 deletions(-) diff --git a/pyre/pyre-interpreter/src/importing.rs b/pyre/pyre-interpreter/src/importing.rs index 19dd67fc5a6..0e47ed133c1 100644 --- a/pyre/pyre-interpreter/src/importing.rs +++ b/pyre/pyre-interpreter/src/importing.rs @@ -1086,7 +1086,9 @@ fn fix_up_source_module_spec( }; let fix_slot = shadow_stack_len(); pin_root(fix); - if crate::call::call_function_impl_result( + // An error raised by `_fix_up_module` itself propagates — the appexec + // at importing.py:293-298 does not shield the call. + crate::call::call_function_impl_result( shadow_stack_get(fix_slot), &[ shadow_stack_get(ns_slot), @@ -1094,11 +1096,7 @@ fn fix_up_source_module_spec( shadow_stack_get(path_slot), shadow_stack_get(cpath_slot), ], - ) - .is_err() - { - return Ok(false); - } + )?; Ok(true) } @@ -1165,22 +1163,25 @@ pub fn init_sys_path(script_dir: &Path) { } } // PYTHONPATH entries follow the script/cwd seed and precede the - // lazily-detected stdlib (pathconfig.c). Honoured regardless of the - // safe-path flag, which only suppresses the script/cwd seed. The - // sandbox interpreter takes its search path from the controller, so it - // does not read the host environment here. + // stdlib (pathconfig.c), split on the platform path-list separator. + // Honoured regardless of the safe-path flag, which only suppresses the + // script/cwd seed, but skipped under `-E` / `-I` (ignore_environment), + // which ignore every `PYTHON*` variable. The sandbox interpreter takes + // its search path from the controller, so it does not read the host + // environment here. #[cfg(not(feature = "sandbox"))] - if let Ok(pythonpath) = host_os::var("PYTHONPATH") { - for entry in pythonpath.split(':').filter(|e| !e.is_empty()) { - let pb = PathBuf::from(entry); - if !path.contains(&pb) { - path.push(pb); - } + if !ignore_environment_flag() { + if let Ok(pythonpath) = host_os::var("PYTHONPATH") { + let sep = if cfg!(windows) { ';' } else { ':' }; + // Empty components are preserved — an empty `sys.path` entry + // denotes the current directory (app_main.setup_and_fix_paths + // extends with the raw split). + path.extend(pythonpath.split(sep).map(PathBuf::from)); } } - // The stdlib path is detected lazily on first stdlib import to avoid - // spawning a python3 subprocess on every startup. - // See find_module() → ensure_stdlib_path(). + // The stdlib entry is appended when the `sys` module is created — + // `create_sys_path_list` forces `ensure_stdlib_path` before flushing + // this seed into `sys.path`. }); } @@ -1795,8 +1796,7 @@ fn find_in_sys_path(partname: &str) -> Option { // at startup. Until the `nt` registration lands, a live-list miss // falls back to the native seed so the stdlib stays importable. #[cfg(windows)] - let found = - found.or_else(|| SYS_PATH.with(|p| find_in_dirs(partname, &p.borrow()))); + let found = found.or_else(|| SYS_PATH.with(|p| find_in_dirs(partname, &p.borrow()))); found } None => SYS_PATH.with(|p| find_in_dirs(partname, &p.borrow())), diff --git a/pyre/pyre-interpreter/src/module/_io/_io_app.py b/pyre/pyre-interpreter/src/module/_io/_io_app.py index b461fbe5213..3c5b8c84e84 100644 --- a/pyre/pyre-interpreter/src/module/_io/_io_app.py +++ b/pyre/pyre-interpreter/src/module/_io/_io_app.py @@ -384,6 +384,25 @@ class IncrementalNewlineDecoder: _CRLF = 4 def __init__(self, decoder, translate, errors="strict"): + if errors is None: + errors = "strict" + elif not isinstance(errors, str): + raise TypeError( + "TextIOWrapper() argument 'errors' must be str or None, not %s" + % type(errors).__name__ + ) + else: + # io_check_errors minus the dev-mode handler lookup — a codecs + # import here would recurse through decode_source. + errors.encode("utf-8", "strict") + if not isinstance(translate, int): + try: + translate = translate.__index__() + except AttributeError: + raise TypeError( + "'%s' object cannot be interpreted as an integer" + % type(translate).__name__ + ) from None self.errors = errors self.translate = translate self.decoder = decoder @@ -395,7 +414,9 @@ def decode(self, input, final=False): if self.decoder is None: output = input else: - output = self.decoder.decode(input, final=final) + output = self.decoder.decode(input, final) + if not isinstance(output, str): + raise TypeError("decoder should return a string result") if self.pendingcr and (output or final): output = "\r" + output self.pendingcr = False