socket.ioctl, if_nameindex over IP Helper, _locale.getencoding, the legacy Windows filesystem codec, and 13 posix interpreter-release guards

pyre/pyre-interpreter/Cargo.toml

@@ -108,6 +108,8 @@ windows-sys = { version = "0.61", features = [
     "Win32_Foundation",
     "Win32_Globalization",
     "Win32_Media_Audio",
+    "Win32_NetworkManagement_IpHelper",
+    "Win32_NetworkManagement_Ndis",
     "Win32_Networking_WinSock",
     "Win32_Storage_FileSystem",
     "Win32_Security",

pyre/pyre-interpreter/src/builtins.rs

@@ -17360,16 +17360,16 @@ pub fn builtin_open(args: &[PyObjectRef]) -> Result<PyObjectRef, crate::PyError>
     // The console stream is a host-console type, so a sandbox build — which
     // compiles it out — keeps the plain `FileIO` construction.
     #[cfg(all(windows, feature = "host_env", not(feature = "sandbox")))]
-    let raw_type = {
+    let (raw_type, console) = {
         let file = pyre_object::gc_roots::shadow_stack_get(file_slot);
         if crate::module::_io::winconsoleio::pyio_get_console_type(file) != '\0' {
-            crate::module::_io::windows_console_io_type()
+            (crate::module::_io::windows_console_io_type(), true)
         } else {
-            crate::module::_io::fileio_type()
+            (crate::module::_io::fileio_type(), false)
         }
     };
     #[cfg(not(all(windows, feature = "host_env", not(feature = "sandbox"))))]
-    let raw_type = crate::module::_io::fileio_type();
+    let (raw_type, console) = (crate::module::_io::fileio_type(), false);
     let raw = crate::call::call_function_impl_result(
         raw_type,
         &[
@@ -17445,12 +17445,20 @@ pub fn builtin_open(args: &[PyObjectRef]) -> Result<PyObjectRef, crate::PyError>
             return Ok(pyre_object::gc_roots::shadow_stack_get(buffer_slot));
         }
 
-        // This native open call does not add a Python frame between the user
-        // and `_io.text_encoding`; one frame of warning stack depth is enough.
-        let resolved_encoding = crate::module::_io::text_encoding(
-            pyre_object::gc_roots::shadow_stack_get(encoding_slot),
-            1,
-        )?;
+        // `_open` writes `encoding = "utf-8"` in the same step that picks the
+        // console class, ahead of the argument the caller gave: a console
+        // stream reports no other codec, and never counts as relying on the
+        // default one.  Everything else goes through `_io.text_encoding`, and
+        // this native open call adds no Python frame between the user and it,
+        // so one frame of warning stack depth is enough.
+        let resolved_encoding = if console {
+            w_str_new("utf-8")
+        } else {
+            crate::module::_io::text_encoding(
+                pyre_object::gc_roots::shadow_stack_get(encoding_slot),
+                1,
+            )?
+        };
         pyre_object::gc_roots::pin_root(resolved_encoding);
         let resolved_encoding_slot = pyre_object::gc_roots::shadow_stack_len() - 1;
         let wrapper = crate::call::call_function_impl_result(

pyre/pyre-interpreter/src/gateway.rs

@@ -1612,6 +1612,51 @@ pub fn fsencode_os_str(name: &std::ffi::OsStr) -> Vec<u8> {
     }
 }
 
+/// A path argument the caller spelled as `bytes`, in the units the
+/// interpreter carries a name in.
+///
+/// `path_converter` decodes such an argument with `PyUnicode_DecodeFSDefault`
+/// and keeps only the wide string it produced, so the code page pair
+/// `sys._enablelegacywindowsfsencoding` installs is read here, at the
+/// boundary, and never again downstream: everything past this point — the
+/// stored `co_filename`, the `wide_path` each syscall takes — holds the one
+/// spelling a `str` argument already arrives in.  The two spellings coincide
+/// outside that mode, where the bytes are their own answer.
+pub fn fs_arg_bytes(data: Vec<u8>) -> Result<Vec<u8>, crate::PyError> {
+    #[cfg(windows)]
+    if crate::typedef::legacy_windows_fs_encoding() {
+        return crate::unicodehelper_win32::decode_code_page(
+            windows_sys::Win32::Globalization::CP_ACP,
+            &data,
+            "replace",
+            true,
+        )
+        .map(|(text, _)| text.as_bytes().to_vec());
+    }
+    Ok(data)
+}
+
+/// [`fs_arg_bytes`]'s other direction: a host name handed back to a caller
+/// that asked for its path in `bytes`, which each of those sites spells with
+/// `PyUnicode_EncodeFSDefault` over the name it read as wide.
+///
+/// Total, like [`fsencode_os_str`]: the name came from the host, and the
+/// substitution `replace` makes is the answer rather than a failure.  Only a
+/// Win32 error ends the encode, and there is no caller here to report one to,
+/// so the interpreter's own spelling stands in for it.
+pub fn fs_result_bytes(data: &[u8]) -> Vec<u8> {
+    #[cfg(windows)]
+    if crate::typedef::legacy_windows_fs_encoding()
+        && let Ok(bytes) = crate::unicodehelper_win32::encode_code_page_replace(
+            windows_sys::Win32::Globalization::CP_ACP,
+            &crate::typedef::fsdecode_wtf8_total(data),
+        )
+    {
+        return bytes;
+    }
+    data.to_vec()
+}
+
 /// [`fsencode_os_str`]'s other direction: the host name filesystem bytes
 /// spell, for a caller that has to hand them to an API taking an `OsStr`.
 pub fn os_string_from_fs_bytes(data: &[u8]) -> std::ffi::OsString {
@@ -1763,7 +1808,7 @@ fn path_or_fd_w(
             // `bytearray` is now turned away by the same message every other
             // rejected type gets.
             (
-                pyre_object::bytesobject::w_bytes_data(obj).to_vec(),
+                fs_arg_bytes(pyre_object::bytesobject::w_bytes_data(obj).to_vec())?,
                 obj_slot,
                 -1,
             )
@@ -1845,7 +1890,7 @@ fn path_or_fd_w(
             // `__fspath__`; a `bytearray` is a readable buffer but not a path.
             if pyre_object::bytesobject::is_bytes(result) {
                 (
-                    pyre_object::bytesobject::w_bytes_data(result).to_vec(),
+                    fs_arg_bytes(pyre_object::bytesobject::w_bytes_data(result).to_vec())?,
                     result_slot,
                     -1,
                 )

pyre/pyre-interpreter/src/importing.rs

@@ -3138,6 +3138,12 @@ pub fn set_runtime_flags(flags: &crate::launch_env::LaunchFlags) {
     SYS_ISOLATED.store(flags.isolated, Ordering::Relaxed);
     SYS_DEV_MODE.store(flags.dev_mode, Ordering::Relaxed);
     SYS_WARN_DEFAULT_ENCODING.store(flags.warn_default_encoding, Ordering::Relaxed);
+    // The filesystem codec is picked in `preconfig_read`, ahead of every
+    // import, so `os`'s own `_fscodec` closes over the legacy pair rather than
+    // the one `sys._enablelegacywindowsfsencoding` leaves it holding.
+    #[cfg(windows)]
+    crate::typedef::LEGACY_WINDOWS_FS_ENCODING
+        .store(flags.legacy_windows_fs_encoding, Ordering::Relaxed);
     SYS_UTF8_MODE.store(flags.utf8_mode.unwrap_or(0), Ordering::Relaxed);
     SYS_SAFE_PATH.store(flags.safe_path, Ordering::Relaxed);
     SYS_OPTIMIZE.store(flags.optimize, Ordering::Relaxed);

pyre/pyre-interpreter/src/launch_env.rs

@@ -25,6 +25,10 @@ pub struct LaunchFlags {
     pub isolated: bool,
     pub dev_mode: bool,
     pub warn_default_encoding: bool,
+    /// PYTHONLEGACYWINDOWSFSENCODING, which no command-line option spells.
+    /// Read only on Windows, the only platform whose `PyPreConfig` carries it,
+    /// and left false everywhere else.
+    pub legacy_windows_fs_encoding: bool,
     /// `None` until [`finalize`] resolves it; a command line that named
     /// `-X utf8` carries that value through instead.
     pub utf8_mode: Option<i64>,
@@ -83,6 +87,7 @@ pub const LAUNCH_ENV_NAMES: &[&str] = &[
     "PYTHONDONTWRITEBYTECODE",
     "PYTHONUTF8",
     "PYTHONWARNDEFAULTENCODING",
+    "PYTHONLEGACYWINDOWSFSENCODING",
     "PYTHONWARNINGS",
     "PYTHONIOENCODING",
     "LC_ALL",
@@ -256,6 +261,17 @@ pub fn finalize(mut flags: LaunchFlags) -> Result<LaunchFlags, PreConfigError> {
         flags.warn_default_encoding,
         "PYTHONWARNDEFAULTENCODING",
     );
+    // Read in `preconfig_read`, so it takes the same integer fold as the
+    // other variables there rather than the presence one: `=0` leaves the
+    // filesystem codec on the PEP 529 pair.
+    #[cfg(windows)]
+    {
+        flags.legacy_windows_fs_encoding = fold_int_flag(
+            &flags,
+            flags.legacy_windows_fs_encoding,
+            "PYTHONLEGACYWINDOWSFSENCODING",
+        );
+    }
     flags.stdio_encoding = if flags.ignore_environment {
         None
     } else {

pyre/pyre-interpreter/src/module/_io/mod.rs

@@ -1350,7 +1350,7 @@ pub(crate) fn text_encoding(
     }
     if crate::importing::warn_default_encoding_flag() {
         crate::warn::warn_category(
-            "'encoding' argument not specified.",
+            "'encoding' argument not specified",
             "EncodingWarning",
             stacklevel + 1,
         )?;

pyre/pyre-interpreter/src/module/_io/textio.rs

@@ -415,11 +415,14 @@ impl W_TextIOWrapper {
         Ok(())
     }
 
-    /// `encoding="locale"` selects the current locale's encoding; the sandbox
-    /// and default environment resolve that to UTF-8.
+    /// `encoding="locale"` selects the current locale's encoding, which is
+    /// what `_Py_GetLocaleEncodingObject` answers for it and for the
+    /// unspecified argument that resolves to it.  A sandbox build has no host
+    /// locale to ask and reads utf-8, the answer a `_Py_FORCE_UTF8_LOCALE`
+    /// build gives without asking one.
     fn resolve_locale_encoding(encoding: String) -> String {
         if encoding == "locale" {
-            "utf-8".to_string()
+            crate::module::_locale::interp_locale::locale_encoding()
         } else {
             encoding
         }
@@ -1050,25 +1053,47 @@ impl W_TextIOWrapper {
         #[default(pyre_object::w_bool_from(false))] line_buffering: PyObjectRef,
         #[default(pyre_object::w_bool_from(false))] write_through: PyObjectRef,
     ) -> Result<(), crate::PyError> {
+        // Every argument is converted before the body runs, in the order the
+        // signature gives them: `encoding` and `newline` are accepted only as
+        // `str` or `None`, and the two flags are truth-tested.  3.14's
+        // constructor uses the `bool` converter for those, unlike
+        // `reconfigure`'s `int` one — an object with `__bool__` but no
+        // `__index__` is accepted here — and a `__bool__` that raises ends
+        // the call before `self->ok = 0`, leaving a stream that was already
+        // open still open.
+        let unspecified = if crate::importing::utf8_mode_flag() != 0 {
+            "utf-8"
+        } else {
+            "locale"
+        };
+        let encoding_text = Self::checked_text0(encoding, unspecified, "encoding")?;
+        if !unsafe { pyre_object::is_none(newline) || pyre_object::is_str(newline) } {
+            return Err(crate::PyError::type_error("illegal newline type"));
+        }
+        let line_buffering = crate::baseobjspace::is_true(line_buffering)?;
+        let write_through = crate::baseobjspace::is_true(write_through)?;
+
         // PyPy starts every initialization attempt in STATE_ZERO.  A failed
         // reinitialization must leave all I/O operations uninitialized.
         self.state = STATE_ZERO;
         self.w_buffer = PY_NULL;
         pyre_object::gc_hook::try_gc_write_barrier(self as *mut Self as *mut u8);
 
-        let encoding =
-            Self::resolve_locale_encoding(Self::checked_text0(encoding, "utf-8", "encoding")?);
+        // An unspecified `encoding` reads the locale's, unless UTF-8 mode has
+        // already answered the question.  `_io.text_encoding` is not on this
+        // path - a direct `TextIOWrapper(...)` call reaches the constructor
+        // itself - so the warning that argument's absence carries is raised
+        // here, at this frame.
+        if unsafe { pyre_object::is_none(encoding) }
+            && crate::importing::warn_default_encoding_flag()
+        {
+            crate::warn::warn_category("'encoding' argument not specified", "EncodingWarning", 1)?;
+        }
         let errors = Self::checked_text0(errors, "strict", "errors")?;
         Self::io_check_errors(&errors)?;
         let newline_value = Self::unwrap_newline(newline)?;
+        let encoding = Self::resolve_locale_encoding(encoding_text);
         let codec = Self::lookup_text_codec(&encoding)?;
-        // 3.14's constructor uses the `bool` Argument Clinic converter (truth
-        // testing), unlike `reconfigure`'s `int` converter — an object with
-        // `__bool__` but no `__index__` is accepted here.  Argument parsing
-        // runs before the body, so an object whose `__bool__` raises leaves
-        // the stream unattached rather than half filled in.
-        let line_buffering = crate::baseobjspace::is_true(line_buffering)?;
-        let write_through = crate::baseobjspace::is_true(write_through)?;
 
         self.attach_buffer(
             buffer,

pyre/pyre-interpreter/src/module/_io/winconsoleio.rs

@@ -235,7 +235,12 @@ impl W_WindowsConsoleIO {
                     "Cannot use closefd=False with file name",
                 ));
             }
-            let path = crate::gateway::fsencode_path_w(name_obj)?;
+            // `closefd` is whatever the caller passed, so `is_true` can run a
+            // `__bool__` and a moving collection with it: the name comes back
+            // out of its slot rather than from the borrow taken above.
+            let path = crate::gateway::fsencode_path_w(pyre_object::gc_roots::shadow_stack_get(
+                name_slot,
+            ))?;
             let os_name = crate::gateway::os_string_from_fs_bytes(&path.as_bytes);
             let mut kind = host_nt::console_type_from_name(&os_name.to_string_lossy());
             if kind == 'x' {
@@ -293,8 +298,10 @@ impl W_WindowsConsoleIO {
         // `W_IOBase.__init__` installs the per-instance closed flag.  A
         // successful second `__init__` reopens the same object, so its base
         // flush/close methods must observe the new live state as well as this
-        // payload's fd-backed `closed` property.
-        super::iobase_set_internal_closed(this.self_obj(), false)?;
+        // payload's fd-backed `closed` property.  Storing `name` allocates
+        // the instance dict, so the receiver is read back out of its slot
+        // rather than taken from the borrow held across that store.
+        super::iobase_set_internal_closed(Self::from_slot(self_slot).self_obj(), false)?;
         Ok(())
     }
 

pyre/pyre-interpreter/src/module/_locale/interp_locale.rs

@@ -20,6 +20,51 @@ type CollationArg = Vec<u16>;
 #[cfg(not(windows))]
 type CollationArg = std::ffi::CString;
 
+/// `_Py_GetLocaleEncoding` — the codeset the active `LC_CTYPE` names.
+///
+/// `locale.py` takes `_locale.getencoding` when the module carries it and
+/// falls back to `sys.getfilesystemencoding()` when it does not, and the two
+/// are different answers wherever the filesystem encoding is not the locale
+/// one — every Windows host, where PEP 529 makes the filesystem utf-8.
+///
+/// `_io.TextIOWrapper` reads it as well, for the `encoding="locale"` its
+/// unspecified argument resolves to.
+#[cfg(all(windows, not(feature = "sandbox")))]
+pub(crate) fn locale_encoding() -> String {
+    // The active ANSI code page, spelled `cp<n>` whatever it is: code page
+    // 65001 answers `cp65001`, not `utf-8`.
+    format!("cp{}", unsafe {
+        windows_sys::Win32::Globalization::GetACP()
+    })
+}
+
+#[cfg(all(
+    unix,
+    feature = "host_env",
+    not(feature = "sandbox"),
+    not(any(target_os = "ios", target_os = "android", target_os = "redox"))
+))]
+pub(crate) fn locale_encoding() -> String {
+    match rustpython_host_env::locale::nl_langinfo_codeset() {
+        // An empty codeset answers utf-8: `nl_langinfo` returns one on macOS
+        // when the `LC_CTYPE` locale is not supported.
+        Some(bytes) if !bytes.is_empty() => String::from_utf8_lossy(&bytes).into_owned(),
+        _ => "utf-8".to_string(),
+    }
+}
+
+#[cfg(not(any(all(windows, not(feature = "sandbox")), all(
+    unix,
+    feature = "host_env",
+    not(feature = "sandbox"),
+    not(any(target_os = "ios", target_os = "android", target_os = "redox"))
+))))]
+pub(crate) fn locale_encoding() -> String {
+    // No host locale to ask, which is the answer `_Py_FORCE_UTF8_LOCALE`
+    // builds give without asking one.
+    "utf-8".to_string()
+}
+
 fn collation_arg(obj: pyre_object::PyObjectRef) -> Result<CollationArg, crate::PyError> {
     let text = crate::baseobjspace::str_utf8_w(obj)?.to_string();
     #[cfg(windows)]
@@ -599,12 +644,13 @@ pub fn register_module(ns: pyre_object::PyObjectRef) {
             1,
         ),
     );
+    // `_locale.getencoding` — `_Py_GetLocaleEncodingObject`.
     crate::module_ns_store(
         ns,
         "getencoding",
         crate::make_builtin_function_with_arity(
             "getencoding",
-            |_| Ok(pyre_object::w_str_new("utf-8")),
+            |_| Ok(pyre_object::w_str_new(&locale_encoding())),
             0,
         ),
     );