From 82fcdccbc500812ed6d5cbed09f8f36d90bddf66 Mon Sep 17 00:00:00 2001 From: Steve Dower Date: Wed, 13 Dec 2023 15:38:45 +0000 Subject: [PATCH] gh-111650: Ensure pyconfig.h includes Py_GIL_DISABLED on Windows (GH-112778) --- .github/workflows/reusable-windows.yml | 6 ++-- Lib/sysconfig/__init__.py | 11 ++++++- Lib/test/test_sysconfig.py | 8 +++-- Lib/test/test_venv.py | 18 ++++++----- ...-12-05-22-56-30.gh-issue-111650.xlWmvM.rst | 3 ++ Modules/_ctypes/_ctypes_test.c | 2 -- Modules/_scproxy.c | 2 -- Modules/_stat.c | 2 -- Modules/_testcapi/heaptype_relative.c | 2 -- Modules/_testcapi/vectorcall_limited.c | 2 -- Modules/_testclinic_limited.c | 2 -- Modules/_testimportmultiple.c | 2 -- Modules/_uuidmodule.c | 2 -- Modules/errnomodule.c | 2 -- Modules/resource.c | 2 -- Modules/xxlimited.c | 2 -- Modules/xxlimited_35.c | 2 -- PC/layout/main.py | 8 +++-- PC/{pyconfig.h => pyconfig.h.in} | 3 ++ PC/winsound.c | 2 ++ PCbuild/_freeze_module.vcxproj | 30 ++++++++++++++++++ PCbuild/pyproject.props | 6 ++-- PCbuild/pythoncore.vcxproj | 31 ++++++++++++++++++- Tools/msi/dev/dev_files.wxs | 2 +- Tools/peg_generator/pegen/build.py | 4 +++ 25 files changed, 112 insertions(+), 44 deletions(-) create mode 100644 Misc/NEWS.d/next/Windows/2023-12-05-22-56-30.gh-issue-111650.xlWmvM.rst rename PC/{pyconfig.h => pyconfig.h.in} (99%) diff --git a/.github/workflows/reusable-windows.yml b/.github/workflows/reusable-windows.yml index 29e0a7e35b54501..47a3c10d2ca4c1e 100644 --- a/.github/workflows/reusable-windows.yml +++ b/.github/workflows/reusable-windows.yml @@ -16,7 +16,7 @@ jobs: steps: - uses: actions/checkout@v4 - name: Build CPython - run: .\PCbuild\build.bat -e -d -p Win32 ${{ inputs.free-threaded && '--disable-gil' || '' }} + run: .\PCbuild\build.bat -e -d -v -p Win32 ${{ inputs.free-threaded && '--disable-gil' || '' }} - name: Display build info run: .\python.bat -m test.pythoninfo - name: Tests @@ -33,7 +33,7 @@ jobs: - name: Register MSVC problem matcher run: echo "::add-matcher::.github/problem-matchers/msvc.json" - name: Build CPython - run: .\PCbuild\build.bat -e -d -p x64 ${{ inputs.free-threaded && '--disable-gil' || '' }} + run: .\PCbuild\build.bat -e -d -v -p x64 ${{ inputs.free-threaded && '--disable-gil' || '' }} - name: Display build info run: .\python.bat -m test.pythoninfo - name: Tests @@ -50,4 +50,4 @@ jobs: - name: Register MSVC problem matcher run: echo "::add-matcher::.github/problem-matchers/msvc.json" - name: Build CPython - run: .\PCbuild\build.bat -e -d -p arm64 ${{ inputs.free-threaded && '--disable-gil' || '' }} + run: .\PCbuild\build.bat -e -d -v -p arm64 ${{ inputs.free-threaded && '--disable-gil' || '' }} diff --git a/Lib/sysconfig/__init__.py b/Lib/sysconfig/__init__.py index 2a7fa45be079de7..c60c9f3440615be 100644 --- a/Lib/sysconfig/__init__.py +++ b/Lib/sysconfig/__init__.py @@ -404,7 +404,16 @@ def get_config_h_filename(): """Return the path of pyconfig.h.""" if _PYTHON_BUILD: if os.name == "nt": - inc_dir = os.path.join(_PROJECT_BASE, "PC") + # This ought to be as simple as dirname(sys._base_executable), but + # if a venv uses symlinks to a build in the source tree, then this + # fails. So instead we guess the subdirectory name from sys.winver + if sys.winver.endswith('-32'): + arch = 'win32' + elif sys.winver.endswith('-arm64'): + arch = 'arm64' + else: + arch = 'amd64' + inc_dir = os.path.join(_PROJECT_BASE, 'PCbuild', arch) else: inc_dir = _PROJECT_BASE else: diff --git a/Lib/test/test_sysconfig.py b/Lib/test/test_sysconfig.py index 2a6813f00bccc65..a19c04b1b2cde55 100644 --- a/Lib/test/test_sysconfig.py +++ b/Lib/test/test_sysconfig.py @@ -472,11 +472,15 @@ def test_srcdir(self): # should be a full source checkout. Python_h = os.path.join(srcdir, 'Include', 'Python.h') self.assertTrue(os.path.exists(Python_h), Python_h) - # /PC/pyconfig.h always exists even if unused on POSIX. - pyconfig_h = os.path.join(srcdir, 'PC', 'pyconfig.h') + # /PC/pyconfig.h.in always exists even if unused + pyconfig_h = os.path.join(srcdir, 'PC', 'pyconfig.h.in') self.assertTrue(os.path.exists(pyconfig_h), pyconfig_h) pyconfig_h_in = os.path.join(srcdir, 'pyconfig.h.in') self.assertTrue(os.path.exists(pyconfig_h_in), pyconfig_h_in) + if os.name == 'nt': + # /pyconfig.h exists on Windows in a build tree + pyconfig_h = os.path.join(sys.executable, '..', 'pyconfig.h') + self.assertTrue(os.path.exists(pyconfig_h), pyconfig_h) elif os.name == 'posix': makefile_dir = os.path.dirname(sysconfig.get_makefile_filename()) # Issue #19340: srcdir has been realpath'ed already diff --git a/Lib/test/test_venv.py b/Lib/test/test_venv.py index 890672c5d27eec4..617d14dcb9c5fe6 100644 --- a/Lib/test/test_venv.py +++ b/Lib/test/test_venv.py @@ -46,14 +46,18 @@ def check_output(cmd, encoding=None): p = subprocess.Popen(cmd, stdout=subprocess.PIPE, - stderr=subprocess.PIPE, - encoding=encoding) + stderr=subprocess.PIPE) out, err = p.communicate() if p.returncode: if verbose and err: - print(err.decode('utf-8', 'backslashreplace')) + print(err.decode(encoding or 'utf-8', 'backslashreplace')) raise subprocess.CalledProcessError( p.returncode, cmd, out, err) + if encoding: + return ( + out.decode(encoding, 'backslashreplace'), + err.decode(encoding, 'backslashreplace'), + ) return out, err class BaseTest(unittest.TestCase): @@ -281,8 +285,8 @@ def test_sysconfig(self): ('get_config_h_filename()', sysconfig.get_config_h_filename())): with self.subTest(call): cmd[2] = 'import sysconfig; print(sysconfig.%s)' % call - out, err = check_output(cmd) - self.assertEqual(out.strip(), expected.encode(), err) + out, err = check_output(cmd, encoding='utf-8') + self.assertEqual(out.strip(), expected, err) @requireVenvCreate @unittest.skipUnless(can_symlink(), 'Needs symlinks') @@ -303,8 +307,8 @@ def test_sysconfig_symlinks(self): ('get_config_h_filename()', sysconfig.get_config_h_filename())): with self.subTest(call): cmd[2] = 'import sysconfig; print(sysconfig.%s)' % call - out, err = check_output(cmd) - self.assertEqual(out.strip(), expected.encode(), err) + out, err = check_output(cmd, encoding='utf-8') + self.assertEqual(out.strip(), expected, err) if sys.platform == 'win32': ENV_SUBDIRS = ( diff --git a/Misc/NEWS.d/next/Windows/2023-12-05-22-56-30.gh-issue-111650.xlWmvM.rst b/Misc/NEWS.d/next/Windows/2023-12-05-22-56-30.gh-issue-111650.xlWmvM.rst new file mode 100644 index 000000000000000..5a3493356e30bee --- /dev/null +++ b/Misc/NEWS.d/next/Windows/2023-12-05-22-56-30.gh-issue-111650.xlWmvM.rst @@ -0,0 +1,3 @@ +Ensures the ``Py_GIL_DISABLED`` preprocessor variable is defined in +:file:`pyconfig.h` so that extension modules written in C are able to use +it. diff --git a/Modules/_ctypes/_ctypes_test.c b/Modules/_ctypes/_ctypes_test.c index fc9fc131f6249a6..2681b9c58ecb9da 100644 --- a/Modules/_ctypes/_ctypes_test.c +++ b/Modules/_ctypes/_ctypes_test.c @@ -1,6 +1,4 @@ -#ifndef _MSC_VER #include "pyconfig.h" // Py_GIL_DISABLED -#endif #ifndef Py_GIL_DISABLED // Need limited C API version 3.12 for Py_MOD_PER_INTERPRETER_GIL_SUPPORTED diff --git a/Modules/_scproxy.c b/Modules/_scproxy.c index 7920d2c2b8739dd..fe82e918677f9a3 100644 --- a/Modules/_scproxy.c +++ b/Modules/_scproxy.c @@ -3,9 +3,7 @@ * using the SystemConfiguration framework. */ -#ifndef _MSC_VER #include "pyconfig.h" // Py_GIL_DISABLED -#endif #ifndef Py_GIL_DISABLED // Need limited C API version 3.12 for Py_MOD_PER_INTERPRETER_GIL_SUPPORTED diff --git a/Modules/_stat.c b/Modules/_stat.c index 1ef1e97f4b7dcae..80f8a92668976b7 100644 --- a/Modules/_stat.c +++ b/Modules/_stat.c @@ -11,9 +11,7 @@ * */ -#ifndef _MSC_VER #include "pyconfig.h" // Py_GIL_DISABLED -#endif #ifndef Py_GIL_DISABLED // Need limited C API version 3.13 for PyModule_Add() on Windows diff --git a/Modules/_testcapi/heaptype_relative.c b/Modules/_testcapi/heaptype_relative.c index 52286f05f7154ce..52bda75736b3167 100644 --- a/Modules/_testcapi/heaptype_relative.c +++ b/Modules/_testcapi/heaptype_relative.c @@ -1,6 +1,4 @@ -#ifndef _MSC_VER #include "pyconfig.h" // Py_GIL_DISABLED -#endif #ifndef Py_GIL_DISABLED #define Py_LIMITED_API 0x030c0000 // 3.12 diff --git a/Modules/_testcapi/vectorcall_limited.c b/Modules/_testcapi/vectorcall_limited.c index 0a650f1b351d2dd..d7b8d33b7f71620 100644 --- a/Modules/_testcapi/vectorcall_limited.c +++ b/Modules/_testcapi/vectorcall_limited.c @@ -1,8 +1,6 @@ /* Test Vectorcall in the limited API */ -#ifndef _MSC_VER #include "pyconfig.h" // Py_GIL_DISABLED -#endif #ifndef Py_GIL_DISABLED #define Py_LIMITED_API 0x030c0000 // 3.12 diff --git a/Modules/_testclinic_limited.c b/Modules/_testclinic_limited.c index 61bc84134458dab..ef595be0b626db8 100644 --- a/Modules/_testclinic_limited.c +++ b/Modules/_testclinic_limited.c @@ -4,9 +4,7 @@ #undef Py_BUILD_CORE_MODULE #undef Py_BUILD_CORE_BUILTIN -#ifndef _MSC_VER #include "pyconfig.h" // Py_GIL_DISABLED -#endif #ifndef Py_GIL_DISABLED // For now, only limited C API 3.13 is supported diff --git a/Modules/_testimportmultiple.c b/Modules/_testimportmultiple.c index 245e81b2dce7f8f..7e6556ad400cdec 100644 --- a/Modules/_testimportmultiple.c +++ b/Modules/_testimportmultiple.c @@ -4,9 +4,7 @@ * foo, bar), only the first one is called the same as the compiled file. */ -#ifndef _MSC_VER #include "pyconfig.h" // Py_GIL_DISABLED -#endif #ifndef Py_GIL_DISABLED #define Py_LIMITED_API 0x03020000 diff --git a/Modules/_uuidmodule.c b/Modules/_uuidmodule.c index d8b211c632eef16..4b6852c0d0ec73d 100644 --- a/Modules/_uuidmodule.c +++ b/Modules/_uuidmodule.c @@ -3,9 +3,7 @@ * DCE compatible Universally Unique Identifier library. */ -#ifndef _MSC_VER #include "pyconfig.h" // Py_GIL_DISABLED -#endif #ifndef Py_GIL_DISABLED // Need limited C API version 3.12 for Py_MOD_PER_INTERPRETER_GIL_SUPPORTED diff --git a/Modules/errnomodule.c b/Modules/errnomodule.c index 8287edbfb47f6c8..1100e9f6094352b 100644 --- a/Modules/errnomodule.c +++ b/Modules/errnomodule.c @@ -1,8 +1,6 @@ /* Errno module */ -#ifndef _MSC_VER #include "pyconfig.h" // Py_GIL_DISABLED -#endif #ifndef Py_GIL_DISABLED // Need limited C API version 3.12 for Py_MOD_PER_INTERPRETER_GIL_SUPPORTED diff --git a/Modules/resource.c b/Modules/resource.c index a4b8f648c329e32..19020b8cc1b6db2 100644 --- a/Modules/resource.c +++ b/Modules/resource.c @@ -1,6 +1,4 @@ -#ifndef _MSC_VER #include "pyconfig.h" // Py_GIL_DISABLED -#endif #ifndef Py_GIL_DISABLED // Need limited C API version 3.13 for PySys_Audit() diff --git a/Modules/xxlimited.c b/Modules/xxlimited.c index 19f61216255cfa1..0bb5e12d7c3dd98 100644 --- a/Modules/xxlimited.c +++ b/Modules/xxlimited.c @@ -62,9 +62,7 @@ pass */ -#ifndef _MSC_VER #include "pyconfig.h" // Py_GIL_DISABLED -#endif #ifndef Py_GIL_DISABLED // Need limited C API version 3.12 for Py_MOD_PER_INTERPRETER_GIL_SUPPORTED diff --git a/Modules/xxlimited_35.c b/Modules/xxlimited_35.c index 867820a6cb93fae..754a368f77e9401 100644 --- a/Modules/xxlimited_35.c +++ b/Modules/xxlimited_35.c @@ -5,9 +5,7 @@ * See the xxlimited module for an extension module template. */ -#ifndef _MSC_VER #include "pyconfig.h" // Py_GIL_DISABLED -#endif #ifndef Py_GIL_DISABLED #define Py_LIMITED_API 0x03050000 diff --git a/PC/layout/main.py b/PC/layout/main.py index cb2e4878da26b10..accfd51dd978fb9 100644 --- a/PC/layout/main.py +++ b/PC/layout/main.py @@ -73,7 +73,10 @@ def copy_if_modified(src, dest): ) if do_copy: - shutil.copy2(src, dest) + try: + shutil.copy2(src, dest) + except FileNotFoundError: + raise FileNotFoundError(src) from None def get_lib_layout(ns): @@ -208,8 +211,7 @@ def _c(d): for dest, src in rglob(ns.source / "Include", "**/*.h"): yield "include/{}".format(dest), src - src = ns.source / "PC" / "pyconfig.h" - yield "include/pyconfig.h", src + yield "include/pyconfig.h", ns.build / "pyconfig.h" for dest, src in get_tcltk_lib(ns): yield dest, src diff --git a/PC/pyconfig.h b/PC/pyconfig.h.in similarity index 99% rename from PC/pyconfig.h rename to PC/pyconfig.h.in index e6b368caffe2801..d8f0a6be69c21a6 100644 --- a/PC/pyconfig.h +++ b/PC/pyconfig.h.in @@ -739,4 +739,7 @@ Py_NO_ENABLE_SHARED to find out. Also support MS_NO_COREDLL for b/w compat */ /* Define if libssl has X509_VERIFY_PARAM_set1_host and related function */ #define HAVE_X509_VERIFY_PARAM_SET1_HOST 1 +/* Define if you want to disable the GIL */ +#undef Py_GIL_DISABLED + #endif /* !Py_CONFIG_H */ diff --git a/PC/winsound.c b/PC/winsound.c index b0e416cfec46996..7e4ebd90f50c2e4 100644 --- a/PC/winsound.c +++ b/PC/winsound.c @@ -35,6 +35,8 @@ winsound.PlaySound(None, 0) */ +#include "pyconfig.h" // Py_GIL_DISABLED + #ifndef Py_GIL_DISABLED // Need limited C API version 3.12 for Py_MOD_PER_INTERPRETER_GIL_SUPPORTED #define Py_LIMITED_API 0x030c0000 diff --git a/PCbuild/_freeze_module.vcxproj b/PCbuild/_freeze_module.vcxproj index a1c37e183f21c7f..f8c5fafa561efa5 100644 --- a/PCbuild/_freeze_module.vcxproj +++ b/PCbuild/_freeze_module.vcxproj @@ -89,6 +89,7 @@ Py_NO_ENABLE_SHARED;Py_BUILD_CORE;_CONSOLE;%(PreprocessorDefinitions) + $(IntDir);%(AdditionalIncludeDirectories) Disabled false @@ -257,6 +258,9 @@ + + + @@ -414,6 +418,32 @@ + + + + + + + + + + + @(PyConfigH->'%(FullPath)', ';') + $([System.IO.File]::ReadAllText($(PyConfigH))) + $([System.IO.File]::ReadAllText('$(IntDir)pyconfig.h')) + + + $(PyConfigHText.Replace('#undef Py_GIL_DISABLED', '#define Py_GIL_DISABLED 1')) + + + + + diff --git a/PCbuild/pyproject.props b/PCbuild/pyproject.props index 0acc7045c39a269..68c0550f7603b7f 100644 --- a/PCbuild/pyproject.props +++ b/PCbuild/pyproject.props @@ -10,6 +10,8 @@ $(MSBuildThisFileDirectory)obj\ $(Py_IntDir)\$(MajorVersionNumber)$(MinorVersionNumber)$(ArchName)_$(Configuration)\$(ProjectName)\ $(IntDir.Replace(`\\`, `\`)) + + $(Py_IntDir)\$(MajorVersionNumber)$(MinorVersionNumber)$(ArchName)_$(Configuration)\pythoncore\ $(ProjectName) $(TargetName)$(PyDebugExt) false @@ -38,9 +40,8 @@ - $(PySourcePath)Include;$(PySourcePath)Include\internal;$(PySourcePath)Include\internal\mimalloc;$(PySourcePath)PC;$(IntDir);%(AdditionalIncludeDirectories) + $(PySourcePath)Include;$(PySourcePath)Include\internal;$(PySourcePath)Include\internal\mimalloc;$(GeneratedPyConfigDir);$(PySourcePath)PC;%(AdditionalIncludeDirectories) WIN32;$(_Py3NamePreprocessorDefinition);$(_PlatformPreprocessorDefinition)$(_DebugPreprocessorDefinition)$(_PydPreprocessorDefinition)%(PreprocessorDefinitions) - Py_GIL_DISABLED=1;%(PreprocessorDefinitions) _Py_USING_PGO=1;%(PreprocessorDefinitions) MaxSpeed @@ -60,6 +61,7 @@ -Wno-deprecated-non-prototype -Wno-unused-label -Wno-pointer-sign -Wno-incompatible-pointer-types-discards-qualifiers -Wno-unused-function %(AdditionalOptions) -flto %(AdditionalOptions) -d2pattern-opt-disable:-932189325 %(AdditionalOptions) + /sourceDependencies "$(IntDir.Trim(`\`))" %(AdditionalOptions) OnlyExplicitInline diff --git a/PCbuild/pythoncore.vcxproj b/PCbuild/pythoncore.vcxproj index 778fc834c0db9c1..90aa8cf28f8c5d5 100644 --- a/PCbuild/pythoncore.vcxproj +++ b/PCbuild/pythoncore.vcxproj @@ -378,7 +378,7 @@ - + @@ -646,6 +646,35 @@ + + + + + + + + + + @(PyConfigH->'%(FullPath)', ';') + $([System.IO.File]::ReadAllText($(PyConfigH))) + $([System.IO.File]::ReadAllText('$(IntDir)pyconfig.h')) + + + $(PyConfigHText.Replace('#undef Py_GIL_DISABLED', '#define Py_GIL_DISABLED 1')) + + + + + + + + + + + git diff --git a/Tools/msi/dev/dev_files.wxs b/Tools/msi/dev/dev_files.wxs index 21f9c848cc6be58..4357dc86d9d3564 100644 --- a/Tools/msi/dev/dev_files.wxs +++ b/Tools/msi/dev/dev_files.wxs @@ -3,7 +3,7 @@ - + diff --git a/Tools/peg_generator/pegen/build.py b/Tools/peg_generator/pegen/build.py index 30bfb31471c7b26..7df39a3b0ae48ed 100644 --- a/Tools/peg_generator/pegen/build.py +++ b/Tools/peg_generator/pegen/build.py @@ -143,6 +143,10 @@ def compile_c_extension( str(MOD_DIR.parent.parent.parent / "Parser" / "lexer"), str(MOD_DIR.parent.parent.parent / "Parser" / "tokenizer"), ] + if sys.platform == "win32": + # HACK: The location of pyconfig.h has moved within our build, and + # setuptools hasn't updated for it yet. So add the path manually for now + include_dirs.append(pathlib.Path(sysconfig.get_config_h_filename()).parent) extension = Extension( extension_name, sources=[generated_source_path],