From 895fc66320beee40e5bc2caf4ff585ce8a067a8b Mon Sep 17 00:00:00 2001 From: "Ralf W. Grosse-Kunstleve" Date: Thu, 20 Oct 2022 05:49:52 -0700 Subject: [PATCH 01/43] ci: update PGI build (old one no longer signed) (#4260) * Simply replace "22.3" with "22.9" to see what happens. * Remove PYBIND11_TEST_FILTER to see what happens. * Revert "Remove PYBIND11_TEST_FILTER to see what happens." This reverts commit 0cba2cef0c5c5bcbf57cad219c7d5dfb0cda241c. * Remove only test_smart_ptr.cpp to see what happens. * Revert "Remove only test_smart_ptr.cpp to see what happens." This reverts commit 8e9df22c85292003abcab50260393e547567fc18. * Remove only test_virtual_functions.cpp to see what happens. --- .github/workflows/ci.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index bc7175d829..2951a0ad19 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -391,7 +391,7 @@ jobs: # Testing on CentOS 7 + PGI compilers, which seems to require more workarounds centos-nvhpc7: runs-on: ubuntu-latest - name: "🐍 3 • CentOS7 / PGI 22.3 • x64" + name: "🐍 3 • CentOS7 / PGI 22.9 • x64" container: centos:7 steps: @@ -401,7 +401,7 @@ jobs: run: yum update -y && yum install -y epel-release && yum install -y git python3-devel make environment-modules cmake3 yum-utils - name: Install NVidia HPC SDK - run: yum-config-manager --add-repo https://developer.download.nvidia.com/hpc-sdk/rhel/nvhpc.repo && yum -y install nvhpc-22.3 + run: yum-config-manager --add-repo https://developer.download.nvidia.com/hpc-sdk/rhel/nvhpc.repo && yum -y install nvhpc-22.9 # On CentOS 7, we have to filter a few tests (compiler internal error) # and allow deeper template recursion (not needed on CentOS 8 with a newer @@ -411,12 +411,12 @@ jobs: shell: bash run: | source /etc/profile.d/modules.sh - module load /opt/nvidia/hpc_sdk/modulefiles/nvhpc/22.3 + module load /opt/nvidia/hpc_sdk/modulefiles/nvhpc/22.9 cmake3 -S . -B build -DDOWNLOAD_CATCH=ON \ -DCMAKE_CXX_STANDARD=11 \ -DPYTHON_EXECUTABLE=$(python3 -c "import sys; print(sys.executable)") \ -DCMAKE_CXX_FLAGS="-Wc,--pending_instantiations=0" \ - -DPYBIND11_TEST_FILTER="test_smart_ptr.cpp;test_virtual_functions.cpp" + -DPYBIND11_TEST_FILTER="test_smart_ptr.cpp" # Building before installing Pip should produce a warning but not an error - name: Build From 412918d12c12dca5acca813b37328bb280b68660 Mon Sep 17 00:00:00 2001 From: Henry Schreiner Date: Thu, 20 Oct 2022 10:35:18 -0400 Subject: [PATCH 02/43] feat: add entrypoint for cmake modules dir (#4258) Signed-off-by: Henry Schreiner Signed-off-by: Henry Schreiner --- setup.py | 2 ++ tests/extra_python_package/test_files.py | 2 ++ tools/setup_main.py.in | 3 +++ 3 files changed, 7 insertions(+) diff --git a/setup.py b/setup.py index 68573519c1..9097440bfe 100644 --- a/setup.py +++ b/setup.py @@ -144,6 +144,8 @@ def remove_output(*sources: str) -> Iterator[None]: stdout=sys.stdout, stderr=sys.stderr, ) + if not global_sdist: + Path("pybind11/share/cmake/pybind11/__init__.py").touch() txt = get_and_replace(setup_py, version=version, extra_cmd=extra_cmd) code = compile(txt, setup_py, "exec") diff --git a/tests/extra_python_package/test_files.py b/tests/extra_python_package/test_files.py index 8e1ddd8508..caf3206711 100644 --- a/tests/extra_python_package/test_files.py +++ b/tests/extra_python_package/test_files.py @@ -160,6 +160,7 @@ def test_build_sdist(monkeypatch, tmpdir): files |= {f"pybind11{n}" for n in local_sdist_files} files.add("pybind11.egg-info/entry_points.txt") files.add("pybind11.egg-info/requires.txt") + files.add("pybind11/share/cmake/pybind11/__init__.py") assert simpler == files with open(os.path.join(MAIN_DIR, "tools", "setup_main.py.in"), "rb") as f: @@ -246,6 +247,7 @@ def tests_build_wheel(monkeypatch, tmpdir): "dist-info/entry_points.txt", "dist-info/top_level.txt", } + files.add("pybind11/share/cmake/pybind11/__init__.py") with zipfile.ZipFile(str(wheel)) as z: names = z.namelist() diff --git a/tools/setup_main.py.in b/tools/setup_main.py.in index 65198bdb66..92ba04a957 100644 --- a/tools/setup_main.py.in +++ b/tools/setup_main.py.in @@ -36,6 +36,9 @@ setup( ], "pipx.run": [ "pybind11 = pybind11.__main__:main", + ], + "cmake.modules": [ + "pybind11 = pybind11.share.cmake.pybind11", ] }, cmdclass=cmdclass From a8f211073bb4020f351e3ca8e241a3b8f865be2d Mon Sep 17 00:00:00 2001 From: Henry Schreiner Date: Thu, 20 Oct 2022 10:58:04 -0400 Subject: [PATCH 03/43] docs: update changelog (#4265) Signed-off-by: Henry Schreiner Signed-off-by: Henry Schreiner --- docs/changelog.rst | 101 +++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 101 insertions(+) diff --git a/docs/changelog.rst b/docs/changelog.rst index 9c8ff423f4..1199ecdedf 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -16,6 +16,107 @@ IN DEVELOPMENT Changes will be summarized here periodically. +Version 2.10.1 (Oct 2?, 2022) +----------------------------- + + +Changes: + + +* Allow ``pybind11::capsule`` constructor to take null destructor pointers. + `#4221 `_ + +* ``embed.h`` was changed so that ``PYTHONPATH`` is used also with Python 3.11 + (established behavior). + `#4119 `_ + +Bug fixes: + +* Fix MSVC 2019 v.1924 & C++14 mode error for ``overload_cast``. + `#4188 `_ + +* Make augmented assignment operators non-const for the object-api. Behavior + was previously broken for augmented assignment operators. + `#4065 `_ + +* Add proper error checking to C++ bindings for Python list append and insert. + `#4208 `_ + +* Work-around for Nvidia's CUDA nvcc compiler in versions 11.4.0 - 11.8.0. + `#4220 `_ + +* A workaround for PyPy was added in the ``py::error_already_set`` + implementation, related to PR `#1895 `_ + released with v2.10.0. + `#4079 `_ + +* Fixed compiler errors when C++23 ``std::forward_like`` is available. + `#4136 `_ + +* Properly raise exceptions in contains methods (like when an object in unhashable). + `#4209 `_ + +* Further improve another error in exception handling. + `#4232 `_ + +* ``get_local_internals()`` was made compatible with + ``finalize_interpreter()``, fixing potential freezes during interpreter + finalization. + `#4192 `_ + + +Performance and style: + +* Reserve space in set and STL map casters if possible. This will prevent + unnecessary rehashing / resizing by knowing the number of keys ahead of time + for Python to C++ casting. This improvement will greatly speed up the casting + of large unordered maps and sets. + `#4194 `_ + +* GIL RAII scopes are non-copyable to avoid potential bugs. + `#4183 `_ + +* Explicitly default all relevant ctors for pytypes in the ``PYBIND11_OBJECT`` + macros and enforce the clang-tidy checks ``modernize-use-equals-default`` in + macros as well. + `#4017 `_ + +* Optimize iterator advancement in C++ bindings. + `#4237 `_ + +* Use the modern ``PyObject_GenericGetDict`` and ``PyObject_GenericSetDict`` + for handling dynamic attribute dictionaries. + `#4106 `_ + +* Document that users should use ``PYBIND11_NAMESPACE`` instead of using ``pybind11`` when + opening namespaces. Using namespace declarations and namespace qualification + remain the same as ``pybind11``. This is done to ensure consistent symbol + visibility. + `#4098 `_ + +* Mark ``detail::forward_like`` as constexpr. + `#4147 `_ + +* Optimize unpacking_collector when processing ``arg_v`` arguments. + `#4219 `_ + + +Build system improvements: + +* Experimental support for ``cmake.modules`` entrypoint. + `#4258 `_ + +* Include a pkg-config file when installing pybind11, such as in the Python + package. + `#4077 `_ + +* Avoid stripping debug symbols when ``CMAKE_BUILD_TYPE`` is set to ``DEBUG`` + instead of ``Debug``. + `#4078 `_ + +* Followup to `#3948 `_, fixing vcpkg again. + `#4123 `_ + Version 2.10.0 (Jul 15, 2022) ----------------------------- From 251516bce69f42007d403eebf526194b7e2a803b Mon Sep 17 00:00:00 2001 From: Aaron Gokaslan Date: Fri, 21 Oct 2022 12:51:26 -0400 Subject: [PATCH 04/43] Cleanup casters to release none() to avoid ref counting (#4269) --- include/pybind11/cast.h | 6 +++--- include/pybind11/functional.h | 2 +- include/pybind11/stl.h | 2 +- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/include/pybind11/cast.h b/include/pybind11/cast.h index b89e4946ad..5699d4fb00 100644 --- a/include/pybind11/cast.h +++ b/include/pybind11/cast.h @@ -248,7 +248,7 @@ struct void_caster { return false; } static handle cast(T, return_value_policy /* policy */, handle /* parent */) { - return none().inc_ref(); + return none().release(); } PYBIND11_TYPE_CASTER(T, const_name("None")); }; @@ -291,7 +291,7 @@ class type_caster : public type_caster { if (ptr) { return capsule(ptr).release(); } - return none().inc_ref(); + return none().release(); } template @@ -537,7 +537,7 @@ struct type_caster::value>> { static handle cast(const CharT *src, return_value_policy policy, handle parent) { if (src == nullptr) { - return pybind11::none().inc_ref(); + return pybind11::none().release(); } return StringCaster::cast(StringType(src), policy, parent); } diff --git a/include/pybind11/functional.h b/include/pybind11/functional.h index 4034990d89..102d1a938b 100644 --- a/include/pybind11/functional.h +++ b/include/pybind11/functional.h @@ -110,7 +110,7 @@ struct type_caster> { template static handle cast(Func &&f_, return_value_policy policy, handle /* parent */) { if (!f_) { - return none().inc_ref(); + return none().release(); } auto result = f_.template target(); diff --git a/include/pybind11/stl.h b/include/pybind11/stl.h index 8f243502ef..2d144b598b 100644 --- a/include/pybind11/stl.h +++ b/include/pybind11/stl.h @@ -311,7 +311,7 @@ struct optional_caster { template static handle cast(T &&src, return_value_policy policy, handle parent) { if (!src) { - return none().inc_ref(); + return none().release(); } if (!std::is_lvalue_reference::value) { policy = return_value_policy_override::policy(policy); From eaa5f7bd40b0339a78d97b448b78af158b20c15e Mon Sep 17 00:00:00 2001 From: Henry Schreiner Date: Fri, 21 Oct 2022 17:25:53 -0400 Subject: [PATCH 05/43] Revert "feat: add entrypoint for cmake modules dir" (#4270) * Revert "feat: add entrypoint for cmake modules dir (#4258)" This reverts commit 1d4a65e2f145a25b66e44e9d5edae9ec11abbc73. * docs: revert changelog mention too --- docs/changelog.rst | 3 --- setup.py | 2 -- tests/extra_python_package/test_files.py | 2 -- tools/setup_main.py.in | 3 --- 4 files changed, 10 deletions(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index 1199ecdedf..df3181c524 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -103,9 +103,6 @@ Performance and style: Build system improvements: -* Experimental support for ``cmake.modules`` entrypoint. - `#4258 `_ - * Include a pkg-config file when installing pybind11, such as in the Python package. `#4077 `_ diff --git a/setup.py b/setup.py index 9097440bfe..68573519c1 100644 --- a/setup.py +++ b/setup.py @@ -144,8 +144,6 @@ def remove_output(*sources: str) -> Iterator[None]: stdout=sys.stdout, stderr=sys.stderr, ) - if not global_sdist: - Path("pybind11/share/cmake/pybind11/__init__.py").touch() txt = get_and_replace(setup_py, version=version, extra_cmd=extra_cmd) code = compile(txt, setup_py, "exec") diff --git a/tests/extra_python_package/test_files.py b/tests/extra_python_package/test_files.py index caf3206711..8e1ddd8508 100644 --- a/tests/extra_python_package/test_files.py +++ b/tests/extra_python_package/test_files.py @@ -160,7 +160,6 @@ def test_build_sdist(monkeypatch, tmpdir): files |= {f"pybind11{n}" for n in local_sdist_files} files.add("pybind11.egg-info/entry_points.txt") files.add("pybind11.egg-info/requires.txt") - files.add("pybind11/share/cmake/pybind11/__init__.py") assert simpler == files with open(os.path.join(MAIN_DIR, "tools", "setup_main.py.in"), "rb") as f: @@ -247,7 +246,6 @@ def tests_build_wheel(monkeypatch, tmpdir): "dist-info/entry_points.txt", "dist-info/top_level.txt", } - files.add("pybind11/share/cmake/pybind11/__init__.py") with zipfile.ZipFile(str(wheel)) as z: names = z.namelist() diff --git a/tools/setup_main.py.in b/tools/setup_main.py.in index 92ba04a957..65198bdb66 100644 --- a/tools/setup_main.py.in +++ b/tools/setup_main.py.in @@ -36,9 +36,6 @@ setup( ], "pipx.run": [ "pybind11 = pybind11.__main__:main", - ], - "cmake.modules": [ - "pybind11 = pybind11.share.cmake.pybind11", ] }, cmdclass=cmdclass From 305c471131a385e067738f589dde0f8f1da5f683 Mon Sep 17 00:00:00 2001 From: Aaron Gokaslan Date: Fri, 21 Oct 2022 18:04:01 -0400 Subject: [PATCH 06/43] fix: Revert pfect args make iterator (#4234) * Revert "chore: perfectly forward all make_iterator args (#3980)" This reverts commit 8da58da53996e9aa40d051b3b84cb3220fdbbb58. * Redo unrelated optimization in commit * Add tests for ambiguous overloads --- include/pybind11/pybind11.h | 22 ++++++++-------------- tests/test_sequences_and_iterators.cpp | 19 +++++++++++++++++++ tests/test_sequences_and_iterators.py | 8 ++++++++ 3 files changed, 35 insertions(+), 14 deletions(-) diff --git a/include/pybind11/pybind11.h b/include/pybind11/pybind11.h index fb4b7578d9..e662236d01 100644 --- a/include/pybind11/pybind11.h +++ b/include/pybind11/pybind11.h @@ -2333,7 +2333,7 @@ template -iterator make_iterator_impl(Iterator &&first, Sentinel &&last, Extra &&...extra) { +iterator make_iterator_impl(Iterator first, Sentinel last, Extra &&...extra) { using state = detail::iterator_state; // TODO: state captures only the types of Extra, not the values @@ -2359,7 +2359,7 @@ iterator make_iterator_impl(Iterator &&first, Sentinel &&last, Extra &&...extra) Policy); } - return cast(state{std::forward(first), std::forward(last), true}); + return cast(state{first, last, true}); } PYBIND11_NAMESPACE_END(detail) @@ -2370,15 +2370,13 @@ template ::result_type, typename... Extra> -iterator make_iterator(Iterator &&first, Sentinel &&last, Extra &&...extra) { +iterator make_iterator(Iterator first, Sentinel last, Extra &&...extra) { return detail::make_iterator_impl, Policy, Iterator, Sentinel, ValueType, - Extra...>(std::forward(first), - std::forward(last), - std::forward(extra)...); + Extra...>(first, last, std::forward(extra)...); } /// Makes a python iterator over the keys (`.first`) of a iterator over pairs from a @@ -2388,15 +2386,13 @@ template ::result_type, typename... Extra> -iterator make_key_iterator(Iterator &&first, Sentinel &&last, Extra &&...extra) { +iterator make_key_iterator(Iterator first, Sentinel last, Extra &&...extra) { return detail::make_iterator_impl, Policy, Iterator, Sentinel, KeyType, - Extra...>(std::forward(first), - std::forward(last), - std::forward(extra)...); + Extra...>(first, last, std::forward(extra)...); } /// Makes a python iterator over the values (`.second`) of a iterator over pairs from a @@ -2406,15 +2402,13 @@ template ::result_type, typename... Extra> -iterator make_value_iterator(Iterator &&first, Sentinel &&last, Extra &&...extra) { +iterator make_value_iterator(Iterator first, Sentinel last, Extra &&...extra) { return detail::make_iterator_impl, Policy, Iterator, Sentinel, ValueType, - Extra...>(std::forward(first), - std::forward(last), - std::forward(extra)...); + Extra...>(first, last, std::forward(extra)...); } /// Makes an iterator over values of an stl container or other container supporting diff --git a/tests/test_sequences_and_iterators.cpp b/tests/test_sequences_and_iterators.cpp index b867f49a25..1de65edbf2 100644 --- a/tests/test_sequences_and_iterators.cpp +++ b/tests/test_sequences_and_iterators.cpp @@ -559,4 +559,23 @@ TEST_SUBMODULE(sequences_and_iterators, m) { []() { return py::make_iterator(list); }); m.def("make_iterator_2", []() { return py::make_iterator(list); }); + + // test_iterator on c arrays + // #4100: ensure lvalue required as increment operand + class CArrayHolder { + public: + CArrayHolder(double x, double y, double z) { + values[0] = x; + values[1] = y; + values[2] = z; + }; + double values[3]; + }; + + py::class_(m, "CArrayHolder") + .def(py::init()) + .def( + "__iter__", + [](const CArrayHolder &v) { return py::make_iterator(v.values, v.values + 3); }, + py::keep_alive<0, 1>()); } diff --git a/tests/test_sequences_and_iterators.py b/tests/test_sequences_and_iterators.py index 062e3b3d30..de486e3e8e 100644 --- a/tests/test_sequences_and_iterators.py +++ b/tests/test_sequences_and_iterators.py @@ -241,3 +241,11 @@ def test_iterator_rvp(): assert list(m.make_iterator_1()) == [1, 2, 3] assert list(m.make_iterator_2()) == [1, 2, 3] assert not isinstance(m.make_iterator_1(), type(m.make_iterator_2())) + + +def test_carray_iterator(): + """#4100: Check for proper iterator overload with C-Arrays""" + args_gt = list(float(i) for i in range(3)) + arr_h = m.CArrayHolder(*args_gt) + args = list(arr_h) + assert args_gt == args From ce63bcb9808d6bb1e7e6dddb4fde081b10d24877 Mon Sep 17 00:00:00 2001 From: Lalaland Date: Sat, 22 Oct 2022 16:52:35 -0700 Subject: [PATCH 07/43] Fix casts to void* (#4275) * Fix casts to void* * Improve tests * style: pre-commit fixes * remove c style cast Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> Co-authored-by: Aaron Gokaslan --- include/pybind11/cast.h | 6 ++---- tests/test_class.cpp | 18 +++++++++++++++++- tests/test_class.py | 2 ++ 3 files changed, 21 insertions(+), 5 deletions(-) diff --git a/include/pybind11/cast.h b/include/pybind11/cast.h index 5699d4fb00..430c62f357 100644 --- a/include/pybind11/cast.h +++ b/include/pybind11/cast.h @@ -1179,11 +1179,9 @@ enable_if_t::value, T> cast_safe(object &&) pybind11_fail("Internal error: cast_safe fallback invoked"); } template -enable_if_t>::value, void> cast_safe(object &&) {} +enable_if_t::value, void> cast_safe(object &&) {} template -enable_if_t, - std::is_same>>::value, - T> +enable_if_t, std::is_void>::value, T> cast_safe(object &&o) { return pybind11::cast(std::move(o)); } diff --git a/tests/test_class.cpp b/tests/test_class.cpp index 18c8d358bb..fc1c17b179 100644 --- a/tests/test_class.cpp +++ b/tests/test_class.cpp @@ -384,6 +384,8 @@ TEST_SUBMODULE(class_, m) { protected: virtual int foo() const { return value; } + virtual void *void_foo() { return static_cast(&value); } + virtual void *get_self() { return static_cast(this); } private: int value = 42; @@ -392,6 +394,8 @@ TEST_SUBMODULE(class_, m) { class TrampolineB : public ProtectedB { public: int foo() const override { PYBIND11_OVERRIDE(int, ProtectedB, foo, ); } + void *void_foo() override { PYBIND11_OVERRIDE(void *, ProtectedB, void_foo, ); } + void *get_self() override { PYBIND11_OVERRIDE(void *, ProtectedB, get_self, ); } }; class PublicistB : public ProtectedB { @@ -401,11 +405,23 @@ TEST_SUBMODULE(class_, m) { // (in Debug builds only, tested with icpc (ICC) 2021.1 Beta 20200827) ~PublicistB() override{}; // NOLINT(modernize-use-equals-default) using ProtectedB::foo; + using ProtectedB::get_self; + using ProtectedB::void_foo; }; + m.def("read_foo", [](const void *original) { + const int *ptr = reinterpret_cast(original); + return *ptr; + }); + + m.def("pointers_equal", + [](const void *original, const void *comparison) { return original == comparison; }); + py::class_(m, "ProtectedB") .def(py::init<>()) - .def("foo", &PublicistB::foo); + .def("foo", &PublicistB::foo) + .def("void_foo", &PublicistB::void_foo) + .def("get_self", &PublicistB::get_self); // test_brace_initialization struct BraceInitialization { diff --git a/tests/test_class.py b/tests/test_class.py index 47ba450fc2..7c1ed20608 100644 --- a/tests/test_class.py +++ b/tests/test_class.py @@ -313,6 +313,8 @@ def test_bind_protected_functions(): b = m.ProtectedB() assert b.foo() == 42 + assert m.read_foo(b.void_foo()) == 42 + assert m.pointers_equal(b.get_self(), b) class C(m.ProtectedB): def __init__(self): From 0e82c360942907f5a2f379e64e0d211aaff80774 Mon Sep 17 00:00:00 2001 From: Henry Schreiner Date: Sun, 23 Oct 2022 00:32:17 -0400 Subject: [PATCH 08/43] fix: add flag for overriding classic Python search values (#4195) * fix: PyPy needs to overrite broken FindPythonInterp values Signed-off-by: Henry Schreiner * fix: add flag to opt-in to new (cross-compile) behavior Signed-off-by: Henry Schreiner * Apply suggestions from code review Signed-off-by: Henry Schreiner --- tools/FindPythonLibsNew.cmake | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/tools/FindPythonLibsNew.cmake b/tools/FindPythonLibsNew.cmake index a5a628fd7a..bbbd9f9ec1 100644 --- a/tools/FindPythonLibsNew.cmake +++ b/tools/FindPythonLibsNew.cmake @@ -151,9 +151,13 @@ if(NOT _PYTHON_SUCCESS MATCHES 0) return() endif() +option( + PYBIND11_PYTHONLIBS_OVERWRITE + "Overwrite cached values read from Python library (classic search). Turn off if cross-compiling and manually setting these values." + ON) # Can manually set values when cross-compiling macro(_PYBIND11_GET_IF_UNDEF lst index name) - if(NOT DEFINED "${name}") + if(PYBIND11_PYTHONLIBS_OVERWRITE OR NOT DEFINED "${name}") list(GET "${lst}" "${index}" "${name}") endif() endmacro() From 1f187d9a35642f04bfa2e95aa16a81bb2d2220fe Mon Sep 17 00:00:00 2001 From: Vemund Handeland Date: Sun, 23 Oct 2022 20:57:45 +0200 Subject: [PATCH 09/43] Fix char8_t support (#4278) Standard library macro __cpp_lib_char8_t is only available after including standard header --- include/pybind11/detail/common.h | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/include/pybind11/detail/common.h b/include/pybind11/detail/common.h index 6da7ef859c..b43100b95a 100644 --- a/include/pybind11/detail/common.h +++ b/include/pybind11/detail/common.h @@ -205,10 +205,6 @@ # endif #endif -#if defined(__cpp_lib_char8_t) && __cpp_lib_char8_t >= 201811L -# define PYBIND11_HAS_U8STRING -#endif - #include #if PY_VERSION_HEX < 0x03060000 # error "PYTHON < 3.6 IS UNSUPPORTED. pybind11 v2.9 was the last to support Python 2 and 3.5." @@ -259,6 +255,11 @@ # endif #endif +// Must be after including or one of the other headers specified by the standard +#if defined(__cpp_lib_char8_t) && __cpp_lib_char8_t >= 201811L +# define PYBIND11_HAS_U8STRING +#endif + // #define PYBIND11_STR_LEGACY_PERMISSIVE // If DEFINED, pybind11::str can hold PyUnicodeObject or PyBytesObject // (probably surprising and never documented, but this was the From 738a6f83432e45c09bdd6b284c42f442cbec8bdc Mon Sep 17 00:00:00 2001 From: Henry Schreiner Date: Wed, 26 Oct 2022 10:41:51 -0400 Subject: [PATCH 10/43] ci: move to final release of 3.11 (#4286) Signed-off-by: Henry Schreiner Signed-off-by: Henry Schreiner --- .github/workflows/ci.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 2951a0ad19..b56e8f447f 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -30,7 +30,7 @@ jobs: - '3.6' - '3.9' - '3.10' - - '3.11-dev' + - '3.11' - 'pypy-3.7' - 'pypy-3.8' - 'pypy-3.9' @@ -119,7 +119,7 @@ jobs: - name: C++11 tests # TODO: Figure out how to load the DLL on Python 3.8+ - if: "!(runner.os == 'Windows' && (matrix.python == 3.8 || matrix.python == 3.9 || matrix.python == '3.10' || matrix.python == '3.11-dev' || matrix.python == 'pypy-3.8'))" + if: "!(runner.os == 'Windows' && (matrix.python == 3.8 || matrix.python == 3.9 || matrix.python == '3.10' || matrix.python == '3.11' || matrix.python == 'pypy-3.8'))" run: cmake --build . --target cpptest -j 2 - name: Interface test C++11 @@ -146,7 +146,7 @@ jobs: - name: C++ tests # TODO: Figure out how to load the DLL on Python 3.8+ - if: "!(runner.os == 'Windows' && (matrix.python == 3.8 || matrix.python == 3.9 || matrix.python == '3.10' || matrix.python == '3.11-dev' || matrix.python == 'pypy-3.8'))" + if: "!(runner.os == 'Windows' && (matrix.python == 3.8 || matrix.python == 3.9 || matrix.python == '3.10' || matrix.python == '3.11' || matrix.python == 'pypy-3.8'))" run: cmake --build build2 --target cpptest # Third build - C++17 mode with unstable ABI @@ -186,7 +186,7 @@ jobs: - python-version: "3.9" python-debug: true valgrind: true - - python-version: "3.11-dev" + - python-version: "3.11" python-debug: false name: "🐍 ${{ matrix.python-version }}${{ matrix.python-debug && '-dbg' || '' }} (deadsnakes)${{ matrix.valgrind && ' • Valgrind' || '' }} • x64" From 5b395c9b44d9bac2082bd71a1d2ed3cbb4af35db Mon Sep 17 00:00:00 2001 From: Aaron Gokaslan Date: Sat, 29 Oct 2022 11:12:24 -0400 Subject: [PATCH 11/43] fix: improve bytes to str decoding error handling (#4294) * (bugfix): Improve bytes to str decoding error handling * regroup test * Further broaden tests * Add another decode error test * Fix bug in tests * Reviewer suggestions --- include/pybind11/pytypes.h | 9 +++++++++ tests/test_pytypes.cpp | 5 +++++ tests/test_pytypes.py | 14 ++++++++++++++ 3 files changed, 28 insertions(+) diff --git a/include/pybind11/pytypes.h b/include/pybind11/pytypes.h index 37fc49a0e1..80a2e2228e 100644 --- a/include/pybind11/pytypes.h +++ b/include/pybind11/pytypes.h @@ -1432,6 +1432,9 @@ class str : public object { str(const char *c, const SzType &n) : object(PyUnicode_FromStringAndSize(c, ssize_t_cast(n)), stolen_t{}) { if (!m_ptr) { + if (PyErr_Occurred()) { + throw error_already_set(); + } pybind11_fail("Could not allocate string object!"); } } @@ -1441,6 +1444,9 @@ class str : public object { // NOLINTNEXTLINE(google-explicit-constructor) str(const char *c = "") : object(PyUnicode_FromString(c), stolen_t{}) { if (!m_ptr) { + if (PyErr_Occurred()) { + throw error_already_set(); + } pybind11_fail("Could not allocate string object!"); } } @@ -1598,6 +1604,9 @@ inline str::str(const bytes &b) { } auto obj = reinterpret_steal(PyUnicode_FromStringAndSize(buffer, length)); if (!obj) { + if (PyErr_Occurred()) { + throw error_already_set(); + } pybind11_fail("Could not allocate string object!"); } m_ptr = obj.release().ptr(); diff --git a/tests/test_pytypes.cpp b/tests/test_pytypes.cpp index 99237ccef7..ea8d03958b 100644 --- a/tests/test_pytypes.cpp +++ b/tests/test_pytypes.cpp @@ -206,7 +206,12 @@ TEST_SUBMODULE(pytypes, m) { m.def("str_from_char_ssize_t", []() { return py::str{"red", (py::ssize_t) 3}; }); m.def("str_from_char_size_t", []() { return py::str{"blue", (py::size_t) 4}; }); m.def("str_from_string", []() { return py::str(std::string("baz")); }); + m.def("str_from_std_string_input", [](const std::string &stri) { return py::str(stri); }); + m.def("str_from_cstr_input", [](const char *c_str) { return py::str(c_str); }); m.def("str_from_bytes", []() { return py::str(py::bytes("boo", 3)); }); + m.def("str_from_bytes_input", + [](const py::bytes &encoded_str) { return py::str(encoded_str); }); + m.def("str_from_object", [](const py::object &obj) { return py::str(obj); }); m.def("repr_from_object", [](const py::object &obj) { return py::repr(obj); }); m.def("str_from_handle", [](py::handle h) { return py::str(h); }); diff --git a/tests/test_pytypes.py b/tests/test_pytypes.py index 3ed7b9c946..079ee7ca50 100644 --- a/tests/test_pytypes.py +++ b/tests/test_pytypes.py @@ -244,6 +244,20 @@ def __repr__(self): m.str_from_string_from_str(ucs_surrogates_str) +@pytest.mark.parametrize( + "func", + [ + m.str_from_bytes_input, + m.str_from_cstr_input, + m.str_from_std_string_input, + ], +) +def test_surrogate_pairs_unicode_error(func): + input_str = "\ud83d\ude4f".encode("utf-8", "surrogatepass") + with pytest.raises(UnicodeDecodeError): + func(input_str) + + def test_bytes(doc): assert m.bytes_from_char_ssize_t().decode() == "green" assert m.bytes_from_char_size_t().decode() == "purple" From 15fde1def2a8b620aea5fd557b71adb29e73b3d6 Mon Sep 17 00:00:00 2001 From: Chekov2k Date: Sun, 30 Oct 2022 15:57:23 +0000 Subject: [PATCH 12/43] Add `PYBIND11_SIMPLE_GIL_MANAGEMENT` option (cmake, C++ define) (#4216) * Add option to force the use of the PYPY GIL scoped acquire/release logic to support nested gil access, see https://github.com/pybind/pybind11/issues/1276 and https://github.com/pytorch/pytorch/issues/83101 * Apply suggestions from code review * Update CMakeLists.txt * docs: update upgrade guide * Update docs/upgrade.rst * All bells & whistles. * Add Reminder to common.h, so that we will not forget to purge `!WITH_THREAD` branches when dropping Python 3.6 * New sentence instead of semicolon. * Temporarily pull in snapshot of PR #4246 * Add `test_release_acquire` * Add more unit tests for nested gil locking * Add test_report_builtins_internals_keys * Very minor enhancement: sort list only after filtering. * Revert change in docs/upgrade.rst * Add test_multi_acquire_release_cross_module, while also forcing unique PYBIND11_INTERNALS_VERSION for cross_module_gil_utils.cpp * Hopefully fix apparently new ICC error. ``` 2022-10-28T07:57:54.5187728Z -- The CXX compiler identification is Intel 2021.7.0.20220726 ... 2022-10-28T07:58:53.6758994Z icpc: remark #10441: The Intel(R) C++ Compiler Classic (ICC) is deprecated and will be removed from product release in the second half of 2023. The Intel(R) oneAPI DPC++/C++ Compiler (ICX) is the recommended compiler moving forward. Please transition to use this compiler. Use '-diag-disable=10441' to disable this message. 2022-10-28T07:58:54.5801597Z In file included from /home/runner/work/pybind11/pybind11/include/pybind11/detail/../detail/type_caster_base.h(15), 2022-10-28T07:58:54.5803794Z from /home/runner/work/pybind11/pybind11/include/pybind11/detail/../cast.h(15), 2022-10-28T07:58:54.5805740Z from /home/runner/work/pybind11/pybind11/include/pybind11/detail/../attr.h(14), 2022-10-28T07:58:54.5809556Z from /home/runner/work/pybind11/pybind11/include/pybind11/detail/class.h(12), 2022-10-28T07:58:54.5812154Z from /home/runner/work/pybind11/pybind11/include/pybind11/pybind11.h(13), 2022-10-28T07:58:54.5948523Z from /home/runner/work/pybind11/pybind11/tests/cross_module_gil_utils.cpp(13): 2022-10-28T07:58:54.5949009Z /home/runner/work/pybind11/pybind11/include/pybind11/detail/../detail/internals.h(177): error #2282: unrecognized GCC pragma 2022-10-28T07:58:54.5949374Z PYBIND11_TLS_KEY_INIT(tstate) 2022-10-28T07:58:54.5949579Z ^ 2022-10-28T07:58:54.5949695Z ``` * clang-tidy fixes * Workaround for PYPY WIN exitcode None * Revert "Temporarily pull in snapshot of PR #4246" This reverts commit 23ac16e859150f27fda25ca865cabcb4444e0770. * Another workaround for PYPY WIN exitcode None * Clean up how the tests are run "run in process" Part 1: uniformity * Clean up how the tests are run "run in process" Part 2: use `@pytest.mark.parametrize` and clean up the naming. * Skip some tests `#if defined(THREAD_SANITIZER)` (tested with TSAN using the Google-internal toolchain). * Run all tests again but ignore ThreadSanitizer exitcode 66 (this is less likely to mask unrelated ThreadSanitizer issues in the future). * bug fix: missing common.h include before using `PYBIND11_SIMPLE_GIL_MANAGEMENT` For the tests in the github CI this does not matter, because `PYBIND11_SIMPLE_GIL_MANAGEMENT` is always defined from the command line, but when monkey-patching common.h locally, it matters. * if process.exitcode is None: assert t_delta > 9.9 * More sophisiticated `_run_in_process()` implementation, clearly reporting `DEADLOCK`, additionally exercised via added `intentional_deadlock()` * Wrap m.intentional_deadlock in a Python function, for `ForkingPickler` compatibility. ``` > ForkingPickler(file, protocol).dump(obj) E TypeError: cannot pickle 'PyCapsule' object ``` Observed with all Windows builds including mingw but not PyPy, and macos-latest with Python 3.9, 3.10, 3.11 but not 3.6. * Add link to potential solution for WOULD-BE-NICE-TO-HAVE feature. * Add `SKIP_IF_DEADLOCK = True` option, to not pollute the CI results with expected `DEADLOCK` failures while we figure out what to do about them. * Add COPY-PASTE-THIS: gdb ... command (to be used for debugging the detected deadlock) * style: pre-commit fixes * Do better than automatic pre-commit fixes. * Add `PYBIND11_SIMPLE_GIL_MANAGEMENT` to `pytest_report_header()` (so that we can easily know when harvesting deadlock information from the CI logs). Co-authored-by: Arnim Balzer Co-authored-by: Henry Schreiner Co-authored-by: Ralf W. Grosse-Kunstleve Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> --- .github/workflows/ci.yml | 4 + CMakeLists.txt | 6 + include/pybind11/detail/common.h | 5 + include/pybind11/detail/internals.h | 16 +- include/pybind11/gil.h | 53 ++++-- tests/conftest.py | 1 + tests/cross_module_gil_utils.cpp | 67 +++++++- tests/pybind11_tests.cpp | 6 + tests/test_embed/test_interpreter.cpp | 1 - tests/test_gil_scoped.cpp | 107 +++++++++++- tests/test_gil_scoped.py | 227 +++++++++++++++++++++----- 11 files changed, 433 insertions(+), 60 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index b56e8f447f..a11cae1ab0 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -102,10 +102,12 @@ jobs: run: python -m pip install pytest-github-actions-annotate-failures # First build - C++11 mode and inplace + # More-or-less randomly adding -DPYBIND11_SIMPLE_GIL_MANAGEMENT=ON here. - name: Configure C++11 ${{ matrix.args }} run: > cmake -S . -B . -DPYBIND11_WERROR=ON + -DPYBIND11_SIMPLE_GIL_MANAGEMENT=ON -DDOWNLOAD_CATCH=ON -DDOWNLOAD_EIGEN=ON -DCMAKE_CXX_STANDARD=11 @@ -129,10 +131,12 @@ jobs: run: git clean -fdx # Second build - C++17 mode and in a build directory + # More-or-less randomly adding -DPYBIND11_SIMPLE_GIL_MANAGEMENT=OFF here. - name: Configure C++17 run: > cmake -S . -B build2 -DPYBIND11_WERROR=ON + -DPYBIND11_SIMPLE_GIL_MANAGEMENT=OFF -DDOWNLOAD_CATCH=ON -DDOWNLOAD_EIGEN=ON -DCMAKE_CXX_STANDARD=17 diff --git a/CMakeLists.txt b/CMakeLists.txt index ee0975bc1f..3be6f7a49b 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -91,10 +91,16 @@ endif() option(PYBIND11_INSTALL "Install pybind11 header files?" ${PYBIND11_MASTER_PROJECT}) option(PYBIND11_TEST "Build pybind11 test suite?" ${PYBIND11_MASTER_PROJECT}) option(PYBIND11_NOPYTHON "Disable search for Python" OFF) +option(PYBIND11_SIMPLE_GIL_MANAGEMENT + "Use simpler GIL management logic that does not support disassociation" OFF) set(PYBIND11_INTERNALS_VERSION "" CACHE STRING "Override the ABI version, may be used to enable the unstable ABI.") +if(PYBIND11_SIMPLE_GIL_MANAGEMENT) + add_compile_definitions(PYBIND11_SIMPLE_GIL_MANAGEMENT) +endif() + cmake_dependent_option( USE_PYTHON_INCLUDE_DIR "Install pybind11 headers in Python include directory instead of default installation prefix" diff --git a/include/pybind11/detail/common.h b/include/pybind11/detail/common.h index b43100b95a..a3e0bc9b37 100644 --- a/include/pybind11/detail/common.h +++ b/include/pybind11/detail/common.h @@ -206,6 +206,7 @@ #endif #include +// Reminder: WITH_THREAD is always defined if PY_VERSION_HEX >= 0x03070000 #if PY_VERSION_HEX < 0x03060000 # error "PYTHON < 3.6 IS UNSUPPORTED. pybind11 v2.9 was the last to support Python 2 and 3.5." #endif @@ -229,6 +230,10 @@ # undef copysign #endif +#if defined(PYPY_VERSION) && !defined(PYBIND11_SIMPLE_GIL_MANAGEMENT) +# define PYBIND11_SIMPLE_GIL_MANAGEMENT +#endif + #if defined(_MSC_VER) # if defined(PYBIND11_DEBUG_MARKER) # define _DEBUG diff --git a/include/pybind11/detail/internals.h b/include/pybind11/detail/internals.h index d47084e263..7de7794344 100644 --- a/include/pybind11/detail/internals.h +++ b/include/pybind11/detail/internals.h @@ -9,6 +9,12 @@ #pragma once +#include "common.h" + +#if defined(WITH_THREAD) && defined(PYBIND11_SIMPLE_GIL_MANAGEMENT) +# include "../gil.h" +#endif + #include "../pytypes.h" #include @@ -49,7 +55,7 @@ inline PyObject *make_object_base_type(PyTypeObject *metaclass); // `Py_LIMITED_API` anyway. # if PYBIND11_INTERNALS_VERSION > 4 # define PYBIND11_TLS_KEY_REF Py_tss_t & -# ifdef __GNUC__ +# if defined(__GNUC__) && !defined(__INTEL_COMPILER) // Clang on macOS warns due to `Py_tss_NEEDS_INIT` not specifying an initializer // for every field. # define PYBIND11_TLS_KEY_INIT(var) \ @@ -169,10 +175,12 @@ struct internals { PyTypeObject *default_metaclass; PyObject *instance_base; #if defined(WITH_THREAD) + // Unused if PYBIND11_SIMPLE_GIL_MANAGEMENT is defined: PYBIND11_TLS_KEY_INIT(tstate) # if PYBIND11_INTERNALS_VERSION > 4 PYBIND11_TLS_KEY_INIT(loader_life_support_tls_key) # endif // PYBIND11_INTERNALS_VERSION > 4 + // Unused if PYBIND11_SIMPLE_GIL_MANAGEMENT is defined: PyInterpreterState *istate = nullptr; ~internals() { # if PYBIND11_INTERNALS_VERSION > 4 @@ -408,6 +416,10 @@ PYBIND11_NOINLINE internals &get_internals() { return **internals_pp; } +#if defined(WITH_THREAD) +# if defined(PYBIND11_SIMPLE_GIL_MANAGEMENT) + gil_scoped_acquire gil; +# else // Ensure that the GIL is held since we will need to make Python calls. // Cannot use py::gil_scoped_acquire here since that constructor calls get_internals. struct gil_scoped_acquire_local { @@ -417,6 +429,8 @@ PYBIND11_NOINLINE internals &get_internals() { ~gil_scoped_acquire_local() { PyGILState_Release(state); } const PyGILState_STATE state; } gil; +# endif +#endif error_scope err_scope; PYBIND11_STR_TYPE id(PYBIND11_INTERNALS_ID); diff --git a/include/pybind11/gil.h b/include/pybind11/gil.h index 1ef5f0a8c8..cb0028d505 100644 --- a/include/pybind11/gil.h +++ b/include/pybind11/gil.h @@ -10,7 +10,10 @@ #pragma once #include "detail/common.h" -#include "detail/internals.h" + +#if defined(WITH_THREAD) && !defined(PYBIND11_SIMPLE_GIL_MANAGEMENT) +# include "detail/internals.h" +#endif PYBIND11_NAMESPACE_BEGIN(PYBIND11_NAMESPACE) @@ -21,7 +24,9 @@ PyThreadState *get_thread_state_unchecked(); PYBIND11_NAMESPACE_END(detail) -#if defined(WITH_THREAD) && !defined(PYPY_VERSION) +#if defined(WITH_THREAD) + +# if !defined(PYBIND11_SIMPLE_GIL_MANAGEMENT) /* The functions below essentially reproduce the PyGILState_* API using a RAII * pattern, but there are a few important differences: @@ -62,11 +67,11 @@ class gil_scoped_acquire { if (!tstate) { tstate = PyThreadState_New(internals.istate); -# if defined(PYBIND11_DETAILED_ERROR_MESSAGES) +# if defined(PYBIND11_DETAILED_ERROR_MESSAGES) if (!tstate) { pybind11_fail("scoped_acquire: could not create thread state!"); } -# endif +# endif tstate->gilstate_counter = 0; PYBIND11_TLS_REPLACE_VALUE(internals.tstate, tstate); } else { @@ -87,20 +92,20 @@ class gil_scoped_acquire { PYBIND11_NOINLINE void dec_ref() { --tstate->gilstate_counter; -# if defined(PYBIND11_DETAILED_ERROR_MESSAGES) +# if defined(PYBIND11_DETAILED_ERROR_MESSAGES) if (detail::get_thread_state_unchecked() != tstate) { pybind11_fail("scoped_acquire::dec_ref(): thread state must be current!"); } if (tstate->gilstate_counter < 0) { pybind11_fail("scoped_acquire::dec_ref(): reference count underflow!"); } -# endif +# endif if (tstate->gilstate_counter == 0) { -# if defined(PYBIND11_DETAILED_ERROR_MESSAGES) +# if defined(PYBIND11_DETAILED_ERROR_MESSAGES) if (!release) { pybind11_fail("scoped_acquire::dec_ref(): internal error!"); } -# endif +# endif PyThreadState_Clear(tstate); if (active) { PyThreadState_DeleteCurrent(); @@ -178,12 +183,14 @@ class gil_scoped_release { bool disassoc; bool active = true; }; -#elif defined(PYPY_VERSION) + +# else // PYBIND11_SIMPLE_GIL_MANAGEMENT + class gil_scoped_acquire { PyGILState_STATE state; public: - gil_scoped_acquire() { state = PyGILState_Ensure(); } + gil_scoped_acquire() : state{PyGILState_Ensure()} {} gil_scoped_acquire(const gil_scoped_acquire &) = delete; gil_scoped_acquire &operator=(const gil_scoped_acquire &) = delete; ~gil_scoped_acquire() { PyGILState_Release(state); } @@ -194,19 +201,39 @@ class gil_scoped_release { PyThreadState *state; public: - gil_scoped_release() { state = PyEval_SaveThread(); } + gil_scoped_release() : state{PyEval_SaveThread()} {} gil_scoped_release(const gil_scoped_release &) = delete; gil_scoped_release &operator=(const gil_scoped_acquire &) = delete; ~gil_scoped_release() { PyEval_RestoreThread(state); } void disarm() {} }; -#else + +# endif // PYBIND11_SIMPLE_GIL_MANAGEMENT + +#else // WITH_THREAD + class gil_scoped_acquire { +public: + gil_scoped_acquire() { + // Trick to suppress `unused variable` error messages (at call sites). + (void) (this != (this + 1)); + } + gil_scoped_acquire(const gil_scoped_acquire &) = delete; + gil_scoped_acquire &operator=(const gil_scoped_acquire &) = delete; void disarm() {} }; + class gil_scoped_release { +public: + gil_scoped_release() { + // Trick to suppress `unused variable` error messages (at call sites). + (void) (this != (this + 1)); + } + gil_scoped_release(const gil_scoped_release &) = delete; + gil_scoped_release &operator=(const gil_scoped_acquire &) = delete; void disarm() {} }; -#endif + +#endif // WITH_THREAD PYBIND11_NAMESPACE_END(PYBIND11_NAMESPACE) diff --git a/tests/conftest.py b/tests/conftest.py index 02ce263afc..f5ddb9f129 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -210,4 +210,5 @@ def pytest_report_header(config): f" {pybind11_tests.compiler_info}" f" {pybind11_tests.cpp_std}" f" {pybind11_tests.PYBIND11_INTERNALS_ID}" + f" PYBIND11_SIMPLE_GIL_MANAGEMENT={pybind11_tests.PYBIND11_SIMPLE_GIL_MANAGEMENT}" ) diff --git a/tests/cross_module_gil_utils.cpp b/tests/cross_module_gil_utils.cpp index 1436c35d6e..7c20849dd9 100644 --- a/tests/cross_module_gil_utils.cpp +++ b/tests/cross_module_gil_utils.cpp @@ -6,9 +6,15 @@ All rights reserved. Use of this source code is governed by a BSD-style license that can be found in the LICENSE file. */ +#if defined(PYBIND11_INTERNALS_VERSION) +# undef PYBIND11_INTERNALS_VERSION +#endif +#define PYBIND11_INTERNALS_VERSION 21814642 // Ensure this module has its own `internals` instance. #include #include +#include +#include // This file mimics a DSO that makes pybind11 calls but does not define a // PYBIND11_MODULE. The purpose is to test that such a DSO can create a @@ -21,8 +27,54 @@ namespace { namespace py = pybind11; + void gil_acquire() { py::gil_scoped_acquire gil; } +std::string gil_multi_acquire_release(unsigned bits) { + if ((bits & 0x1u) != 0u) { + py::gil_scoped_acquire gil; + } + if ((bits & 0x2u) != 0u) { + py::gil_scoped_release gil; + } + if ((bits & 0x4u) != 0u) { + py::gil_scoped_acquire gil; + } + if ((bits & 0x8u) != 0u) { + py::gil_scoped_release gil; + } + return PYBIND11_INTERNALS_ID; +} + +struct CustomAutoGIL { + CustomAutoGIL() : gstate(PyGILState_Ensure()) {} + ~CustomAutoGIL() { PyGILState_Release(gstate); } + + PyGILState_STATE gstate; +}; +struct CustomAutoNoGIL { + CustomAutoNoGIL() : save(PyEval_SaveThread()) {} + ~CustomAutoNoGIL() { PyEval_RestoreThread(save); } + + PyThreadState *save; +}; + +template +void gil_acquire_inner() { + Acquire acquire_outer; + Acquire acquire_inner; + Release release; +} + +template +void gil_acquire_nested() { + Acquire acquire_outer; + Acquire acquire_inner; + Release release; + auto thread = std::thread(&gil_acquire_inner); + thread.join(); +} + constexpr char kModuleName[] = "cross_module_gil_utils"; struct PyModuleDef moduledef = { @@ -30,6 +82,9 @@ struct PyModuleDef moduledef = { } // namespace +#define ADD_FUNCTION(Name, ...) \ + PyModule_AddObject(m, Name, PyLong_FromVoidPtr(reinterpret_cast(&__VA_ARGS__))); + extern "C" PYBIND11_EXPORT PyObject *PyInit_cross_module_gil_utils() { PyObject *m = PyModule_Create(&moduledef); @@ -37,8 +92,16 @@ extern "C" PYBIND11_EXPORT PyObject *PyInit_cross_module_gil_utils() { if (m != nullptr) { static_assert(sizeof(&gil_acquire) == sizeof(void *), "Function pointer must have the same size as void*"); - PyModule_AddObject( - m, "gil_acquire_funcaddr", PyLong_FromVoidPtr(reinterpret_cast(&gil_acquire))); + ADD_FUNCTION("gil_acquire_funcaddr", gil_acquire) + ADD_FUNCTION("gil_multi_acquire_release_funcaddr", gil_multi_acquire_release) + ADD_FUNCTION("gil_acquire_inner_custom_funcaddr", + gil_acquire_inner) + ADD_FUNCTION("gil_acquire_nested_custom_funcaddr", + gil_acquire_nested) + ADD_FUNCTION("gil_acquire_inner_pybind11_funcaddr", + gil_acquire_inner) + ADD_FUNCTION("gil_acquire_nested_pybind11_funcaddr", + gil_acquire_nested) } return m; diff --git a/tests/pybind11_tests.cpp b/tests/pybind11_tests.cpp index aa3095594b..6240346487 100644 --- a/tests/pybind11_tests.cpp +++ b/tests/pybind11_tests.cpp @@ -89,6 +89,12 @@ PYBIND11_MODULE(pybind11_tests, m) { #endif m.attr("cpp_std") = cpp_std(); m.attr("PYBIND11_INTERNALS_ID") = PYBIND11_INTERNALS_ID; + m.attr("PYBIND11_SIMPLE_GIL_MANAGEMENT") = +#if defined(PYBIND11_SIMPLE_GIL_MANAGEMENT) + true; +#else + false; +#endif bind_ConstructorStats(m); diff --git a/tests/test_embed/test_interpreter.cpp b/tests/test_embed/test_interpreter.cpp index 6299293b91..44dcd1fdb0 100644 --- a/tests/test_embed/test_interpreter.cpp +++ b/tests/test_embed/test_interpreter.cpp @@ -293,7 +293,6 @@ TEST_CASE("Threads") { { py::gil_scoped_release gil_release{}; - REQUIRE(has_pybind11_internals_static()); auto threads = std::vector(); for (auto i = 0; i < num_threads; ++i) { diff --git a/tests/test_gil_scoped.cpp b/tests/test_gil_scoped.cpp index 97efdc1616..f136086e84 100644 --- a/tests/test_gil_scoped.cpp +++ b/tests/test_gil_scoped.cpp @@ -11,6 +11,13 @@ #include "pybind11_tests.h" +#include +#include + +#define CROSS_MODULE(Function) \ + auto cm = py::module_::import("cross_module_gil_utils"); \ + auto target = reinterpret_cast(PyLong_AsVoidPtr(cm.attr(Function).ptr())); + class VirtClass { public: virtual ~VirtClass() = default; @@ -28,6 +35,16 @@ class PyVirtClass : public VirtClass { }; TEST_SUBMODULE(gil_scoped, m) { + m.attr("defined_THREAD_SANITIZER") = +#if defined(THREAD_SANITIZER) + true; +#else + false; +#endif + + m.def("intentional_deadlock", + []() { std::thread([]() { py::gil_scoped_acquire gil_acquired; }).join(); }); + py::class_(m, "VirtClass") .def(py::init<>()) .def("virtual_func", &VirtClass::virtual_func) @@ -37,11 +54,91 @@ TEST_SUBMODULE(gil_scoped, m) { m.def("test_callback_std_func", [](const std::function &func) { func(); }); m.def("test_callback_virtual_func", [](VirtClass &virt) { virt.virtual_func(); }); m.def("test_callback_pure_virtual_func", [](VirtClass &virt) { virt.pure_virtual_func(); }); - m.def("test_cross_module_gil", []() { - auto cm = py::module_::import("cross_module_gil_utils"); - auto gil_acquire = reinterpret_cast( - PyLong_AsVoidPtr(cm.attr("gil_acquire_funcaddr").ptr())); + m.def("test_cross_module_gil_released", []() { + CROSS_MODULE("gil_acquire_funcaddr") + py::gil_scoped_release gil_release; + target(); + }); + m.def("test_cross_module_gil_acquired", []() { + CROSS_MODULE("gil_acquire_funcaddr") + py::gil_scoped_acquire gil_acquire; + target(); + }); + m.def("test_cross_module_gil_inner_custom_released", []() { + CROSS_MODULE("gil_acquire_inner_custom_funcaddr") + py::gil_scoped_release gil_release; + target(); + }); + m.def("test_cross_module_gil_inner_custom_acquired", []() { + CROSS_MODULE("gil_acquire_inner_custom_funcaddr") + py::gil_scoped_acquire gil_acquire; + target(); + }); + m.def("test_cross_module_gil_inner_pybind11_released", []() { + CROSS_MODULE("gil_acquire_inner_pybind11_funcaddr") py::gil_scoped_release gil_release; - gil_acquire(); + target(); + }); + m.def("test_cross_module_gil_inner_pybind11_acquired", []() { + CROSS_MODULE("gil_acquire_inner_pybind11_funcaddr") + py::gil_scoped_acquire gil_acquire; + target(); + }); + m.def("test_cross_module_gil_nested_custom_released", []() { + CROSS_MODULE("gil_acquire_nested_custom_funcaddr") + py::gil_scoped_release gil_release; + target(); + }); + m.def("test_cross_module_gil_nested_custom_acquired", []() { + CROSS_MODULE("gil_acquire_nested_custom_funcaddr") + py::gil_scoped_acquire gil_acquire; + target(); + }); + m.def("test_cross_module_gil_nested_pybind11_released", []() { + CROSS_MODULE("gil_acquire_nested_pybind11_funcaddr") + py::gil_scoped_release gil_release; + target(); + }); + m.def("test_cross_module_gil_nested_pybind11_acquired", []() { + CROSS_MODULE("gil_acquire_nested_pybind11_funcaddr") + py::gil_scoped_acquire gil_acquire; + target(); + }); + m.def("test_release_acquire", [](const py::object &obj) { + py::gil_scoped_release gil_released; + py::gil_scoped_acquire gil_acquired; + return py::str(obj); + }); + m.def("test_nested_acquire", [](const py::object &obj) { + py::gil_scoped_release gil_released; + py::gil_scoped_acquire gil_acquired_outer; + py::gil_scoped_acquire gil_acquired_inner; + return py::str(obj); + }); + m.def("test_multi_acquire_release_cross_module", [](unsigned bits) { + py::set internals_ids; + internals_ids.add(PYBIND11_INTERNALS_ID); + { + py::gil_scoped_release gil_released; + auto thread_f = [bits, &internals_ids]() { + py::gil_scoped_acquire gil_acquired; + auto cm = py::module_::import("cross_module_gil_utils"); + auto target = reinterpret_cast( + PyLong_AsVoidPtr(cm.attr("gil_multi_acquire_release_funcaddr").ptr())); + std::string cm_internals_id = target(bits >> 3); + internals_ids.add(cm_internals_id); + }; + if ((bits & 0x1u) != 0u) { + thread_f(); + } + if ((bits & 0x2u) != 0u) { + std::thread non_python_thread(thread_f); + non_python_thread.join(); + } + if ((bits & 0x4u) != 0u) { + thread_f(); + } + } + return internals_ids; }); } diff --git a/tests/test_gil_scoped.py b/tests/test_gil_scoped.py index 52374b0cce..e890a7b0c8 100644 --- a/tests/test_gil_scoped.py +++ b/tests/test_gil_scoped.py @@ -1,45 +1,199 @@ import multiprocessing +import sys import threading +import time + +import pytest from pybind11_tests import gil_scoped as m +class ExtendedVirtClass(m.VirtClass): + def virtual_func(self): + pass + + def pure_virtual_func(self): + pass + + +def test_callback_py_obj(): + m.test_callback_py_obj(lambda: None) + + +def test_callback_std_func(): + m.test_callback_std_func(lambda: None) + + +def test_callback_virtual_func(): + extended = ExtendedVirtClass() + m.test_callback_virtual_func(extended) + + +def test_callback_pure_virtual_func(): + extended = ExtendedVirtClass() + m.test_callback_pure_virtual_func(extended) + + +def test_cross_module_gil_released(): + """Makes sure that the GIL can be acquired by another module from a GIL-released state.""" + m.test_cross_module_gil_released() # Should not raise a SIGSEGV + + +def test_cross_module_gil_acquired(): + """Makes sure that the GIL can be acquired by another module from a GIL-acquired state.""" + m.test_cross_module_gil_acquired() # Should not raise a SIGSEGV + + +def test_cross_module_gil_inner_custom_released(): + """Makes sure that the GIL can be acquired/released by another module + from a GIL-released state using custom locking logic.""" + m.test_cross_module_gil_inner_custom_released() + + +def test_cross_module_gil_inner_custom_acquired(): + """Makes sure that the GIL can be acquired/acquired by another module + from a GIL-acquired state using custom locking logic.""" + m.test_cross_module_gil_inner_custom_acquired() + + +def test_cross_module_gil_inner_pybind11_released(): + """Makes sure that the GIL can be acquired/released by another module + from a GIL-released state using pybind11 locking logic.""" + m.test_cross_module_gil_inner_pybind11_released() + + +def test_cross_module_gil_inner_pybind11_acquired(): + """Makes sure that the GIL can be acquired/acquired by another module + from a GIL-acquired state using pybind11 locking logic.""" + m.test_cross_module_gil_inner_pybind11_acquired() + + +def test_cross_module_gil_nested_custom_released(): + """Makes sure that the GIL can be nested acquired/released by another module + from a GIL-released state using custom locking logic.""" + m.test_cross_module_gil_nested_custom_released() + + +def test_cross_module_gil_nested_custom_acquired(): + """Makes sure that the GIL can be nested acquired/acquired by another module + from a GIL-acquired state using custom locking logic.""" + m.test_cross_module_gil_nested_custom_acquired() + + +def test_cross_module_gil_nested_pybind11_released(): + """Makes sure that the GIL can be nested acquired/released by another module + from a GIL-released state using pybind11 locking logic.""" + m.test_cross_module_gil_nested_pybind11_released() + + +def test_cross_module_gil_nested_pybind11_acquired(): + """Makes sure that the GIL can be nested acquired/acquired by another module + from a GIL-acquired state using pybind11 locking logic.""" + m.test_cross_module_gil_nested_pybind11_acquired() + + +def test_release_acquire(): + assert m.test_release_acquire(0xAB) == "171" + + +def test_nested_acquire(): + assert m.test_nested_acquire(0xAB) == "171" + + +def test_multi_acquire_release_cross_module(): + for bits in range(16 * 8): + internals_ids = m.test_multi_acquire_release_cross_module(bits) + assert len(internals_ids) == 2 if bits % 8 else 1 + + +# Intentionally putting human review in the loop here, to guard against accidents. +VARS_BEFORE_ALL_BASIC_TESTS = dict(vars()) # Make a copy of the dict (critical). +ALL_BASIC_TESTS = ( + test_callback_py_obj, + test_callback_std_func, + test_callback_virtual_func, + test_callback_pure_virtual_func, + test_cross_module_gil_released, + test_cross_module_gil_acquired, + test_cross_module_gil_inner_custom_released, + test_cross_module_gil_inner_custom_acquired, + test_cross_module_gil_inner_pybind11_released, + test_cross_module_gil_inner_pybind11_acquired, + test_cross_module_gil_nested_custom_released, + test_cross_module_gil_nested_custom_acquired, + test_cross_module_gil_nested_pybind11_released, + test_cross_module_gil_nested_pybind11_acquired, + test_release_acquire, + test_nested_acquire, + test_multi_acquire_release_cross_module, +) + + +def test_all_basic_tests_completeness(): + num_found = 0 + for key, value in VARS_BEFORE_ALL_BASIC_TESTS.items(): + if not key.startswith("test_"): + continue + assert value in ALL_BASIC_TESTS + num_found += 1 + assert len(ALL_BASIC_TESTS) == num_found + + +def _intentional_deadlock(): + m.intentional_deadlock() + + +ALL_BASIC_TESTS_PLUS_INTENTIONAL_DEADLOCK = ALL_BASIC_TESTS + (_intentional_deadlock,) +SKIP_IF_DEADLOCK = True # See PR #4216 + + def _run_in_process(target, *args, **kwargs): - """Runs target in process and returns its exitcode after 10s (None if still alive).""" + if len(args) == 0: + test_fn = target + else: + test_fn = args[0] + # Do not need to wait much, 10s should be more than enough. + timeout = 0.1 if test_fn is _intentional_deadlock else 10 process = multiprocessing.Process(target=target, args=args, kwargs=kwargs) process.daemon = True try: + t_start = time.time() process.start() - # Do not need to wait much, 10s should be more than enough. - process.join(timeout=10) + if timeout >= 100: # For debugging. + print( + "\nprocess.pid STARTED", process.pid, (sys.argv, target, args, kwargs) + ) + print(f"COPY-PASTE-THIS: gdb {sys.argv[0]} -p {process.pid}", flush=True) + process.join(timeout=timeout) + if timeout >= 100: + print("\nprocess.pid JOINED", process.pid, flush=True) + t_delta = time.time() - t_start + if process.exitcode == 66 and m.defined_THREAD_SANITIZER: # Issue #2754 + # WOULD-BE-NICE-TO-HAVE: Check that the message below is actually in the output. + # Maybe this could work: + # https://gist.github.com/alexeygrigorev/01ce847f2e721b513b42ea4a6c96905e + pytest.skip( + "ThreadSanitizer: starting new threads after multi-threaded fork is not supported." + ) + elif test_fn is _intentional_deadlock: + assert process.exitcode is None + return 0 + elif process.exitcode is None: + assert t_delta > 0.9 * timeout + msg = "DEADLOCK, most likely, exactly what this test is meant to detect." + if SKIP_IF_DEADLOCK: + pytest.skip(msg) + raise RuntimeError(msg) return process.exitcode finally: if process.is_alive(): process.terminate() -def _python_to_cpp_to_python(): - """Calls different C++ functions that come back to Python.""" - - class ExtendedVirtClass(m.VirtClass): - def virtual_func(self): - pass - - def pure_virtual_func(self): - pass - - extended = ExtendedVirtClass() - m.test_callback_py_obj(lambda: None) - m.test_callback_std_func(lambda: None) - m.test_callback_virtual_func(extended) - m.test_callback_pure_virtual_func(extended) - - -def _python_to_cpp_to_python_from_threads(num_threads, parallel=False): - """Calls different C++ functions that come back to Python, from Python threads.""" +def _run_in_threads(test_fn, num_threads, parallel): threads = [] for _ in range(num_threads): - thread = threading.Thread(target=_python_to_cpp_to_python) + thread = threading.Thread(target=test_fn) thread.daemon = True thread.start() if parallel: @@ -51,43 +205,40 @@ def _python_to_cpp_to_python_from_threads(num_threads, parallel=False): # TODO: FIXME, sometimes returns -11 (segfault) instead of 0 on macOS Python 3.9 -def test_python_to_cpp_to_python_from_thread(): +@pytest.mark.parametrize("test_fn", ALL_BASIC_TESTS_PLUS_INTENTIONAL_DEADLOCK) +def test_run_in_process_one_thread(test_fn): """Makes sure there is no GIL deadlock when running in a thread. It runs in a separate process to be able to stop and assert if it deadlocks. """ - assert _run_in_process(_python_to_cpp_to_python_from_threads, 1) == 0 + assert _run_in_process(_run_in_threads, test_fn, num_threads=1, parallel=False) == 0 # TODO: FIXME on macOS Python 3.9 -def test_python_to_cpp_to_python_from_thread_multiple_parallel(): +@pytest.mark.parametrize("test_fn", ALL_BASIC_TESTS_PLUS_INTENTIONAL_DEADLOCK) +def test_run_in_process_multiple_threads_parallel(test_fn): """Makes sure there is no GIL deadlock when running in a thread multiple times in parallel. It runs in a separate process to be able to stop and assert if it deadlocks. """ - assert _run_in_process(_python_to_cpp_to_python_from_threads, 8, parallel=True) == 0 + assert _run_in_process(_run_in_threads, test_fn, num_threads=8, parallel=True) == 0 # TODO: FIXME on macOS Python 3.9 -def test_python_to_cpp_to_python_from_thread_multiple_sequential(): +@pytest.mark.parametrize("test_fn", ALL_BASIC_TESTS_PLUS_INTENTIONAL_DEADLOCK) +def test_run_in_process_multiple_threads_sequential(test_fn): """Makes sure there is no GIL deadlock when running in a thread multiple times sequentially. It runs in a separate process to be able to stop and assert if it deadlocks. """ - assert ( - _run_in_process(_python_to_cpp_to_python_from_threads, 8, parallel=False) == 0 - ) + assert _run_in_process(_run_in_threads, test_fn, num_threads=8, parallel=False) == 0 # TODO: FIXME on macOS Python 3.9 -def test_python_to_cpp_to_python_from_process(): +@pytest.mark.parametrize("test_fn", ALL_BASIC_TESTS_PLUS_INTENTIONAL_DEADLOCK) +def test_run_in_process_direct(test_fn): """Makes sure there is no GIL deadlock when using processes. This test is for completion, but it was never an issue. """ - assert _run_in_process(_python_to_cpp_to_python) == 0 - - -def test_cross_module_gil(): - """Makes sure that the GIL can be acquired by another module from a GIL-released state.""" - m.test_cross_module_gil() # Should not raise a SIGSEGV + assert _run_in_process(test_fn) == 0 From 3fb36a99f68c082ce9fd58cbb3e6058ababd3b3e Mon Sep 17 00:00:00 2001 From: "Ralf W. Grosse-Kunstleve" Date: Mon, 31 Oct 2022 09:18:05 -0700 Subject: [PATCH 13/43] fix: unicode surrogate character in Python exception message. (#4297) * Fix & test for issue #4288 (unicode surrogate character in Python exception message). * DRY `message_unavailable_exc` * fix: add a constexpr Co-authored-by: Aaron Gokaslan * style: pre-commit fixes Co-authored-by: Henry Schreiner Co-authored-by: Aaron Gokaslan Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> --- include/pybind11/pytypes.h | 22 ++++++++++++++++++++-- tests/test_exceptions.cpp | 5 ----- tests/test_exceptions.py | 14 ++++++++++++++ 3 files changed, 34 insertions(+), 7 deletions(-) diff --git a/include/pybind11/pytypes.h b/include/pybind11/pytypes.h index 80a2e2228e..91cbaaba28 100644 --- a/include/pybind11/pytypes.h +++ b/include/pybind11/pytypes.h @@ -501,11 +501,29 @@ struct error_fetch_and_normalize { std::string message_error_string; if (m_value) { auto value_str = reinterpret_steal(PyObject_Str(m_value.ptr())); + constexpr const char *message_unavailable_exc + = ""; if (!value_str) { message_error_string = detail::error_string(); - result = ""; + result = message_unavailable_exc; } else { - result = value_str.cast(); + // Not using `value_str.cast()`, to not potentially throw a secondary + // error_already_set that will then result in process termination (#4288). + auto value_bytes = reinterpret_steal( + PyUnicode_AsEncodedString(value_str.ptr(), "utf-8", "backslashreplace")); + if (!value_bytes) { + message_error_string = detail::error_string(); + result = message_unavailable_exc; + } else { + char *buffer = nullptr; + Py_ssize_t length = 0; + if (PyBytes_AsStringAndSize(value_bytes.ptr(), &buffer, &length) == -1) { + message_error_string = detail::error_string(); + result = message_unavailable_exc; + } else { + result = std::string(buffer, static_cast(length)); + } + } } } else { result = ""; diff --git a/tests/test_exceptions.cpp b/tests/test_exceptions.cpp index 3583f22a50..f57e095068 100644 --- a/tests/test_exceptions.cpp +++ b/tests/test_exceptions.cpp @@ -105,11 +105,6 @@ struct PythonAlreadySetInDestructor { py::str s; }; -std::string error_already_set_what(const py::object &exc_type, const py::object &exc_value) { - PyErr_SetObject(exc_type.ptr(), exc_value.ptr()); - return py::error_already_set().what(); -} - TEST_SUBMODULE(exceptions, m) { m.def("throw_std_exception", []() { throw std::runtime_error("This exception was intentionally thrown."); }); diff --git a/tests/test_exceptions.py b/tests/test_exceptions.py index 70b6ffea95..7eb1a9d62c 100644 --- a/tests/test_exceptions.py +++ b/tests/test_exceptions.py @@ -275,6 +275,20 @@ def test_local_translator(msg): assert msg(excinfo.value) == "this mod" +def test_error_already_set_message_with_unicode_surrogate(): # Issue #4288 + assert m.error_already_set_what(RuntimeError, "\ud927") == ( + "RuntimeError: \\ud927", + False, + ) + + +def test_error_already_set_message_with_malformed_utf8(): + assert m.error_already_set_what(RuntimeError, b"\x80") == ( + "RuntimeError: b'\\x80'", + False, + ) + + class FlakyException(Exception): def __init__(self, failure_point): if failure_point == "failure_point_init": From f2ee641ee3c613c07c84cd4a473e1398002ded7f Mon Sep 17 00:00:00 2001 From: Henry Schreiner Date: Mon, 31 Oct 2022 14:11:23 -0400 Subject: [PATCH 14/43] docs: prepare for 2.10.1 release (#4279) * docs: prepare for 2.10.1 release Signed-off-by: Henry Schreiner * Update changelog.rst * docs: update changelog with final list of PRs Signed-off-by: Henry Schreiner * Update docs/changelog.rst * chore: one more changelog bump Signed-off-by: Henry Schreiner --- docs/changelog.rst | 43 ++++++++++++++++++++++++++++++++++++++++--- 1 file changed, 40 insertions(+), 3 deletions(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index df3181c524..7d6d0c0f56 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -10,19 +10,18 @@ Changes will be added here periodically from the "Suggested changelog entry" block in pull request descriptions. - IN DEVELOPMENT -------------- Changes will be summarized here periodically. -Version 2.10.1 (Oct 2?, 2022) +Version 2.10.1 (Oct 31, 2022) ----------------------------- +This is the first version to fully support embedding the newly released Python 3.11. Changes: - * Allow ``pybind11::capsule`` constructor to take null destructor pointers. `#4221 `_ @@ -30,8 +29,40 @@ Changes: (established behavior). `#4119 `_ +* A ``PYBIND11_SIMPLE_GIL_MANAGEMENT`` option was added (cmake, C++ define), + along with many additional tests in ``test_gil_scoped.py``. The option may be + useful to try when debugging GIL-related issues, to determine if the more + complex default implementation is or is not to blame. See #4216 for + background. WARNING: Please be careful to not create ODR violations when + using the option: everything that is linked together with mutual symbol + visibility needs to be rebuilt. + `#4216 `_ + +* ``PYBIND11_EXPORT_EXCEPTION`` was made non-empty only under macOS. This makes + Linux builds safer, and enables the removal of warning suppression pragmas for + Windows. + `#4298 `_ + Bug fixes: +* Fixed a bug where ``UnicodeDecodeError`` was not propagated from various + ``py::str`` ctors when decoding surrogate utf characters. + `#4294 `_ + +* Revert perfect forwarding for ``make_iterator``. This broke at least one + valid use case. May revisit later. + `#4234 `_ + +* Fix support for safe casts to ``void*`` (regression in 2.10.0). + `#4275 `_ + +* Fix ``char8_t`` support (regression in 2.9). + `#4278 `_ + +* Unicode surrogate character in Python exception message leads to process + termination in ``error_already_set::what()``. + `#4297 `_ + * Fix MSVC 2019 v.1924 & C++14 mode error for ``overload_cast``. `#4188 `_ @@ -100,9 +131,15 @@ Performance and style: * Optimize unpacking_collector when processing ``arg_v`` arguments. `#4219 `_ +* Optimize casting C++ object to ``None``. + `#4269 `_ + Build system improvements: +* CMake: revert overwrite behavior, now opt-in with ``PYBIND11_PYTHONLIBS_OVERRWRITE OFF``. + `#4195 `_ + * Include a pkg-config file when installing pybind11, such as in the Python package. `#4077 `_ From 80dc998efced8ceb2be59756668a7e90e8bef917 Mon Sep 17 00:00:00 2001 From: Henry Schreiner Date: Mon, 31 Oct 2022 14:44:51 -0400 Subject: [PATCH 15/43] chore: bump versions for 2.10.1 Signed-off-by: Henry Schreiner --- include/pybind11/detail/common.h | 6 +++--- pybind11/_version.py | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/include/pybind11/detail/common.h b/include/pybind11/detail/common.h index a3e0bc9b37..f34fd279c2 100644 --- a/include/pybind11/detail/common.h +++ b/include/pybind11/detail/common.h @@ -10,12 +10,12 @@ #pragma once #define PYBIND11_VERSION_MAJOR 2 -#define PYBIND11_VERSION_MINOR 11 -#define PYBIND11_VERSION_PATCH 0.dev1 +#define PYBIND11_VERSION_MINOR 10 +#define PYBIND11_VERSION_PATCH 1 // Similar to Python's convention: https://docs.python.org/3/c-api/apiabiversion.html // Additional convention: 0xD = dev -#define PYBIND11_VERSION_HEX 0x020B00D1 +#define PYBIND11_VERSION_HEX 0x020A0100 #define PYBIND11_NAMESPACE_BEGIN(name) namespace name { #define PYBIND11_NAMESPACE_END(name) } diff --git a/pybind11/_version.py b/pybind11/_version.py index 1cb51fc5c2..3d8c2cd360 100644 --- a/pybind11/_version.py +++ b/pybind11/_version.py @@ -8,5 +8,5 @@ def _to_int(s: str) -> Union[int, str]: return s -__version__ = "2.11.0.dev1" +__version__ = "2.10.1" version_info = tuple(_to_int(s) for s in __version__.split(".")) From 2de6e398e1c0453de51886e18366ac58aebd8b92 Mon Sep 17 00:00:00 2001 From: Ethan Steinberg Date: Fri, 30 Dec 2022 10:44:55 -0800 Subject: [PATCH 16/43] [v2.10] Revert the addition of the GIL check feature (#4432) * Revert the GIL check * Update include/pybind11/detail/common.h Co-authored-by: Henry Schreiner --- include/pybind11/detail/common.h | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/include/pybind11/detail/common.h b/include/pybind11/detail/common.h index 068ae0137a..62bf521fb9 100644 --- a/include/pybind11/detail/common.h +++ b/include/pybind11/detail/common.h @@ -329,7 +329,8 @@ PYBIND11_WARNING_POP && defined(_MSC_VER)) /* PyPy Windows: pytest hangs indefinitely at the end of the \ process (see PR #4268) */ \ && !defined(PYBIND11_ASSERT_GIL_HELD_INCREF_DECREF) -# define PYBIND11_ASSERT_GIL_HELD_INCREF_DECREF +// The following define will be enabled by default in the 2.11 release +// define PYBIND11_ASSERT_GIL_HELD_INCREF_DECREF #endif // #define PYBIND11_STR_LEGACY_PERMISSIVE From e414c4bdd64cdc220eabb5eb69420fd49dbb6d55 Mon Sep 17 00:00:00 2001 From: Ethan Steinberg Date: Fri, 30 Dec 2022 10:46:55 -0800 Subject: [PATCH 17/43] fix: improve the error reporting for inc_ref GIL failures (#4427) * First * Fixs * Improve * Additional assertions comment * Improve docs --- docs/advanced/misc.rst | 28 ++++++++++++++++++++++++++++ include/pybind11/pytypes.h | 30 +++++++++++++++++++++++++----- 2 files changed, 53 insertions(+), 5 deletions(-) diff --git a/docs/advanced/misc.rst b/docs/advanced/misc.rst index 35a6ebcd60..805ec838fc 100644 --- a/docs/advanced/misc.rst +++ b/docs/advanced/misc.rst @@ -118,6 +118,34 @@ The ``call_go`` wrapper can also be simplified using the ``call_guard`` policy m.def("call_go", &call_go, py::call_guard()); +Common Sources Of Global Interpreter Lock Errors +================================================================== + +Failing to properly hold the Global Interpreter Lock (GIL) is one of the +more common sources of bugs within code that uses pybind11. If you are +running into GIL related errors, we highly recommend you consult the +following checklist. + +- Do you have any global variables that are pybind11 objects or invoke + pybind11 functions in either their constructor or destructor? You are generally + not allowed to invoke any Python function in a global static context. We recommend + using lazy initialization and then intentionally leaking at the end of the program. + +- Do you have any pybind11 objects that are members of other C++ structures? One + commonly overlooked requirement is that pybind11 objects have to increase their reference count + whenever their copy constructor is called. Thus, you need to be holding the GIL to invoke + the copy constructor of any C++ class that has a pybind11 member. This can sometimes be very + tricky to track for complicated programs Think carefully when you make a pybind11 object + a member in another struct. + +- C++ destructors that invoke Python functions can be particularly troublesome as + destructors can sometimes get invoked in weird and unexpected circumstances as a result + of exceptions. + +- You should try running your code in a debug build. That will enable additional assertions + within pybind11 that will throw exceptions on certain GIL handling errors + (reference counting operations). + Binding sequence data types, iterators, the slicing protocol, etc. ================================================================== diff --git a/include/pybind11/pytypes.h b/include/pybind11/pytypes.h index 56e0423ac4..3660c180d9 100644 --- a/include/pybind11/pytypes.h +++ b/include/pybind11/pytypes.h @@ -250,9 +250,9 @@ class handle : public detail::object_api { #ifdef PYBIND11_HANDLE_REF_DEBUG inc_ref_counter(1); #endif -#if defined(PYBIND11_ASSERT_GIL_HELD_INCREF_DECREF) +#ifdef PYBIND11_ASSERT_GIL_HELD_INCREF_DECREF if (m_ptr != nullptr && !PyGILState_Check()) { - throw std::runtime_error("pybind11::handle::inc_ref() PyGILState_Check() failure."); + throw_gilstate_error("pybind11::handle::inc_ref()"); } #endif Py_XINCREF(m_ptr); @@ -265,9 +265,9 @@ class handle : public detail::object_api { this function automatically. Returns a reference to itself. \endrst */ const handle &dec_ref() const & { -#if defined(PYBIND11_ASSERT_GIL_HELD_INCREF_DECREF) +#ifdef PYBIND11_ASSERT_GIL_HELD_INCREF_DECREF if (m_ptr != nullptr && !PyGILState_Check()) { - throw std::runtime_error("pybind11::handle::dec_ref() PyGILState_Check() failure."); + throw_gilstate_error("pybind11::handle::dec_ref()"); } #endif Py_XDECREF(m_ptr); @@ -296,8 +296,28 @@ class handle : public detail::object_api { protected: PyObject *m_ptr = nullptr; -#ifdef PYBIND11_HANDLE_REF_DEBUG private: +#ifdef PYBIND11_ASSERT_GIL_HELD_INCREF_DECREF + void throw_gilstate_error(const std::string &function_name) const { + fprintf( + stderr, + "%s is being called while the GIL is either not held or invalid. Please see " + "https://pybind11.readthedocs.io/en/stable/advanced/" + "misc.html#common-sources-of-global-interpreter-lock-errors for debugging advice.\n", + function_name.c_str()); + fflush(stderr); + if (Py_TYPE(m_ptr)->tp_name != nullptr) { + fprintf(stderr, + "The failing %s call was triggered on a %s object.\n", + function_name.c_str(), + Py_TYPE(m_ptr)->tp_name); + fflush(stderr); + } + throw std::runtime_error(function_name + " PyGILState_Check() failure."); + } +#endif + +#ifdef PYBIND11_HANDLE_REF_DEBUG static std::size_t inc_ref_counter(std::size_t add) { thread_local std::size_t counter = 0; counter += add; From 0abe64c5f6a96a4e7f795e4cd2b2dd0cae27f8d4 Mon Sep 17 00:00:00 2001 From: "Ralf W. Grosse-Kunstleve" Date: Mon, 2 Jan 2023 03:46:17 -0800 Subject: [PATCH 18/43] Fix `detail::obj_class_name()` to work correctly for meta classes. (#4436) * Fix `detail::obj_class_name()` to work correctly for meta classes. * Adjust expected name for PyPy --- include/pybind11/pytypes.h | 2 +- tests/test_class.cpp | 2 ++ tests/test_class.py | 11 ++++++++++- tests/test_pytypes.cpp | 2 ++ tests/test_pytypes.py | 6 ++++++ 5 files changed, 21 insertions(+), 2 deletions(-) diff --git a/include/pybind11/pytypes.h b/include/pybind11/pytypes.h index 3660c180d9..a2824a0e01 100644 --- a/include/pybind11/pytypes.h +++ b/include/pybind11/pytypes.h @@ -463,7 +463,7 @@ PYBIND11_NAMESPACE_BEGIN(detail) // Equivalent to obj.__class__.__name__ (or obj.__name__ if obj is a class). inline const char *obj_class_name(PyObject *obj) { - if (Py_TYPE(obj) == &PyType_Type) { + if (PyType_Check(obj)) { return reinterpret_cast(obj)->tp_name; } return Py_TYPE(obj)->tp_name; diff --git a/tests/test_class.cpp b/tests/test_class.cpp index 9ea10fae1e..ca925917e6 100644 --- a/tests/test_class.cpp +++ b/tests/test_class.cpp @@ -55,6 +55,8 @@ void bind_empty0(py::module_ &m) { } // namespace test_class TEST_SUBMODULE(class_, m) { + m.def("obj_class_name", [](py::handle obj) { return py::detail::obj_class_name(obj.ptr()); }); + // test_instance struct NoConstructor { NoConstructor() = default; diff --git a/tests/test_class.py b/tests/test_class.py index 7c1ed20608..9c964e001b 100644 --- a/tests/test_class.py +++ b/tests/test_class.py @@ -1,10 +1,19 @@ import pytest -import env # noqa: F401 +import env from pybind11_tests import ConstructorStats, UserType from pybind11_tests import class_ as m +def test_obj_class_name(): + if env.PYPY: + expected_name = "UserType" + else: + expected_name = "pybind11_tests.UserType" + assert m.obj_class_name(UserType(1)) == expected_name + assert m.obj_class_name(UserType) == expected_name + + def test_repr(): assert "pybind11_type" in repr(type(UserType)) assert "UserType" in repr(UserType) diff --git a/tests/test_pytypes.cpp b/tests/test_pytypes.cpp index ea8d03958b..1028bb58e8 100644 --- a/tests/test_pytypes.cpp +++ b/tests/test_pytypes.cpp @@ -99,6 +99,8 @@ void m_defs(py::module_ &m) { } // namespace handle_from_move_only_type_with_operator_PyObject TEST_SUBMODULE(pytypes, m) { + m.def("obj_class_name", [](py::handle obj) { return py::detail::obj_class_name(obj.ptr()); }); + handle_from_move_only_type_with_operator_PyObject::m_defs(m); // test_bool diff --git a/tests/test_pytypes.py b/tests/test_pytypes.py index 079ee7ca50..8f9f2987e9 100644 --- a/tests/test_pytypes.py +++ b/tests/test_pytypes.py @@ -9,6 +9,12 @@ from pybind11_tests import pytypes as m +def test_obj_class_name(): + assert m.obj_class_name(None) == "NoneType" + assert m.obj_class_name(list) == "list" + assert m.obj_class_name([]) == "list" + + def test_handle_from_move_only_type_with_operator_PyObject(): # noqa: N802 assert m.handle_from_move_only_type_with_operator_PyObject_ncnst() assert m.handle_from_move_only_type_with_operator_PyObject_const() From 050de893cd4e2e9e7257a98420ed588f76aff3f1 Mon Sep 17 00:00:00 2001 From: "Ralf W. Grosse-Kunstleve" Date: Tue, 3 Jan 2023 05:46:55 -0800 Subject: [PATCH 19/43] ci: remove clang 10 C++20 (it broke recently) (#4438) * Remove clang 10 C++20 (it broke recently), add clang 15 C++20 while we are at it. * No luck trying clang15: Error response from daemon: manifest for silkeh/clang:15 not found: manifest unknown: manifest unknown. Giving up, this needs to be a separate PR. --- .github/workflows/ci.yml | 2 -- 1 file changed, 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index af5a29d627..b36bbfe1b9 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -283,8 +283,6 @@ jobs: include: - clang: 5 std: 14 - - clang: 10 - std: 20 - clang: 10 std: 17 - clang: 11 From 9d6a79c090832afe9b3140fe9ea92142f7f2329c Mon Sep 17 00:00:00 2001 From: cyy Date: Tue, 3 Jan 2023 23:20:39 +0800 Subject: [PATCH 20/43] fix: issuses detected by static analyzer (#4440) * fix incorrect variable check * remove duplicated check * remove unneeded const cast --- include/pybind11/attr.h | 2 +- include/pybind11/eigen/tensor.h | 2 +- include/pybind11/pytypes.h | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/include/pybind11/attr.h b/include/pybind11/attr.h index db7cd8efff..b5e3b7b22c 100644 --- a/include/pybind11/attr.h +++ b/include/pybind11/attr.h @@ -399,7 +399,7 @@ struct process_attribute : process_attribute_default { template <> struct process_attribute : process_attribute_default { static void init(const char *d, function_record *r) { r->doc = const_cast(d); } - static void init(const char *d, type_record *r) { r->doc = const_cast(d); } + static void init(const char *d, type_record *r) { r->doc = d; } }; template <> struct process_attribute : process_attribute {}; diff --git a/include/pybind11/eigen/tensor.h b/include/pybind11/eigen/tensor.h index 568c641a2d..0877da8953 100644 --- a/include/pybind11/eigen/tensor.h +++ b/include/pybind11/eigen/tensor.h @@ -176,7 +176,7 @@ struct type_caster::ValidType> { return false; } - if (!convert && !temp.dtype().is(dtype::of())) { + if (!temp.dtype().is(dtype::of())) { return false; } } diff --git a/include/pybind11/pytypes.h b/include/pybind11/pytypes.h index a2824a0e01..f11ed5da78 100644 --- a/include/pybind11/pytypes.h +++ b/include/pybind11/pytypes.h @@ -501,7 +501,7 @@ struct error_fetch_and_normalize { "active exception."); } const char *exc_type_name_norm = detail::obj_class_name(m_type.ptr()); - if (exc_type_name_orig == nullptr) { + if (exc_type_name_norm == nullptr) { pybind11_fail("Internal error: " + std::string(called) + " failed to obtain the name " "of the normalized active exception type."); From d78de29522d9a12505e67b58f5e417587bea9d72 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Tue, 3 Jan 2023 10:21:05 -0500 Subject: [PATCH 21/43] chore(deps): update pre-commit hooks (#4439) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit updates: - [github.com/asottile/pyupgrade: v3.3.0 → v3.3.1](https://github.com/asottile/pyupgrade/compare/v3.3.0...v3.3.1) - [github.com/PyCQA/isort: 5.10.1 → 5.11.4](https://github.com/PyCQA/isort/compare/5.10.1...5.11.4) - [github.com/psf/black: 22.10.0 → 22.12.0](https://github.com/psf/black/compare/22.10.0...22.12.0) - [github.com/PyCQA/pylint: v2.15.8 → v2.15.9](https://github.com/PyCQA/pylint/compare/v2.15.8...v2.15.9) - [github.com/shellcheck-py/shellcheck-py: v0.8.0.4 → v0.9.0.2](https://github.com/shellcheck-py/shellcheck-py/compare/v0.8.0.4...v0.9.0.2) - [github.com/pre-commit/mirrors-clang-format: v15.0.4 → v15.0.6](https://github.com/pre-commit/mirrors-clang-format/compare/v15.0.4...v15.0.6) Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> --- .pre-commit-config.yaml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 4e80fbad15..d625d5726b 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -41,20 +41,20 @@ repos: # Upgrade old Python syntax - repo: https://github.com/asottile/pyupgrade - rev: "v3.3.0" + rev: "v3.3.1" hooks: - id: pyupgrade args: [--py36-plus] # Nicely sort includes - repo: https://github.com/PyCQA/isort - rev: "5.10.1" + rev: "5.11.4" hooks: - id: isort # Black, the code formatter, natively supports pre-commit - repo: https://github.com/psf/black - rev: "22.10.0" # Keep in sync with blacken-docs + rev: "22.12.0" # Keep in sync with blacken-docs hooks: - id: black @@ -116,7 +116,7 @@ repos: # PyLint has native support - not always usable, but works for us - repo: https://github.com/PyCQA/pylint - rev: "v2.15.8" + rev: "v2.15.9" hooks: - id: pylint files: ^pybind11 @@ -160,7 +160,7 @@ repos: # Check for common shell mistakes - repo: https://github.com/shellcheck-py/shellcheck-py - rev: "v0.8.0.4" + rev: "v0.9.0.2" hooks: - id: shellcheck @@ -175,7 +175,7 @@ repos: # Clang format the codebase automatically - repo: https://github.com/pre-commit/mirrors-clang-format - rev: "v15.0.4" + rev: "v15.0.6" hooks: - id: clang-format types_or: [c++, c, cuda] From 0bd8896a4010f2d91b2340570c24fa08606ec406 Mon Sep 17 00:00:00 2001 From: Henry Schreiner Date: Tue, 3 Jan 2023 11:34:22 -0500 Subject: [PATCH 22/43] chore: prepare for 2.10.3 (#4437) * docs: update changelog for v2.10.3 Signed-off-by: Henry Schreiner * chore: bump versions for 2.10.3 Signed-off-by: Henry Schreiner * chore: fix make changelog script with entry is empty Signed-off-by: Henry Schreiner Signed-off-by: Henry Schreiner --- docs/changelog.rst | 35 +++++++++++++++++++++++++++++++- include/pybind11/detail/common.h | 4 ++-- pybind11/_version.py | 2 +- tools/make_changelog.py | 9 ++++---- 4 files changed, 41 insertions(+), 9 deletions(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index 9ff7281bc6..bb111c5f26 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -15,6 +15,39 @@ IN DEVELOPMENT Changes will be summarized here periodically. +Changes: + +* ``PyGILState_Check()``'s in ``pybind11::handle``'s ``inc_ref()`` & + ``dec_ref()`` are now enabled by default again. + `#4246 `_ + +Build system improvements: + +* Update clang-tidy to 15 in CI. + `#4387 `_ + + +Version 2.10.3 (Jan 3, 2023) +---------------------------- + +Changes: + +* Temporarily made our GIL status assertions (added in 2.10.2) disabled by + default (re-enable manually by defining + ``PYBIND11_ASSERT_GIL_HELD_INCREF_DECREF``, will be enabled in 2.11). + `#4432 `_ + +* Improved error messages when ``inc_ref``/``dec_ref`` are called with an + invalid GIL state. + `#4427 `_ + `#4436 `_ + +Bug Fixes: + +* Some minor touchups found by static analyzers. + `#4440 `_ + + Version 2.10.2 (Dec 20, 2022) ----------------------------- @@ -30,7 +63,7 @@ Changes: * ``PyGILState_Check()``'s were integrated to ``pybind11::handle`` ``inc_ref()`` & ``dec_ref()``. The added GIL checks are guarded by ``PYBIND11_ASSERT_GIL_HELD_INCREF_DECREF``, which is the default only if - ``NDEBUG`` is not defined. + ``NDEBUG`` is not defined. (Made non-default in 2.10.3, will be active in 2.11) `#4246 `_ * Add option for enable/disable enum members in docstring. diff --git a/include/pybind11/detail/common.h b/include/pybind11/detail/common.h index 62bf521fb9..12bb173ad1 100644 --- a/include/pybind11/detail/common.h +++ b/include/pybind11/detail/common.h @@ -11,11 +11,11 @@ #define PYBIND11_VERSION_MAJOR 2 #define PYBIND11_VERSION_MINOR 10 -#define PYBIND11_VERSION_PATCH 2 +#define PYBIND11_VERSION_PATCH 3 // Similar to Python's convention: https://docs.python.org/3/c-api/apiabiversion.html // Additional convention: 0xD = dev -#define PYBIND11_VERSION_HEX 0x020A0200 +#define PYBIND11_VERSION_HEX 0x020A0300 // Define some generic pybind11 helper macros for warning management. // diff --git a/pybind11/_version.py b/pybind11/_version.py index 4c814edfac..63078bbe67 100644 --- a/pybind11/_version.py +++ b/pybind11/_version.py @@ -8,5 +8,5 @@ def _to_int(s: str) -> Union[int, str]: return s -__version__ = "2.10.2" +__version__ = "2.10.3" version_info = tuple(_to_int(s) for s in __version__.split(".")) diff --git a/tools/make_changelog.py b/tools/make_changelog.py index 839040a938..b5bd832940 100755 --- a/tools/make_changelog.py +++ b/tools/make_changelog.py @@ -31,8 +31,10 @@ missing = [] for issue in issues: - changelog = ENTRY.findall(issue.body) - if changelog: + changelog = ENTRY.findall(issue.body or "") + if not changelog or not changelog[0]: + missing.append(issue) + else: (msg,) = changelog if not msg.startswith("* "): msg = "* " + msg @@ -44,9 +46,6 @@ print(Syntax(msg, "rst", theme="ansi_light", word_wrap=True)) print() - else: - missing.append(issue) - if missing: print() print("[blue]" + "-" * 30) From 0ed64a047be870c7da22035d27d0742b73550701 Mon Sep 17 00:00:00 2001 From: kajananchinniah Date: Wed, 28 Dec 2022 12:49:57 -0500 Subject: [PATCH 23/43] docs: fixed typo in spelling of first (#4428) --- include/pybind11/detail/common.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/include/pybind11/detail/common.h b/include/pybind11/detail/common.h index 12bb173ad1..cb3f5cf39f 100644 --- a/include/pybind11/detail/common.h +++ b/include/pybind11/detail/common.h @@ -437,7 +437,7 @@ PYBIND11_WARNING_POP /** \rst This macro creates the entry point that will be invoked when the Python interpreter - imports an extension module. The module name is given as the fist argument and it + imports an extension module. The module name is given as the first argument and it should not be in quotes. The second macro argument defines a variable of type `py::module_` which can be used to initialize the module. From f14bb03d15d5a51302587c5667f5bb8813423900 Mon Sep 17 00:00:00 2001 From: "Ralf W. Grosse-Kunstleve" Date: Thu, 12 Jan 2023 17:50:28 -0800 Subject: [PATCH 24/43] Add clang15 C++20 job (#4443) --- .github/workflows/ci.yml | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index b36bbfe1b9..7c5fcb43ac 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -280,6 +280,8 @@ jobs: - dev std: - 11 + container_suffix: + - "" include: - clang: 5 std: 14 @@ -293,9 +295,12 @@ jobs: std: 20 - clang: 14 std: 20 + - clang: 15 + std: 20 + container_suffix: "-bullseye" name: "🐍 3 • Clang ${{ matrix.clang }} • C++${{ matrix.std }} • x64" - container: "silkeh/clang:${{ matrix.clang }}" + container: "silkeh/clang:${{ matrix.clang }}${{ matrix.container_suffix }}" steps: - uses: actions/checkout@v3 From 4f6183cfe2e09b51b9a7964bd8083dc4198ada0e Mon Sep 17 00:00:00 2001 From: "Ralf W. Grosse-Kunstleve" Date: Sat, 14 Jan 2023 13:47:56 -0800 Subject: [PATCH 25/43] Ensure `import pybind11_tests` traceback is shown. (#4455) --- tests/conftest.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/tests/conftest.py b/tests/conftest.py index 402fd4b25b..c24d51d4d1 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -11,11 +11,17 @@ import os import re import textwrap +import traceback import pytest # Early diagnostic for failed imports -import pybind11_tests +try: + import pybind11_tests +except Exception: + # pytest does not show the traceback without this. + traceback.print_exc() + raise @pytest.fixture(scope="session", autouse=True) From da780a00ae6b00e488aa1b02575c34f7e2e88607 Mon Sep 17 00:00:00 2001 From: albanD Date: Wed, 18 Jan 2023 21:11:26 +0100 Subject: [PATCH 26/43] Make sure to properly untrack gc objects before freeing them (#4461) * Make sure to properly untrack gc objects before freeing them * style: pre-commit fixes * Fix lint * Add comment about where the original track comes from Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> --- include/pybind11/detail/class.h | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/include/pybind11/detail/class.h b/include/pybind11/detail/class.h index 528e716f78..bc2b40c50a 100644 --- a/include/pybind11/detail/class.h +++ b/include/pybind11/detail/class.h @@ -445,9 +445,17 @@ inline void clear_instance(PyObject *self) { /// Instance destructor function for all pybind11 types. It calls `type_info.dealloc` /// to destroy the C++ object itself, while the rest is Python bookkeeping. extern "C" inline void pybind11_object_dealloc(PyObject *self) { + auto *type = Py_TYPE(self); + + // If this is a GC tracked object, untrack it first + // Note that the track call is implicitly done by the + // default tp_alloc, which we never override. + if (PyType_HasFeature(type, Py_TPFLAGS_HAVE_GC) != 0) { + PyObject_GC_UnTrack(self); + } + clear_instance(self); - auto *type = Py_TYPE(self); type->tp_free(self); #if PY_VERSION_HEX < 0x03080000 From 5ece09ad301e135c05e0ff758acccbd4162f0a61 Mon Sep 17 00:00:00 2001 From: "Ralf W. Grosse-Kunstleve" Date: Thu, 19 Jan 2023 10:48:46 -0800 Subject: [PATCH 27/43] Resolve new flake8 error (#4462) * Resolve flake8 error by replacing `pytest.raises(Exception)` with `SystemError` * Also remove the obsolete comment. * Tweak comment instead of removing it. --- tests/test_modules.py | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/tests/test_modules.py b/tests/test_modules.py index e11d68e78e..c3e8cea6e1 100644 --- a/tests/test_modules.py +++ b/tests/test_modules.py @@ -107,11 +107,10 @@ def test_def_submodule_failures(): sm_name_orig = sm.__name__ sm.__name__ = malformed_utf8 try: - with pytest.raises(Exception): - # Seen with Python 3.9: SystemError: nameless module - # But we do not want to exercise the internals of PyModule_GetName(), which could - # change in future versions of Python, but a bad __name__ is very likely to cause - # some kind of failure indefinitely. + # We want to assert that a bad __name__ causes some kind of failure, although we do not want to exercise + # the internals of PyModule_GetName(). Currently all supported Python versions raise SystemError. If that + # changes in future Python versions, simply add the new expected exception types here. + with pytest.raises(SystemError): m.def_submodule(sm, b"SubSubModuleName") finally: # Clean up to ensure nothing gets upset by a module with an invalid __name__. From c0e2eeba9593ec6d1b217ad8bf00ce5701080458 Mon Sep 17 00:00:00 2001 From: "Ralf W. Grosse-Kunstleve" Date: Tue, 31 Jan 2023 22:44:18 -0800 Subject: [PATCH 28/43] Bump isort version to 5.12.0 (#4480) --- .pre-commit-config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index d625d5726b..9ec31ae997 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -48,7 +48,7 @@ repos: # Nicely sort includes - repo: https://github.com/PyCQA/isort - rev: "5.11.4" + rev: "5.12.0" hooks: - id: isort From 993eb2b640f1d6c083f75035b129cfb61a61d37d Mon Sep 17 00:00:00 2001 From: Henry Schreiner Date: Wed, 1 Feb 2023 14:23:37 -0500 Subject: [PATCH 29/43] chore: update to black 23 (#4482) Signed-off-by: Henry Schreiner --- .pre-commit-config.yaml | 18 +++++++++--------- docs/conf.py | 1 - pybind11/__main__.py | 1 - pybind11/setup_helpers.py | 3 --- tests/extra_python_package/test_files.py | 2 -- tests/test_chrono.py | 4 ---- tests/test_class.py | 1 - tests/test_custom_type_casters.py | 3 ++- tests/test_eigen_tensor.py | 6 ------ tests/test_local_bindings.py | 3 ++- tests/test_operator_overloading.py | 1 - 11 files changed, 13 insertions(+), 30 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 9ec31ae997..363ac71c6f 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -54,40 +54,40 @@ repos: # Black, the code formatter, natively supports pre-commit - repo: https://github.com/psf/black - rev: "22.12.0" # Keep in sync with blacken-docs + rev: "23.1.0" # Keep in sync with blacken-docs hooks: - id: black # Also code format the docs - repo: https://github.com/asottile/blacken-docs - rev: "v1.12.1" + rev: "1.13.0" hooks: - id: blacken-docs additional_dependencies: - - black==22.10.0 # keep in sync with black hook + - black==23.1.0 # keep in sync with black hook # Changes tabs to spaces - repo: https://github.com/Lucas-C/pre-commit-hooks - rev: "v1.3.1" + rev: "v1.4.2" hooks: - id: remove-tabs - repo: https://github.com/sirosen/texthooks - rev: "0.4.0" + rev: "0.5.0" hooks: - id: fix-ligatures - id: fix-smartquotes # Autoremoves unused imports - repo: https://github.com/hadialqattan/pycln - rev: "v2.1.2" + rev: "v2.1.3" hooks: - id: pycln stages: [manual] # Checking for common mistakes - repo: https://github.com/pre-commit/pygrep-hooks - rev: "v1.9.0" + rev: "v1.10.0" hooks: - id: python-check-blanket-noqa - id: python-check-blanket-type-ignore @@ -116,7 +116,7 @@ repos: # PyLint has native support - not always usable, but works for us - repo: https://github.com/PyCQA/pylint - rev: "v2.15.9" + rev: "v2.16.0" hooks: - id: pylint files: ^pybind11 @@ -175,7 +175,7 @@ repos: # Clang format the codebase automatically - repo: https://github.com/pre-commit/mirrors-clang-format - rev: "v15.0.6" + rev: "v15.0.7" hooks: - id: clang-format types_or: [c++, c, cuda] diff --git a/docs/conf.py b/docs/conf.py index 2da6773f4f..cdb9f63069 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -358,7 +358,6 @@ def clean_up(app, exception): def setup(app): - # Add hook for building doxygen xml when needed app.connect("builder-inited", generate_doxygen_xml) diff --git a/pybind11/__main__.py b/pybind11/__main__.py index 8c89533843..e87b16beb4 100644 --- a/pybind11/__main__.py +++ b/pybind11/__main__.py @@ -24,7 +24,6 @@ def print_includes() -> None: def main() -> None: - parser = argparse.ArgumentParser() parser.add_argument( "--includes", diff --git a/pybind11/setup_helpers.py b/pybind11/setup_helpers.py index 1fd04b9154..1db4795327 100644 --- a/pybind11/setup_helpers.py +++ b/pybind11/setup_helpers.py @@ -118,7 +118,6 @@ def _add_ldflags(self, flags: List[str]) -> None: self.extra_link_args[:0] = flags def __init__(self, *args: Any, **kwargs: Any) -> None: - self._cxx_level = 0 cxx_std = kwargs.pop("cxx_std", 0) @@ -174,7 +173,6 @@ def cxx_std(self) -> int: @cxx_std.setter def cxx_std(self, level: int) -> None: - if self._cxx_level: warnings.warn("You cannot safely change the cxx_level after setting it!") @@ -439,7 +437,6 @@ def compile_function( extra_postargs: Optional[List[str]] = None, depends: Optional[List[str]] = None, ) -> Any: - # These lines are directly from distutils.ccompiler.CCompiler macros, objects, extra_postargs, pp_opts, build = compiler._setup_compile( # type: ignore[attr-defined] output_dir, macros, include_dirs, sources, depends, extra_postargs diff --git a/tests/extra_python_package/test_files.py b/tests/extra_python_package/test_files.py index 9a9bb1556a..dd6393bf0b 100644 --- a/tests/extra_python_package/test_files.py +++ b/tests/extra_python_package/test_files.py @@ -135,7 +135,6 @@ def normalize_line_endings(value: bytes) -> bytes: def test_build_sdist(monkeypatch, tmpdir): - monkeypatch.chdir(MAIN_DIR) subprocess.run( @@ -186,7 +185,6 @@ def test_build_sdist(monkeypatch, tmpdir): def test_build_global_dist(monkeypatch, tmpdir): - monkeypatch.chdir(MAIN_DIR) monkeypatch.setenv("PYBIND11_GLOBAL_SDIST", "1") subprocess.run( diff --git a/tests/test_chrono.py b/tests/test_chrono.py index 7f47b37a25..a29316c38d 100644 --- a/tests/test_chrono.py +++ b/tests/test_chrono.py @@ -7,7 +7,6 @@ def test_chrono_system_clock(): - # Get the time from both c++ and datetime date0 = datetime.datetime.today() date1 = m.test_chrono1() @@ -122,7 +121,6 @@ def test_chrono_system_clock_roundtrip_time(time1, tz, monkeypatch): def test_chrono_duration_roundtrip(): - # Get the difference between two times (a timedelta) date1 = datetime.datetime.today() date2 = datetime.datetime.today() @@ -143,7 +141,6 @@ def test_chrono_duration_roundtrip(): def test_chrono_duration_subtraction_equivalence(): - date1 = datetime.datetime.today() date2 = datetime.datetime.today() @@ -154,7 +151,6 @@ def test_chrono_duration_subtraction_equivalence(): def test_chrono_duration_subtraction_equivalence_date(): - date1 = datetime.date.today() date2 = datetime.date.today() diff --git a/tests/test_class.py b/tests/test_class.py index 9c964e001b..64abac133c 100644 --- a/tests/test_class.py +++ b/tests/test_class.py @@ -185,7 +185,6 @@ def test_inheritance(msg): def test_inheritance_init(msg): - # Single base class Python(m.Pet): def __init__(self): diff --git a/tests/test_custom_type_casters.py b/tests/test_custom_type_casters.py index adfa6cf86e..731380e46a 100644 --- a/tests/test_custom_type_casters.py +++ b/tests/test_custom_type_casters.py @@ -94,7 +94,8 @@ def test_noconvert_args(msg): def test_custom_caster_destruction(): """Tests that returning a pointer to a type that gets converted with a custom type caster gets - destroyed when the function has py::return_value_policy::take_ownership policy applied.""" + destroyed when the function has py::return_value_policy::take_ownership policy applied. + """ cstats = m.destruction_tester_cstats() # This one *doesn't* have take_ownership: the pointer should be used but not destroyed: diff --git a/tests/test_eigen_tensor.py b/tests/test_eigen_tensor.py index dc8aa4643e..cd54efa069 100644 --- a/tests/test_eigen_tensor.py +++ b/tests/test_eigen_tensor.py @@ -59,7 +59,6 @@ def assert_equal_tensor_ref(mat, writeable=True, modified=None): @pytest.mark.parametrize("m", submodules) @pytest.mark.parametrize("member_name", ["member", "member_view"]) def test_reference_internal(m, member_name): - if not hasattr(sys, "getrefcount"): pytest.skip("No reference counting") foo = m.CustomExample() @@ -108,7 +107,6 @@ def test_convert_tensor_to_py(m, func_name): @pytest.mark.parametrize("m", submodules) def test_bad_cpp_to_python_casts(m): - with pytest.raises( RuntimeError, match="Cannot use reference internal when there is no parent" ): @@ -131,7 +129,6 @@ def test_bad_cpp_to_python_casts(m): @pytest.mark.parametrize("m", submodules) def test_bad_python_to_cpp_casts(m): - with pytest.raises( TypeError, match=r"^round_trip_tensor\(\): incompatible function arguments" ): @@ -194,7 +191,6 @@ def test_bad_python_to_cpp_casts(m): @pytest.mark.parametrize("m", submodules) def test_references_actually_refer(m): - a = m.reference_tensor() temp = a[indices] a[indices] = 100 @@ -211,7 +207,6 @@ def test_references_actually_refer(m): @pytest.mark.parametrize("m", submodules) def test_round_trip(m): - assert_equal_tensor_ref(m.round_trip_tensor(tensor_ref)) with pytest.raises(TypeError, match="^Cannot cast array data from"): @@ -260,7 +255,6 @@ def test_round_trip(m): @pytest.mark.parametrize("m", submodules) def test_round_trip_references_actually_refer(m): - # Need to create a copy that matches the type on the C side copy = np.array(tensor_ref, dtype=np.float64, order=m.needed_options) a = m.round_trip_view_tensor(copy) diff --git a/tests/test_local_bindings.py b/tests/test_local_bindings.py index 654d96d490..d641877396 100644 --- a/tests/test_local_bindings.py +++ b/tests/test_local_bindings.py @@ -130,7 +130,8 @@ def test_stl_bind_global(): def test_mixed_local_global(): """Local types take precedence over globally registered types: a module with a `module_local` type can be registered even if the type is already registered globally. With the module, - casting will go to the local type; outside the module casting goes to the global type.""" + casting will go to the local type; outside the module casting goes to the global type. + """ import pybind11_cross_module_tests as cm m.register_mixed_global() diff --git a/tests/test_operator_overloading.py b/tests/test_operator_overloading.py index b228da3cc3..9fde305a03 100644 --- a/tests/test_operator_overloading.py +++ b/tests/test_operator_overloading.py @@ -130,7 +130,6 @@ def test_nested(): def test_overriding_eq_reset_hash(): - assert m.Comparable(15) is not m.Comparable(15) assert m.Comparable(15) == m.Comparable(15) From a491af61211976b0b7d239df79a5b5ea474ccd5f Mon Sep 17 00:00:00 2001 From: Aaron Gokaslan Date: Sat, 4 Feb 2023 13:40:13 -0500 Subject: [PATCH 30/43] bugfix: delete proper ctors in gil.h (#4490) --- include/pybind11/gil.h | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/include/pybind11/gil.h b/include/pybind11/gil.h index cb0028d505..570a5581d5 100644 --- a/include/pybind11/gil.h +++ b/include/pybind11/gil.h @@ -152,8 +152,8 @@ class gil_scoped_release { } } - gil_scoped_release(const gil_scoped_acquire &) = delete; - gil_scoped_release &operator=(const gil_scoped_acquire &) = delete; + gil_scoped_release(const gil_scoped_release &) = delete; + gil_scoped_release &operator=(const gil_scoped_release &) = delete; /// This method will disable the PyThreadState_DeleteCurrent call and the /// GIL won't be acquired. This method should be used if the interpreter @@ -203,7 +203,7 @@ class gil_scoped_release { public: gil_scoped_release() : state{PyEval_SaveThread()} {} gil_scoped_release(const gil_scoped_release &) = delete; - gil_scoped_release &operator=(const gil_scoped_acquire &) = delete; + gil_scoped_release &operator=(const gil_scoped_release &) = delete; ~gil_scoped_release() { PyEval_RestoreThread(state); } void disarm() {} }; @@ -230,7 +230,7 @@ class gil_scoped_release { (void) (this != (this + 1)); } gil_scoped_release(const gil_scoped_release &) = delete; - gil_scoped_release &operator=(const gil_scoped_acquire &) = delete; + gil_scoped_release &operator=(const gil_scoped_release &) = delete; void disarm() {} }; From c06f324ccfecfee4211893a0d7c3217d2d697ca0 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 6 Feb 2023 00:16:52 -0500 Subject: [PATCH 31/43] chore(deps): bump ilammy/msvc-dev-cmd from 1.12.0 to 1.12.1 (#4493) Bumps [ilammy/msvc-dev-cmd](https://github.com/ilammy/msvc-dev-cmd) from 1.12.0 to 1.12.1. - [Release notes](https://github.com/ilammy/msvc-dev-cmd/releases) - [Commits](https://github.com/ilammy/msvc-dev-cmd/compare/v1.12.0...v1.12.1) --- updated-dependencies: - dependency-name: ilammy/msvc-dev-cmd dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/ci.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 7c5fcb43ac..8befccdbc2 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -766,7 +766,7 @@ jobs: uses: jwlawson/actions-setup-cmake@v1.13 - name: Prepare MSVC - uses: ilammy/msvc-dev-cmd@v1.12.0 + uses: ilammy/msvc-dev-cmd@v1.12.1 with: arch: x86 @@ -819,7 +819,7 @@ jobs: uses: jwlawson/actions-setup-cmake@v1.13 - name: Prepare MSVC - uses: ilammy/msvc-dev-cmd@v1.12.0 + uses: ilammy/msvc-dev-cmd@v1.12.1 with: arch: x86 From 3ea37d04cc3c568f192eb71df1ae01a7ac68ffaa Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Tue, 7 Feb 2023 01:16:25 -0500 Subject: [PATCH 32/43] chore(deps): update pre-commit hooks (#4495) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit updates: - [github.com/PyCQA/pylint: v2.16.0 → v2.16.1](https://github.com/PyCQA/pylint/compare/v2.16.0...v2.16.1) Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> --- .pre-commit-config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 363ac71c6f..27474f2f73 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -116,7 +116,7 @@ repos: # PyLint has native support - not always usable, but works for us - repo: https://github.com/PyCQA/pylint - rev: "v2.16.0" + rev: "v2.16.1" hooks: - id: pylint files: ^pybind11 From 1f4cf8fe0b6d5e8259f92ba33b53e5d9099ddfea Mon Sep 17 00:00:00 2001 From: Mike Essenmacher <112431871+mikeessen@users.noreply.github.com> Date: Thu, 9 Feb 2023 13:11:01 -0500 Subject: [PATCH 33/43] Replace "whitelist" with "allowlist" (#4506) --- tests/CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index 9beb268ed7..b1cb222b4a 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -169,7 +169,7 @@ if(PYBIND11_TEST_OVERRIDE) # This allows the override to be done with extensions, preserving backwards compatibility. foreach(test_name ${TEST_FILES_NO_EXT}) if(NOT ${test_name} IN_LIST TEST_OVERRIDE_NO_EXT - )# If not in the whitelist, add to be filtered out. + )# If not in the allowlist, add to be filtered out. list(APPEND PYBIND11_TEST_FILTER ${test_name}) endif() endforeach() From 1cae8dc0918b9627d803a46acb74881195a84151 Mon Sep 17 00:00:00 2001 From: Henry Schreiner Date: Thu, 16 Feb 2023 06:54:27 -0800 Subject: [PATCH 34/43] fix: tests dir has started to show up in packaging (#4510) Signed-off-by: Henry Schreiner --- MANIFEST.in | 1 + 1 file changed, 1 insertion(+) diff --git a/MANIFEST.in b/MANIFEST.in index 033303a74a..31632acc33 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -1,3 +1,4 @@ +prune tests recursive-include pybind11/include/pybind11 *.h recursive-include pybind11 *.py recursive-include pybind11 py.typed From c773a02a949efe13c8fd813734e5075f47bc1656 Mon Sep 17 00:00:00 2001 From: "Ralf W. Grosse-Kunstleve" Date: Fri, 17 Feb 2023 12:31:08 -0800 Subject: [PATCH 35/43] Appease new flake8 B028 error: (#4513) ``` flake8...................................................................Failed - hook id: flake8 - exit code: 1 pybind11/setup_helpers.py:177:13: B028 No explicit stacklevel keyword argument found. The warn method from the warnings module uses a stacklevel of 1 by default. This will only show a stack trace for the line on which the warn method is called. It is therefore recommended to use a stacklevel of 2 or greater to provide more information to the user. warnings.warn("You cannot safely change the cxx_level after setting it!") ^ ``` --- pybind11/setup_helpers.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/pybind11/setup_helpers.py b/pybind11/setup_helpers.py index 1db4795327..4c14498f6f 100644 --- a/pybind11/setup_helpers.py +++ b/pybind11/setup_helpers.py @@ -174,7 +174,9 @@ def cxx_std(self) -> int: @cxx_std.setter def cxx_std(self, level: int) -> None: if self._cxx_level: - warnings.warn("You cannot safely change the cxx_level after setting it!") + warnings.warn( + "You cannot safely change the cxx_level after setting it!", stacklevel=1 + ) # MSVC 2015 Update 3 and later only have 14 (and later 17) modes, so # force a valid flag here. From 9a1eeed03aca501f9f944313f0374ccd6d807695 Mon Sep 17 00:00:00 2001 From: "Ralf W. Grosse-Kunstleve" Date: Fri, 17 Feb 2023 12:58:35 -0800 Subject: [PATCH 36/43] Make warning suppressions MINGW-specific again. (#4515) Background: https://github.com/pybind/pybind11/pull/4285#issuecomment-1435066554 --- include/pybind11/eigen/matrix.h | 2 ++ include/pybind11/eigen/tensor.h | 2 ++ 2 files changed, 4 insertions(+) diff --git a/include/pybind11/eigen/matrix.h b/include/pybind11/eigen/matrix.h index 34fe329a82..bd533bed37 100644 --- a/include/pybind11/eigen/matrix.h +++ b/include/pybind11/eigen/matrix.h @@ -19,7 +19,9 @@ PYBIND11_WARNING_PUSH PYBIND11_WARNING_DISABLE_MSVC(5054) // https://github.com/pybind/pybind11/pull/3741 // C5054: operator '&': deprecated between enumerations of different types +#if defined(__MINGW32__) PYBIND11_WARNING_DISABLE_GCC("-Wmaybe-uninitialized") +#endif #include #include diff --git a/include/pybind11/eigen/tensor.h b/include/pybind11/eigen/tensor.h index 0877da8953..de7dcba89d 100644 --- a/include/pybind11/eigen/tensor.h +++ b/include/pybind11/eigen/tensor.h @@ -17,7 +17,9 @@ static_assert(__GNUC__ > 5, "Eigen Tensor support in pybind11 requires GCC > 5.0 PYBIND11_WARNING_PUSH PYBIND11_WARNING_DISABLE_MSVC(4554) PYBIND11_WARNING_DISABLE_MSVC(4127) +#if defined(__MINGW32__) PYBIND11_WARNING_DISABLE_GCC("-Wmaybe-uninitialized") +#endif #include From f5cff4f25e613f67fd08d9ea95a6cdb599cce7f9 Mon Sep 17 00:00:00 2001 From: Henry Schreiner Date: Fri, 17 Feb 2023 14:13:29 -0800 Subject: [PATCH 37/43] fix: nicer stack level for warning (#4516) Signed-off-by: Henry Schreiner --- pybind11/setup_helpers.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pybind11/setup_helpers.py b/pybind11/setup_helpers.py index 4c14498f6f..b9b433b656 100644 --- a/pybind11/setup_helpers.py +++ b/pybind11/setup_helpers.py @@ -175,7 +175,7 @@ def cxx_std(self) -> int: def cxx_std(self, level: int) -> None: if self._cxx_level: warnings.warn( - "You cannot safely change the cxx_level after setting it!", stacklevel=1 + "You cannot safely change the cxx_level after setting it!", stacklevel=2 ) # MSVC 2015 Update 3 and later only have 14 (and later 17) modes, so From b596235fe0ddf7f935e3d21589dc57c15f9b0a0d Mon Sep 17 00:00:00 2001 From: xkszltl Date: Tue, 21 Feb 2023 06:58:37 +0800 Subject: [PATCH 38/43] Inconsistent comments between 2 templates of `unchecked()`. (#4519) This comment is unrelated to having T in tparam or not. Probably a typo. Copied the correct one here. Resolve https://github.com/pybind/pybind11/issues/4518 --- include/pybind11/numpy.h | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/include/pybind11/numpy.h b/include/pybind11/numpy.h index 8f072af267..854d6e87fa 100644 --- a/include/pybind11/numpy.h +++ b/include/pybind11/numpy.h @@ -1121,10 +1121,10 @@ class array_t : public array { /** * Returns a proxy object that provides const access to the array's data without bounds or - * dimensionality checking. Unlike `unchecked()`, this does not require that the underlying - * array have the `writable` flag. Use with care: the array must not be destroyed or reshaped - * for the duration of the returned object, and the caller must take care not to access invalid - * dimensions or dimension indices. + * dimensionality checking. Unlike `mutable_unchecked()`, this does not require that the + * underlying array have the `writable` flag. Use with care: the array must not be destroyed + * or reshaped for the duration of the returned object, and the caller must take care not to + * access invalid dimensions or dimension indices. */ template detail::unchecked_reference unchecked() const & { From 882cb76913c6db97a460a252a31b39a40cec135e Mon Sep 17 00:00:00 2001 From: Eli Schwartz Date: Thu, 23 Feb 2023 00:59:14 -0500 Subject: [PATCH 39/43] add --version option to pybind11-config (#4526) Without this, it's impossible to get feature parity between detection mechanisms. Both the pkg-config file and the cmake config set their versions, but the python probe script didn't provide an option for this. So you could print the compiler flags for using it, but you could not check what you got. --- pybind11/__main__.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/pybind11/__main__.py b/pybind11/__main__.py index e87b16beb4..180665c23c 100644 --- a/pybind11/__main__.py +++ b/pybind11/__main__.py @@ -4,6 +4,7 @@ import sys import sysconfig +from ._version import __version__ from .commands import get_cmake_dir, get_include, get_pkgconfig_dir @@ -25,6 +26,12 @@ def print_includes() -> None: def main() -> None: parser = argparse.ArgumentParser() + parser.add_argument( + "--version", + action="version", + version=__version__, + help="Print the version and exit.", + ) parser.add_argument( "--includes", action="store_true", From ec3f6e24ad2201c8bdb0d0f64d677eaff265025d Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 16 Mar 2023 18:18:44 +0000 Subject: [PATCH 40/43] chore(deps): bump pypa/gh-action-pypi-publish from 1.6.4 to 1.8.1 (#4576) --- .github/workflows/pip.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/pip.yml b/.github/workflows/pip.yml index 7c6fc67a3e..3d4d2c6548 100644 --- a/.github/workflows/pip.yml +++ b/.github/workflows/pip.yml @@ -98,13 +98,13 @@ jobs: - uses: actions/download-artifact@v3 - name: Publish standard package - uses: pypa/gh-action-pypi-publish@v1.6.4 + uses: pypa/gh-action-pypi-publish@v1.8.1 with: password: ${{ secrets.pypi_password }} - packages_dir: standard/ + packages-dir: standard/ - name: Publish global package - uses: pypa/gh-action-pypi-publish@v1.6.4 + uses: pypa/gh-action-pypi-publish@v1.8.1 with: password: ${{ secrets.pypi_password_global }} - packages_dir: global/ + packages-dir: global/ From 3f5a7e55c614a586d97cfe96d0cdf5bd9cf1bef6 Mon Sep 17 00:00:00 2001 From: Henry Schreiner Date: Thu, 16 Mar 2023 13:01:28 -0700 Subject: [PATCH 41/43] docs: changelog for 2.10.4 (#4532) * docs: changelog for 2.10.4 Signed-off-by: Henry Schreiner * Apply suggestions from code review * Update docs/changelog.rst --------- Signed-off-by: Henry Schreiner --- docs/changelog.rst | 36 ++++++++++++++++++++++++++++++++++++ 1 file changed, 36 insertions(+) diff --git a/docs/changelog.rst b/docs/changelog.rst index bb111c5f26..58cc409839 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -21,11 +21,47 @@ Changes: ``dec_ref()`` are now enabled by default again. `#4246 `_ +* ``py::initialize_interpreter()`` using ``PyConfig_InitPythonConfig()`` + instead of ``PyConfig_InitIsolatedConfig()``, to obtain complete + ``sys.path``. + `#4473 `_ + +* Cast errors now always include Python type information, even if + ``PYBIND11_DETAILED_ERROR_MESSAGES`` is not defined. This increases binary + sizes slightly (~1.5%) but the error messages are much more informative. + `#4463 `_ + + Build system improvements: * Update clang-tidy to 15 in CI. `#4387 `_ +* Moved the linting framework over to Ruff. + `#4483 `_ + +Version 2.10.4 (Mar 16, 2023) +---------------------------- + +Changes: + +* ``python3 -m pybind11`` gained a ``--version`` option (prints the version and + exits). + `#4526 `_ + +Bug Fixes: + +* Fix a warning when pydebug is enabled on Python 3.11. + `#4461 `_ + +* Ensure ``gil_scoped_release`` RAII is non-copyable. + `#4490 `_ + +* Ensure the tests dir does not show up with new versions of setuptools. + `#4510 `_ + +* Better stacklevel for a warning in setuptools helpers. + `#4516 `_ Version 2.10.3 (Jan 3, 2023) ---------------------------- From 5b0a6fc2017fcc176545afe3e09c9f9885283242 Mon Sep 17 00:00:00 2001 From: Henry Schreiner Date: Thu, 16 Mar 2023 17:03:10 -0400 Subject: [PATCH 42/43] chore: bump version to 3.10.4 Signed-off-by: Henry Schreiner --- include/pybind11/detail/common.h | 4 ++-- pybind11/_version.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/include/pybind11/detail/common.h b/include/pybind11/detail/common.h index cb3f5cf39f..c26df990d6 100644 --- a/include/pybind11/detail/common.h +++ b/include/pybind11/detail/common.h @@ -11,11 +11,11 @@ #define PYBIND11_VERSION_MAJOR 2 #define PYBIND11_VERSION_MINOR 10 -#define PYBIND11_VERSION_PATCH 3 +#define PYBIND11_VERSION_PATCH 4 // Similar to Python's convention: https://docs.python.org/3/c-api/apiabiversion.html // Additional convention: 0xD = dev -#define PYBIND11_VERSION_HEX 0x020A0300 +#define PYBIND11_VERSION_HEX 0x020A0400 // Define some generic pybind11 helper macros for warning management. // diff --git a/pybind11/_version.py b/pybind11/_version.py index 63078bbe67..a668258f47 100644 --- a/pybind11/_version.py +++ b/pybind11/_version.py @@ -8,5 +8,5 @@ def _to_int(s: str) -> Union[int, str]: return s -__version__ = "2.10.3" +__version__ = "2.10.4" version_info = tuple(_to_int(s) for s in __version__.split(".")) From fddf3bd3c341434ce0cd341b5bfc99790ae88f4c Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 27 Jun 2023 08:56:52 +0000 Subject: [PATCH 43/43] chore(deps): bump pypa/gh-action-pypi-publish from 1.8.1 to 1.8.7 Bumps [pypa/gh-action-pypi-publish](https://github.com/pypa/gh-action-pypi-publish) from 1.8.1 to 1.8.7. - [Release notes](https://github.com/pypa/gh-action-pypi-publish/releases) - [Commits](https://github.com/pypa/gh-action-pypi-publish/compare/v1.8.1...v1.8.7) --- updated-dependencies: - dependency-name: pypa/gh-action-pypi-publish dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- .github/workflows/pip.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/pip.yml b/.github/workflows/pip.yml index 3d4d2c6548..868e7ad913 100644 --- a/.github/workflows/pip.yml +++ b/.github/workflows/pip.yml @@ -98,13 +98,13 @@ jobs: - uses: actions/download-artifact@v3 - name: Publish standard package - uses: pypa/gh-action-pypi-publish@v1.8.1 + uses: pypa/gh-action-pypi-publish@v1.8.7 with: password: ${{ secrets.pypi_password }} packages-dir: standard/ - name: Publish global package - uses: pypa/gh-action-pypi-publish@v1.8.1 + uses: pypa/gh-action-pypi-publish@v1.8.7 with: password: ${{ secrets.pypi_password_global }} packages-dir: global/