CI fixes and documentation for Pybind v2.12.0 and Numpy 2.0 (#1637)

* CI fixes and documentation

* Use a venv also for the clang14 run

.github/workflows/linux.yml

@@ -128,16 +128,20 @@ jobs:
     - name: Install
       run: |
         sudo apt update
-        sudo apt install clang-14 cmake gfortran libhdf5-dev python3.11 python3.11-dev wget
-        wget https://bootstrap.pypa.io/get-pip.py
-        python3.11 get-pip.py
-        python3.11 -m pip install numpy pandas
+        sudo apt install clang-14 cmake gfortran libhdf5-dev python3.11 python3.11-dev wget python3.11-venv
         sudo .github/workflows/dependencies/install_spack
+        python3.11 -m venv /opt/python_venv
+        . /opt/python_venv/bin/activate
+        python -m pip install numpy pandas
     - name: Build
       env: {CC: clang-14, CXX: clang++-14, CXXFLAGS: -Werror}
       run: |
+        # Build Spack packages against system Python
+        # and activate the virtual environment just for the openPMD build.
+        # Spack does not play nice with venv.
         eval $(spack env activate --sh .github/ci/spack-envs/clang14_py311_nompi_h5_ad2/)
         spack install
+        . /opt/python_venv/bin/activate
 
         share/openPMD/download_samples.sh build
         cmake -S . -B build \
@@ -147,7 +151,7 @@ jobs:
           -DopenPMD_USE_ADIOS2=ON \
           -DopenPMD_USE_INVASIVE_TESTS=ON \
           -DCMAKE_VERBOSE_MAKEFILE=ON     \
-          -DPython_EXECUTABLE=$(which python3.11)
+          -DPython_EXECUTABLE="$(which python)"
         cmake --build build --parallel 2
         ctest --test-dir build --output-on-failure
 
@@ -271,17 +275,22 @@ jobs:
       run: |
         apk update
         apk add hdf5-dev
-        python3.10 -m pip install numpy
+        # Use a virtual environment in order to avoid compatibility issues
+        # between the various Python installations in this image.
+        python3.10 -m venv /opt/python_env
+        . /opt/python_env/bin/activate
+        python -m pip install numpy
     - name: Build
       env: {CXXFLAGS: -Werror}
       run: |
+        . /opt/python_env/bin/activate
         share/openPMD/download_samples.sh build
         cmake -S . -B build \
           -DopenPMD_USE_PYTHON=ON \
           -DopenPMD_USE_MPI=OFF   \
           -DopenPMD_USE_HDF5=ON   \
           -DopenPMD_USE_INVASIVE_TESTS=ON \
-          -DPython_EXECUTABLE=$(which python3.10)
+          -DPython_EXECUTABLE=$(which python)
         cmake --build build --parallel 2
         cd build
         ctest --output-on-failure

CHANGELOG.rst

@@ -17,7 +17,7 @@ Changes to "0.15.0"
 Features
 """"""""
 
-- pybind11: require version 2.11.1+ #1220 #1322
+- pybind11: require version 2.12.0+ #1220 #1322 #1637
 
 Bug Fixes
 """""""""

CMakeLists.txt

@@ -408,9 +408,9 @@ if(openPMD_USE_PYTHON STREQUAL AUTO)
         if(openPMD_USE_INTERNAL_PYBIND11)
             add_subdirectory("${openPMD_SOURCE_DIR}/share/openPMD/thirdParty/pybind11")
             set(openPMD_HAVE_PYTHON TRUE)
-            message(STATUS "pybind11: Using INTERNAL version 2.11.1")
+            message(STATUS "pybind11: Using INTERNAL version 2.12.0")
         else()
-            find_package(pybind11 2.11.1 CONFIG)
+            find_package(pybind11 2.12.0 CONFIG)
             if(pybind11_FOUND)
                 set(openPMD_HAVE_PYTHON TRUE)
                 message(STATUS "pybind11: Found version '${pybind11_VERSION}'")
@@ -426,9 +426,9 @@ elseif(openPMD_USE_PYTHON)
     if(openPMD_USE_INTERNAL_PYBIND11)
         add_subdirectory("${openPMD_SOURCE_DIR}/share/openPMD/thirdParty/pybind11")
         set(openPMD_HAVE_PYTHON TRUE)
-        message(STATUS "pybind11: Using INTERNAL version 2.11.1")
+        message(STATUS "pybind11: Using INTERNAL version 2.12.0")
     else()
-        find_package(pybind11 2.11.1 REQUIRED CONFIG)
+        find_package(pybind11 2.12.0 REQUIRED CONFIG)
         set(openPMD_HAVE_PYTHON TRUE)
         message(STATUS "pybind11: Found version '${pybind11_VERSION}'")
     endif()

NEWS.rst

@@ -13,7 +13,7 @@ Please transition to ADIOS2.
 For reading legacy ADIOS1 BP3 files, either use an older version of openPMD-api or the BP3 backend in ADIOS2.
 Note that ADIOS2 does not support compression in BP3 files.
 
-pybind11 2.11.1 is now the minimally supported version for Python support.
+pybind11 2.12.0 is now the minimally supported version for Python support.
 
 
 0.15.0

README.md

@@ -101,7 +101,7 @@ Required:
 
 Shipped internally in `share/openPMD/thirdParty/`:
 * [Catch2](https://github.com/catchorg/Catch2) 2.13.10+ ([BSL-1.0](https://github.com/catchorg/Catch2/blob/master/LICENSE.txt))
-* [pybind11](https://github.com/pybind/pybind11) 2.11.1+ ([new BSD](https://github.com/pybind/pybind11/blob/master/LICENSE))
+* [pybind11](https://github.com/pybind/pybind11) 2.12.0+ ([new BSD](https://github.com/pybind/pybind11/blob/master/LICENSE))
 * [NLohmann-JSON](https://github.com/nlohmann/json) 3.9.1+ ([MIT](https://github.com/nlohmann/json/blob/develop/LICENSE.MIT))
 * [toml11](https://github.com/ToruNiina/toml11) 3.7.1+ ([MIT](https://github.com/ToruNiina/toml11/blob/master/LICENSE))
 
@@ -116,7 +116,7 @@ while those can be built either with or without:
 Optional language bindings:
 * Python:
   * Python 3.8 - 3.12
-  * pybind11 2.11.1+
+  * pybind11 2.12.0+
   * numpy 1.15+
   * mpi4py 2.1+ (optional, for MPI)
   * pandas 1.0+ (optional, for dataframes)
@@ -267,7 +267,7 @@ The following options allow to switch to external installs:
 | CMake Option                    | Values     | Library       | Version  |
 |---------------------------------|------------|---------------|----------|
 | `openPMD_USE_INTERNAL_CATCH`    | **ON**/OFF | Catch2        | 2.13.10+ |
-| `openPMD_USE_INTERNAL_PYBIND11` | **ON**/OFF | pybind11      |  2.11.1+ |
+| `openPMD_USE_INTERNAL_PYBIND11` | **ON**/OFF | pybind11      |  2.12.0+ |
 | `openPMD_USE_INTERNAL_JSON`     | **ON**/OFF | NLohmann-JSON |   3.9.1+ |
 | `openPMD_USE_INTERNAL_TOML11`   | **ON**/OFF | toml11        |   3.7.1+ |
 

docs/source/dev/buildoptions.rst

@@ -67,7 +67,7 @@ The following options allow to switch to external installs of dependencies:
 CMake Option                      Values      Installs Library       Version
 ================================= =========== ======== ============= ========
 ``openPMD_USE_INTERNAL_CATCH``    **ON**/OFF  No       Catch2        2.13.10+
-``openPMD_USE_INTERNAL_PYBIND11`` **ON**/OFF  No       pybind11       2.11.1+
+``openPMD_USE_INTERNAL_PYBIND11`` **ON**/OFF  No       pybind11       2.12.0+
 ``openPMD_USE_INTERNAL_JSON``     **ON**/OFF  No       NLohmann-JSON   3.9.1+
 ``openPMD_USE_INTERNAL_TOML11``   **ON**/OFF  No       toml11          3.7.1+
 ================================= =========== ======== ============= ========

docs/source/dev/dependencies.rst

@@ -18,7 +18,7 @@ Shipped internally
 The following libraries are shipped internally in ``share/openPMD/thirdParty/`` for convenience:
 
 * `Catch2 <https://github.com/catchorg/Catch2>`_ 2.13.10+ (`BSL-1.0 <https://github.com/catchorg/Catch2/blob/master/LICENSE.txt>`__)
-* `pybind11 <https://github.com/pybind/pybind11>`_ 2.11.1+ (`new BSD <https://github.com/pybind/pybind11/blob/master/LICENSE>`_)
+* `pybind11 <https://github.com/pybind/pybind11>`_ 2.12.0+ (`new BSD <https://github.com/pybind/pybind11/blob/master/LICENSE>`_)
 * `NLohmann-JSON <https://github.com/nlohmann/json>`_ 3.9.1+ (`MIT <https://github.com/nlohmann/json/blob/develop/LICENSE.MIT>`_)
 * `toml11 <https://github.com/ToruNiina/toml11>`_ 3.7.1+ (`MIT <https://github.com/ToruNiina/toml11/blob/master/LICENSE>`__)
 
@@ -39,7 +39,7 @@ Optional: language bindings
 * Python:
 
   * Python 3.8 - 3.12
-  * pybind11 2.11.1+
+  * pybind11 2.12.0+
   * numpy 1.15+
   * mpi4py 2.1+ (optional, for MPI)
   * pandas 1.0+ (optional, for dataframes)

pyproject.toml

@@ -4,6 +4,6 @@ requires = [
     "wheel",
     "cmake>=3.15.0,<4.0.0",
     "packaging>=23",
-    "pybind11>=2.11.1,<3.0.0"
+    "pybind11>=2.12.0,<3.0.0"
 ]
 build-backend = "setuptools.build_meta"