Merge remote-tracking branch 'mainline/dev' into add_cmake_fetchConte…

…nt_new

.circleci/config.yml

@@ -19,6 +19,7 @@ jobs:
             sudo .github/workflows/dependencies/install_spack
             python3 -m pip install -U pip
             python3 -m pip install -U packaging setuptools wheel
+            python3 -m pip install -U six
             python3 -m pip install -U numpy
             python3 -m pip install -U mpi4py
             python3 -m pip install -U pandas

.github/workflows/clang-format/clang-format.sh

@@ -2,11 +2,11 @@
 
 if (( $# > 0 )); then
     # received arguments, format those files
-    clang-format-13 -i "$@"
+    clang-format-18 -i "$@"
 else
     # received no arguments, find files on our own
     find include/ src/ test/ examples/ \
             -regextype egrep \
             -type f -regex '.*\.(hpp|cpp|hpp\.in)$' \
-        | xargs clang-format-13 -i
+        | xargs clang-format-18 -i
 fi

.github/workflows/linux.yml

@@ -128,26 +128,38 @@ jobs:
     - name: Install
       run: |
         sudo apt update
-        sudo apt install clang-14 cmake gfortran libhdf5-dev python3.11 python3.11-dev wget
-        wget https://bootstrap.pypa.io/get-pip.py
-        python3.11 get-pip.py
-        python3.11 -m pip install numpy pandas
+        sudo apt install clang-14 cmake gfortran libhdf5-dev python3.11 python3.11-dev wget python3.11-venv
         sudo .github/workflows/dependencies/install_spack
+        python3.11 -m venv /opt/python_venv
+        . /opt/python_venv/bin/activate
+        python -m pip install numpy pandas
+        git clone -b v4.0.3 https://github.com/ToruNiina/toml11
+        cmake -S toml11 -B build_toml11               \
+          -DCMAKE_INSTALL_PREFIX=toml11_install \
+          -DCMAKE_CXX_STANDARD_REQUIRED=OFF           \
+          -DCMAKE_CXX_STANDARD=11
+        cmake --build build_toml11 -j 2 --target install
     - name: Build
       env: {CC: clang-14, CXX: clang++-14, CXXFLAGS: -Werror}
       run: |
+        # Build Spack packages against system Python
+        # and activate the virtual environment just for the openPMD build.
+        # Spack does not play nice with venv.
         eval $(spack env activate --sh .github/ci/spack-envs/clang14_py311_nompi_h5_ad2/)
         spack install
+        . /opt/python_venv/bin/activate
 
         share/openPMD/download_samples.sh build
+        export CMAKE_PREFIX_PATH="$(realpath toml11_install):$CMAKE_PREFIX_PATH"
         cmake -S . -B build \
           -DopenPMD_USE_PYTHON=ON \
           -DopenPMD_USE_MPI=OFF   \
           -DopenPMD_USE_HDF5=ON   \
           -DopenPMD_USE_ADIOS2=ON \
-          -DopenPMD_USE_INVASIVE_TESTS=ON \
-          -DCMAKE_VERBOSE_MAKEFILE=ON     \
-          -DPython_EXECUTABLE=$(which python3.11)
+          -DopenPMD_USE_INVASIVE_TESTS=ON   \
+          -DopenPMD_USE_INTERNAL_TOML11=OFF \
+          -DCMAKE_VERBOSE_MAKEFILE=ON       \
+          -DPython_EXECUTABLE="$(which python)"
         cmake --build build --parallel 2
         ctest --test-dir build --output-on-failure
 
@@ -271,17 +283,22 @@ jobs:
       run: |
         apk update
         apk add hdf5-dev
-        python3.10 -m pip install numpy
+        # Use a virtual environment in order to avoid compatibility issues
+        # between the various Python installations in this image.
+        python3.10 -m venv /opt/python_env
+        . /opt/python_env/bin/activate
+        python -m pip install numpy
     - name: Build
       env: {CXXFLAGS: -Werror}
       run: |
+        . /opt/python_env/bin/activate
         share/openPMD/download_samples.sh build
         cmake -S . -B build \
           -DopenPMD_USE_PYTHON=ON \
           -DopenPMD_USE_MPI=OFF   \
           -DopenPMD_USE_HDF5=ON   \
           -DopenPMD_USE_INVASIVE_TESTS=ON \
-          -DPython_EXECUTABLE=$(which python3.10)
+          -DPython_EXECUTABLE=$(which python)
         cmake --build build --parallel 2
         cd build
         ctest --output-on-failure
@@ -312,11 +329,14 @@ jobs:
 
         share/openPMD/download_samples.sh build
         cmake -S . -B build \
+          -DCMAKE_CXX_FLAGS="-Wno-error=stringop-overread" \
+          -DCMAKE_C_FLAGS="-Wno-error=stringop-overread" \
           -DopenPMD_USE_PYTHON=ON \
           -DopenPMD_USE_MPI=ON    \
           -DopenPMD_USE_HDF5=ON   \
           -DopenPMD_USE_ADIOS2=ON \
-          -DopenPMD_USE_INVASIVE_TESTS=ON
+          -DopenPMD_USE_INVASIVE_TESTS=ON \
+          -DMPIEXEC_EXECUTABLE=".github/workflows/mpirun_workaround.sh"
         cmake --build build --parallel 2
         cd build
         ctest --output-on-failure

.github/workflows/macos.yml

@@ -12,7 +12,7 @@ jobs:
 #  appleclang10_py37_h5_ad2_libcpp
 #  appleclang11_nopy_nompi_h5_ad2
 
-  appleclang14_py_mpi_h5_ad2:
+  appleclang15_py_mpi_h5_ad2:
     runs-on: macos-latest
     if: github.event.pull_request.draft == false
     steps:
@@ -45,8 +45,8 @@ jobs:
         cmake --build build --parallel 3
         ctest --test-dir build --verbose
 
-  appleclang13_py:
-    runs-on: macos-11
+  appleclang14_py:
+    runs-on: macos-12
     if: github.event.pull_request.draft == false
     steps:
     - uses: actions/checkout@v3

.github/workflows/mpirun_workaround.sh

@@ -14,7 +14,8 @@
 # This script provides a workaround by putting the called sub-command into
 # a script in a temporary file.
 
-mpiexec -n 1 ls --all \
+ls="$(which ls)"
+mpiexec "$ls" -m \
     && echo "MPIRUN WORKING AGAIN, PLEASE REMOVE WORKAROUND" >&2 \
     && exit 1 \
     || true

.github/workflows/source.yml

@@ -36,7 +36,7 @@ jobs:
     runs-on: ubuntu-22.04
     steps:
     - uses: actions/checkout@v3
-    - uses: s-weigand/[email protected].1
+    - uses: s-weigand/[email protected].2
       with:
         update-conda: true
         conda-channels: conda-forge

.pre-commit-config.yaml

@@ -18,7 +18,7 @@ exclude: '^share/openPMD/thirdParty'
 # See https://pre-commit.com/hooks.html for more hooks
 repos:
 -   repo: https://github.com/pre-commit/pre-commit-hooks
-    rev: v4.5.0
+    rev: v4.6.0
     hooks:
     - id: trailing-whitespace
       args: [--markdown-linebreak-ext=md]
@@ -49,7 +49,7 @@ repos:
 
 # Changes tabs to spaces
 - repo: https://github.com/Lucas-C/pre-commit-hooks
-  rev: v1.5.4
+  rev: v1.5.5
   hooks:
   - id: remove-tabs
 
@@ -66,7 +66,7 @@ repos:
 # clang-format v13
 #   to run manually, use .github/workflows/clang-format/clang-format.sh
 - repo: https://github.com/pre-commit/mirrors-clang-format
-  rev: v17.0.6
+  rev: v18.1.5
   hooks:
   - id: clang-format
     # By default, the clang-format hook configures:

CHANGELOG.rst

@@ -17,7 +17,7 @@ Changes to "0.15.0"
 Features
 """"""""
 
-- pybind11: require version 2.11.1+ #1220 #1322
+- pybind11: require version 2.12.0+ #1220 #1322 #1637
 
 Bug Fixes
 """""""""

CMakeLists.txt

@@ -262,7 +262,16 @@ target_link_libraries(openPMD::thirdparty::nlohmann_json
     INTERFACE nlohmann_json::nlohmann_json)
 
 # external library: toml11
-find_package(toml11 CONFIG)
+# toml11 4.0 was a breaking change. This is reflected in the library's CMake
+# logic: version 4.0 is not accepted by a call to find_package(toml11 3.7).
+# Since we support both incompatible versions, we use two find_package()
+# calls. Search for version 4 first in order to prefer that
+# in (the unlikely) case that both versions are installed.
+find_package(toml11 4.0 CONFIG QUIET)
+if(NOT toml11_FOUND)
+    find_package(toml11 3.7.1 CONFIG REQUIRED)
+endif()
+message(STATUS "toml11: Found version '${toml11_VERSION}'")
 if(NOT toml11_FOUND)
     message(STATUS "Fetching toml11 from https://github.com/ToruNiina/toml11")
     if(CMAKE_VERSION VERSION_GREATER_EQUAL "3.23")
@@ -275,7 +284,7 @@ if(NOT toml11_FOUND)
       GIT_REPOSITORY https://github.com/ToruNiina/toml11
       # Migrate to the latest commit to remove CMake Warning which is not yet
       # available in any official release.
-      GIT_TAG v3.7.1
+      GIT_TAG v4.2.0
       "${fetchcontent_args}"
     )
     unset(fetchcontent_args)
@@ -405,7 +414,7 @@ else()
     set(openPMD_HAVE_PYTHON FALSE)
 endif()
 if(Python_FOUND)
-    find_package(pybind11 CONFIG)
+    find_package(pybind11 2.12.0 CONFIG)
     if(NOT pybind11_FOUND)
         message(STATUS "Fetching PyBind11 from https://github.com/pybind/pybind11")
         if(CMAKE_VERSION VERSION_GREATER_EQUAL "3.23")
@@ -416,7 +425,7 @@ if(Python_FOUND)
         FetchContent_Declare(
             pybind11
             GIT_REPOSITORY https://github.com/pybind/pybind11.git
-            GIT_TAG v2.11.1
+            GIT_TAG v2.12.0
             "${fetchcontent_args}"
         )
         unset(fetchcontent_args)
@@ -450,6 +459,7 @@ set(CORE_SOURCE
         src/auxiliary/Date.cpp
         src/auxiliary/Filesystem.cpp
         src/auxiliary/JSON.cpp
+        src/auxiliary/Mpi.cpp
         src/backend/Attributable.cpp
         src/backend/BaseRecordComponent.cpp
         src/backend/MeshRecordComponent.cpp
@@ -472,6 +482,7 @@ set(IO_SOURCE
         src/IO/JSON/JSONIOHandlerImpl.cpp
         src/IO/JSON/JSONFilePosition.cpp
         src/IO/ADIOS/ADIOS2IOHandler.cpp
+        src/IO/ADIOS/ADIOS2File.cpp
         src/IO/ADIOS/ADIOS2Auxiliary.cpp
         src/IO/InvalidatableFile.cpp)
 
@@ -1368,7 +1379,7 @@ if(openPMD_BUILD_TESTING)
                             --outfile                                              \
                                 ../samples/git-sample/thetaMode/data_%T.bp &&      \
                                                                                    \
-                        ${Python_EXECUTABLE}                                       \
+                        ${MPI_TEST_EXE} ${Python_EXECUTABLE}                       \
                             ${openPMD_RUNTIME_OUTPUT_DIRECTORY}/openpmd-pipe       \
                             --infile ../samples/git-sample/thetaMode/data_%T.bp    \
                             --outfile ../samples/git-sample/thetaMode/data%T.json  \

NEWS.rst

@@ -13,7 +13,7 @@ Please transition to ADIOS2.
 For reading legacy ADIOS1 BP3 files, either use an older version of openPMD-api or the BP3 backend in ADIOS2.
 Note that ADIOS2 does not support compression in BP3 files.
 
-pybind11 2.11.1 is now the minimally supported version for Python support.
+pybind11 2.12.0 is now the minimally supported version for Python support.
 
 
 0.15.0

README.md

@@ -101,7 +101,7 @@ Required:
 
 Shipped internally in `share/openPMD/thirdParty/`:
 * [Catch2](https://github.com/catchorg/Catch2) 2.13.10+ ([BSL-1.0](https://github.com/catchorg/Catch2/blob/master/LICENSE.txt))
-* [pybind11](https://github.com/pybind/pybind11) 2.11.1+ ([new BSD](https://github.com/pybind/pybind11/blob/master/LICENSE))
+* [pybind11](https://github.com/pybind/pybind11) 2.12.0+ ([new BSD](https://github.com/pybind/pybind11/blob/master/LICENSE))
 * [NLohmann-JSON](https://github.com/nlohmann/json) 3.9.1+ ([MIT](https://github.com/nlohmann/json/blob/develop/LICENSE.MIT))
 * [toml11](https://github.com/ToruNiina/toml11) 3.7.1+ ([MIT](https://github.com/ToruNiina/toml11/blob/master/LICENSE))
 
@@ -116,7 +116,7 @@ while those can be built either with or without:
 Optional language bindings:
 * Python:
   * Python 3.8 - 3.12
-  * pybind11 2.11.1+
+  * pybind11 2.12.0+
   * numpy 1.15+
   * mpi4py 2.1+ (optional, for MPI)
   * pandas 1.0+ (optional, for dataframes)
@@ -261,10 +261,11 @@ CMake controls options with prefixed `-D`, e.g. `-DopenPMD_USE_MPI=OFF`:
 <sup>1</sup> *e.g. changes C++ visibility keywords, breaks MSVC*
 <sup>2</sup> *this includes most pre-/post-condition checks, disabling without specific cause is highly discouraged*
 
+
 Additionally, the following libraries are downloaded via [FetchContent](https://cmake.org/cmake/help/latest/module/FetchContent.html)
 during the configuration of the project or, if the corresponding `<PACKAGENAME>_ROOT` variable is provided, can be provided externally:
 * [Catch2](https://github.com/catchorg/Catch2) (2.13.10+)
-* [PyBind11](https://github.com/pybind/pybind11) (2.11.1+)
+* [PyBind11](https://github.com/pybind/pybind11) (2.12.0+)
 * [NLohmann-JSON](https://github.com/nlohmann/json) (3.9.1+)
 * [toml11](https://github.com/ToruNiina/toml11) (3.7.1+)
 

docs/source/analysis/pandas.rst

@@ -45,16 +45,7 @@ One can also combine all iterations in a single dataframe like this:
 
 .. code-block:: python
 
-   import pandas as pd
-
-   df = pd.concat(
-        (
-            s.iterations[i].particles["electrons"].to_df().assign(iteration=i)
-            for i in s.iterations
-        ),
-        axis=0,
-        ignore_index=True,
-   )
+   df = s.to_df("electrons")
 
    # like before but with a new column "iteration" and all particles
    print(df)

docs/source/analysis/rapids.rst

@@ -51,14 +51,7 @@ One can also combine all iterations in a single dataframe like this:
 
 .. code-block:: python
 
-   cdf = cudf.concat(
-        (
-            cudf.from_pandas(s.iterations[i].particles["electrons"].to_df().assign(iteration=i))
-            for i in s.iterations
-        ),
-        axis=0,
-        ignore_index=True,
-   )
+   cdf = s.to_cudf("electrons")
 
    # like before but with a new column "iteration" and all particles
    print(cdf)

docs/source/backends/adios2.rst

@@ -48,6 +48,10 @@ Exceptions to this are the BP3 and SST engines which require their endings ``.bp
 
 For file engines, we currently leverage the default ADIOS2 transport parameters, i.e. ``POSIX`` on Unix systems and ``FStream`` on Windows.
 
+.. tip::
+
+   Use the ``adios2.engine.treat_unsupported_engine_as`` :ref:`JSON/TOML parameter <backendconfig-adios2>` for experimentally interacting with an unsupported ADIOS2 engine.
+
 Steps
 -----
 
@@ -81,6 +85,7 @@ environment variable                  default    description
 ``OPENPMD_ADIOS2_HAVE_METADATA_FILE`` ``1``      Online creation of the adios journal file (``1``: yes, ``0``: no).
 ``OPENPMD_ADIOS2_NUM_SUBSTREAMS``     ``0``      Number of files to be created, 0 indicates maximum number possible.
 ``OPENPMD_ADIOS2_ENGINE``             ``File``   `ADIOS2 engine <https://adios2.readthedocs.io/en/latest/engines/engines.html>`_
+``OPENPMD_ADIOS2_PRETEND_ENGINE``     *empty*    Pretend that an (unknown) ADIOS2 engine is in fact another one (also see the ``adios2.pretend_engine`` :ref:`parameter <backendconfig-adios2>`).
 ``OPENPMD2_ADIOS2_USE_GROUP_TABLE``   ``0``      Use group table (see below)
 ``OPENPMD_ADIOS2_STATS_LEVEL``        ``0``      whether to generate statistics for variables in ADIOS2. (``1``: yes, ``0``: no).
 ``OPENPMD_ADIOS2_ASYNC_WRITE``        ``0``      ADIOS2 BP5 engine: 1 means setting "AsyncWrite" in ADIOS2 to "on". Flushes will go to the buffer by default (see ``preferred_flush_target``).

docs/source/backends/hdf5.rst

@@ -15,6 +15,15 @@ I/O Method
 
 HDF5 internally either writes serially, via ``POSIX`` on Unix systems, or parallel to a single logical file via MPI-I/O.
 
+Virtual File Drivers
+********************
+
+Rudimentary support for HDF5 VFDs (`virtual file driver <https://www.hdfgroup.org/wp-content/uploads/2021/10/HDF5-VFD-Plugins-HUG.pdf>`_) is available (currently only the *subfiling* VFD).
+Note that the subfiling VFD needs to be enabled explicitly when configuring HDF5 and threaded MPI must be used.
+
+Virtual file drivers are configured via JSON/TOML.
+Refer to the page on :ref:`JSON/TOML configuration <backendconfig-hdf5>` for further details.
+
 
 Backend-Specific Controls
 -------------------------
@@ -45,6 +54,10 @@ Although we choose the default to be non-collective (independent) for ease of us
 For independent parallel I/O, potentially prefer using a modern version of the MPICH implementation (especially, use ROMIO instead of OpenMPI's ompio implementation).
 Please refer to the `HDF5 manual, function H5Pset_dxpl_mpio <https://support.hdfgroup.org/HDF5/doc/RM/H5P/H5Pset_dxpl_mpio.htm>`_ for more details.
 
+.. tip::
+
+  Instead of using an environment variable, independent/collective data transfer can also be configured at the API level via :ref:`JSON/TOML <backendconfig-hdf5>`.
+
 ``OPENPMD_HDF5_ALIGNMENT``: this sets the alignment in Bytes for writes via the ``H5Pset_alignment`` function.
 According to the `HDF5 documentation <https://support.hdfgroup.org/HDF5/doc/RM/H5P/H5Pset_alignment.htm>`_:
 *For MPI IO and other parallel systems, choose an alignment which is a multiple of the disk block size.*
@@ -56,6 +69,7 @@ Any file object greater than or equal in size to threshold bytes will be aligned
 
 ``OPENPMD_HDF5_CHUNKS``: this sets defaults for data chunking via `H5Pset_chunk <https://support.hdfgroup.org/HDF5/doc/RM/H5P/H5Pset_chunk.htm>`__.
 Chunking generally improves performance and only needs to be disabled in corner-cases, e.g. when heavily relying on independent, parallel I/O that non-collectively declares data records.
+The chunk size can alternatively (or additionally) be specified explicitly per dataset, by specifying a dataset-specific chunk size in the JSON/TOML configuration of ``resetDataset()``/``reset_dataset()``.
 
 ``OPENPMD_HDF5_COLLECTIVE_METADATA``: this is an option to enable collective MPI calls for HDF5 metadata operations via `H5Pset_all_coll_metadata_ops <https://support.hdfgroup.org/HDF5/doc/RM/RM_H5P.html#Property-SetAllCollMetadataOps>`__ and `H5Pset_coll_metadata_write <https://support.hdfgroup.org/HDF5/doc/RM/RM_H5P.html#Property-SetCollMetadataWrite>`__.
 By default, this optimization is enabled as it has proven to provide performance improvements.