diff --git a/.github/ci/spack-envs/clang6_nopy_ompi_h5_ad1_ad2_bp3_libcpp/spack.yaml b/.github/ci/spack-envs/clang6_py38_ompi_h5_ad1_ad2_bp3_libcpp/spack.yaml similarity index 100% rename from .github/ci/spack-envs/clang6_nopy_ompi_h5_ad1_ad2_bp3_libcpp/spack.yaml rename to .github/ci/spack-envs/clang6_py38_ompi_h5_ad1_ad2_bp3_libcpp/spack.yaml diff --git a/.github/workflows/linux.yml b/.github/workflows/linux.yml index 315e8683b6..39257fdeeb 100644 --- a/.github/workflows/linux.yml +++ b/.github/workflows/linux.yml @@ -40,18 +40,18 @@ jobs: cmake --build build --parallel 2 ctest --test-dir build --output-on-failure - clang6_nopy_ompi_h5_ad1_ad2_bp3_libcpp: + clang6_py38_ompi_h5_ad1_ad2_bp3_libcpp: runs-on: ubuntu-18.04 if: github.event.pull_request.draft == false steps: - uses: actions/checkout@v2 - name: Spack Cache uses: actions/cache@v2 - with: {path: /opt/spack, key: clang6_nopy_ompi_h5_ad1_ad2_bp3_libcpp } + with: {path: /opt/spack, key: clang6_py38_ompi_h5_ad1_ad2_bp3_libcpp } - name: Install run: | sudo apt-get update - sudo apt-get install clang-6.0 libc++-dev libc++abi-dev gfortran libopenmpi-dev python3 + sudo apt-get install clang-6.0 libc++-dev libc++abi-dev gfortran libopenmpi-dev python3 python3-numpy python3-mpi4py python3-pandas sudo .github/workflows/dependencies/install_spack - name: Build env: {CC: clang-6.0, CXX: clang++-6.0, CXXFLAGS: -stdlib=libc++ -Werror -Wno-deprecated-declarations -Wno-ignored-attributes -Wno-unused-const-variable} @@ -59,13 +59,13 @@ jobs: # false positive on src/auxiliary/Filesystem.cpp # [[maybe_unused]] MPI_Datatype const MPI_Types< unsigned >::value = MPI_UNSIGNED; run: | - eval $(spack env activate --sh .github/ci/spack-envs/clang6_nopy_ompi_h5_ad1_ad2_bp3_libcpp/) + eval $(spack env activate --sh .github/ci/spack-envs/clang6_py38_ompi_h5_ad1_ad2_bp3_libcpp/) spack install share/openPMD/download_samples.sh build chmod u-w build/samples/git-sample/*.h5 cmake -S . -B build \ - -DopenPMD_USE_PYTHON=OFF \ + -DopenPMD_USE_PYTHON=ON \ -DopenPMD_USE_MPI=ON \ -DopenPMD_USE_HDF5=ON \ -DopenPMD_USE_ADIOS1=ON \ diff --git a/include/openPMD/IO/ADIOS/CommonADIOS1IOHandler.hpp b/include/openPMD/IO/ADIOS/CommonADIOS1IOHandler.hpp index 74733781e9..ea89f033d6 100644 --- a/include/openPMD/IO/ADIOS/CommonADIOS1IOHandler.hpp +++ b/include/openPMD/IO/ADIOS/CommonADIOS1IOHandler.hpp @@ -103,7 +103,12 @@ class CommonADIOS1IOHandlerImpl : public AbstractIOHandlerImpl m_openWriteFileHandles; std::unordered_map, ADIOS_FILE *> m_openReadFileHandles; - std::unordered_map > + struct ScheduledRead + { + ADIOS_SELECTION *selection; + std::shared_ptr data; // needed to avoid early freeing + }; + std::unordered_map > m_scheduledReads; std::unordered_map > m_attributeWrites; diff --git a/include/openPMD/backend/Attribute.hpp b/include/openPMD/backend/Attribute.hpp index 87ffc8a0c0..f5c33d8feb 100644 --- a/include/openPMD/backend/Attribute.hpp +++ b/include/openPMD/backend/Attribute.hpp @@ -29,6 +29,7 @@ #include #include #include +#include #include #include #include @@ -229,3 +230,40 @@ U Attribute::get() const } } // namespace openPMD + +namespace std +{ +inline string to_string(openPMD::Attribute const &attr) +{ + return std::visit( + [](auto const &val) { + using Val_t = remove_cv_t >; + + std::stringstream os; + if constexpr ( + openPMD::auxiliary::IsVector_v || + openPMD::auxiliary::IsArray_v) + { + if (val.empty()) + { + os << "[]"; + } + else + { + os << "[" << val[0]; + for (size_t i = 1; i < val.size(); ++i) + { + os << ", " << val[i]; + } + os << "]"; + } + } + else + { + os << val; + } + return os.str(); + }, + attr.getResource()); +} +} // namespace std diff --git a/src/IO/ADIOS/ADIOS1IOHandler.cpp b/src/IO/ADIOS/ADIOS1IOHandler.cpp index 3c38b1adc6..3a6dc803d2 100644 --- a/src/IO/ADIOS/ADIOS1IOHandler.cpp +++ b/src/IO/ADIOS/ADIOS1IOHandler.cpp @@ -296,7 +296,7 @@ std::future ADIOS1IOHandlerImpl::flush() "dataset reading"); for (auto &sel : file.second) - adios_selection_delete(sel); + adios_selection_delete(sel.selection); } m_scheduledReads.clear(); diff --git a/src/IO/ADIOS/CommonADIOS1IOHandler.cpp b/src/IO/ADIOS/CommonADIOS1IOHandler.cpp index 345052ddfc..e50207a74d 100644 --- a/src/IO/ADIOS/CommonADIOS1IOHandler.cpp +++ b/src/IO/ADIOS/CommonADIOS1IOHandler.cpp @@ -728,7 +728,7 @@ void CommonADIOS1IOHandlerImpl::closeFile( "dataset reading"); for (auto &sel : scheduled->second) - adios_selection_delete(sel); + adios_selection_delete(sel.selection); m_scheduledReads.erase(scheduled); } close(handle_read->second); @@ -1183,7 +1183,7 @@ void CommonADIOS1IOHandlerImpl::readDataset( "[ADIOS1] Internal error: Failed to schedule ADIOS read during dataset " "reading"); - m_scheduledReads[f].push_back(sel); + m_scheduledReads[f].push_back({sel, parameters.data}); } template diff --git a/src/IO/ADIOS/ParallelADIOS1IOHandler.cpp b/src/IO/ADIOS/ParallelADIOS1IOHandler.cpp index 20a2d3936e..df6f817098 100644 --- a/src/IO/ADIOS/ParallelADIOS1IOHandler.cpp +++ b/src/IO/ADIOS/ParallelADIOS1IOHandler.cpp @@ -316,7 +316,7 @@ std::future ParallelADIOS1IOHandlerImpl::flush() "dataset reading"); for (auto &sel : file.second) - adios_selection_delete(sel); + adios_selection_delete(sel.selection); } m_scheduledReads.clear(); diff --git a/src/binding/python/Attributable.cpp b/src/binding/python/Attributable.cpp index 69708fc948..16d7519c65 100644 --- a/src/binding/python/Attributable.cpp +++ b/src/binding/python/Attributable.cpp @@ -462,6 +462,9 @@ void init_Attributable(py::module &m) "get_attribute", [](Attributable &attr, std::string const &key) { auto v = attr.getAttribute(key); + std::cout << "Attribute '" << key << "' has type: " << v.dtype + << std::endl + << " and value: " << std::to_string(v) << std::endl; return v.getResource(); // TODO instead of returning lists, return all arrays (ndim > 0) // as numpy arrays? diff --git a/src/binding/python/PatchRecordComponent.cpp b/src/binding/python/PatchRecordComponent.cpp index cecd57b252..7d47445e6a 100644 --- a/src/binding/python/PatchRecordComponent.cpp +++ b/src/binding/python/PatchRecordComponent.cpp @@ -104,8 +104,14 @@ void init_PatchRecordComponent(py::module &m) py::ssize_t numElements = 1; if (buf.ndim > 0) { + std::cout << "Buffer has dimensionality: " << buf.ndim + << std::endl; for (auto d = 0; d < buf.ndim; ++d) + { + std::cout << "Extent of dimensionality " << d << ": " + << buf.shape.at(d) << std::endl; numElements *= buf.shape.at(d); + } } // Numpy: Handling of arrays and scalars @@ -177,7 +183,8 @@ void init_PatchRecordComponent(py::module &m) { throw std::runtime_error( "store: " - "Only scalar values supported!"); + "Only scalar values supported! (found " + + std::to_string(numElements) + "values)"); } }, py::arg("idx"), diff --git a/src/binding/python/RecordComponent.cpp b/src/binding/python/RecordComponent.cpp index 2006f7cf6a..2ae7185e17 100644 --- a/src/binding/python/RecordComponent.cpp +++ b/src/binding/python/RecordComponent.cpp @@ -539,53 +539,51 @@ void load_chunk( } } + auto load_data = + [&r, &buffer, &buffer_info, &offset, &extent](auto cxxtype) { + using CXXType = decltype(cxxtype); + buffer.inc_ref(); + // buffer_info.inc_ref(); + void *data = buffer_info.ptr; + std::shared_ptr shared( + (CXXType *)data, [buffer](CXXType *) { buffer.dec_ref(); }); + r.loadChunk(std::move(shared), offset, extent); + }; + if (r.getDatatype() == Datatype::CHAR) - r.loadChunk(shareRaw((char *)buffer_info.ptr), offset, extent); + load_data((char)0); else if (r.getDatatype() == Datatype::UCHAR) - r.loadChunk( - shareRaw((unsigned char *)buffer_info.ptr), offset, extent); + load_data((unsigned char)0); else if (r.getDatatype() == Datatype::SHORT) - r.loadChunk(shareRaw((short *)buffer_info.ptr), offset, extent); + load_data((short)0); else if (r.getDatatype() == Datatype::INT) - r.loadChunk(shareRaw((int *)buffer_info.ptr), offset, extent); + load_data((int)0); else if (r.getDatatype() == Datatype::LONG) - r.loadChunk(shareRaw((long *)buffer_info.ptr), offset, extent); + load_data((long)0); else if (r.getDatatype() == Datatype::LONGLONG) - r.loadChunk( - shareRaw((long long *)buffer_info.ptr), offset, extent); + load_data((long long)0); else if (r.getDatatype() == Datatype::USHORT) - r.loadChunk( - shareRaw((unsigned short *)buffer_info.ptr), offset, extent); + load_data((unsigned short)0); else if (r.getDatatype() == Datatype::UINT) - r.loadChunk( - shareRaw((unsigned int *)buffer_info.ptr), offset, extent); + load_data((unsigned int)0); else if (r.getDatatype() == Datatype::ULONG) - r.loadChunk( - shareRaw((unsigned long *)buffer_info.ptr), offset, extent); + load_data((unsigned long)0); else if (r.getDatatype() == Datatype::ULONGLONG) - r.loadChunk( - shareRaw((unsigned long long *)buffer_info.ptr), offset, extent); + load_data((unsigned long long)0); else if (r.getDatatype() == Datatype::LONG_DOUBLE) - r.loadChunk( - shareRaw((long double *)buffer_info.ptr), offset, extent); + load_data((long double)0); else if (r.getDatatype() == Datatype::DOUBLE) - r.loadChunk( - shareRaw((double *)buffer_info.ptr), offset, extent); + load_data((double)0); else if (r.getDatatype() == Datatype::FLOAT) - r.loadChunk(shareRaw((float *)buffer_info.ptr), offset, extent); + load_data((float)0); else if (r.getDatatype() == Datatype::CLONG_DOUBLE) - r.loadChunk>( - shareRaw((std::complex *)buffer_info.ptr), - offset, - extent); + load_data((std::complex)0); else if (r.getDatatype() == Datatype::CDOUBLE) - r.loadChunk>( - shareRaw((std::complex *)buffer_info.ptr), offset, extent); + load_data((std::complex)0); else if (r.getDatatype() == Datatype::CFLOAT) - r.loadChunk>( - shareRaw((std::complex *)buffer_info.ptr), offset, extent); + load_data((std::complex)0); else if (r.getDatatype() == Datatype::BOOL) - r.loadChunk(shareRaw((bool *)buffer_info.ptr), offset, extent); + load_data((bool)0); else throw std::runtime_error( std::string("Datatype not known in 'loadChunk'!")); @@ -777,8 +775,14 @@ void init_RecordComponent(py::module &m) py::ssize_t numElements = 1; if (buf.ndim > 0) { + std::cout << "Buffer has dimensionality: " << buf.ndim + << std::endl; for (auto d = 0; d < buf.ndim; ++d) + { + std::cout << "Extent of dimensionality " << d << ": " + << buf.shape.at(d) << std::endl; numElements *= buf.shape.at(d); + } } // Numpy: Handling of arrays and scalars @@ -869,7 +873,8 @@ void init_RecordComponent(py::module &m) { throw std::runtime_error( "make_constant: " - "Only scalar values supported!"); + "Only scalar values supported! (found " + + std::to_string(numElements) + "values)"); } }, py::arg("value")) diff --git a/test/python/unittest/API/APITest.py b/test/python/unittest/API/APITest.py index 0876e2c14a..7705deba1c 100644 --- a/test/python/unittest/API/APITest.py +++ b/test/python/unittest/API/APITest.py @@ -1905,7 +1905,7 @@ def writeFromTemporary(self, ext): ) r_E_x = read.iterations[0].meshes["E"]["x"] - if read.backend == 'ADIOS2': + if read.backend == 'ADIOS2' or read.backend == 'ADIOS1': self.assertEqual(len(r_E_x.available_chunks()), 3) else: self.assertEqual(len(r_E_x.available_chunks()), 1)