From 0332345e328a5340f5f9dd312e777559bc55fb9f Mon Sep 17 00:00:00 2001 From: Pete Peterson Date: Fri, 27 Dec 2024 13:23:21 -0500 Subject: [PATCH] Add ability to read portions of the data --- Framework/Nexus/inc/MantidNexus/H5Util.h | 3 + Framework/Nexus/src/H5Util.cpp | 73 +++++++++++++++++++++--- Framework/Nexus/test/H5UtilTest.h | 59 ++++++++++++++++++- 3 files changed, 127 insertions(+), 8 deletions(-) diff --git a/Framework/Nexus/inc/MantidNexus/H5Util.h b/Framework/Nexus/inc/MantidNexus/H5Util.h index b3eca15e624db..8f2d52a368127 100644 --- a/Framework/Nexus/inc/MantidNexus/H5Util.h +++ b/Framework/Nexus/inc/MantidNexus/H5Util.h @@ -95,6 +95,9 @@ template std::vector readArray1DCoerce(const H5::Group &gr template void readArray1DCoerce(const H5::DataSet &dataset, std::vector &output); +template +void readArray1DCoerce(const H5::DataSet &dataset, std::vector &output, const size_t length, const size_t offset); + /// Test if a group already exists within an HDF5 file or parent group. MANTID_NEXUS_DLL bool groupExists(H5::H5Object &h5, const std::string &groupPath); diff --git a/Framework/Nexus/src/H5Util.cpp b/Framework/Nexus/src/H5Util.cpp index 61074c54bd51c..fabbb31af462b 100644 --- a/Framework/Nexus/src/H5Util.cpp +++ b/Framework/Nexus/src/H5Util.cpp @@ -6,7 +6,6 @@ // SPDX - License - Identifier: GPL - 3.0 + #include "MantidNexus/H5Util.h" #include "MantidAPI/LogManager.h" -#include "MantidKernel/System.h" #include #include @@ -282,20 +281,25 @@ template std::vector readArray1DCoerce(const H5::Group &gr namespace { template -void convertingRead(const DataSet &dataset, const DataType &dataType, std::vector &output) { - DataSpace dataSpace = dataset.getSpace(); - - std::vector temp(dataSpace.getSelectNpoints()); - dataset.read(temp.data(), dataType, dataSpace); +void convertingRead(const DataSet &dataset, const DataType &dataType, std::vector &output, + const DataSpace &memspace, const DataSpace &filespace) { + std::vector temp(filespace.getSelectNpoints()); + dataset.read(temp.data(), dataType, memspace, filespace); output.resize(temp.size()); std::transform(temp.begin(), temp.end(), output.begin(), - [](const InputNumT a) { // lambda + [](const auto &a) { // lambda return boost::numeric_cast(a); }); } +template +void convertingRead(const DataSet &dataset, const DataType &dataType, std::vector &output) { + const DataSpace dataSpace = dataset.getSpace(); + convertingRead(dataset, dataType, output, dataSpace, dataSpace); +} + template std::vector convertingNumArrayAttributeRead(Attribute &attribute, const DataType &dataType) { DataSpace dataSpace = attribute.getSpace(); @@ -404,6 +408,48 @@ template void readArray1DCoerce(const DataSet &dataset, std::vec } } +template +void readArray1DCoerce(const H5::DataSet &dataset, std::vector &output, const size_t length, + const size_t offset) { + DataSpace filespace = dataset.getSpace(); + + const auto length_actual = static_cast(filespace.getSelectNpoints()); + if ((offset == 0) && (length_actual <= length)) { + // use standard approach + readArray1DCoerce(dataset, output); + } else { + // set extent and offset in DataSpace + hsize_t rankedoffset[1] = {static_cast(offset)}; + hsize_t rankedextent[1] = { + static_cast(std::min(length, length_actual - offset))}; // don't read past the end + filespace.selectHyperslab(H5S_SELECT_SET, rankedextent, rankedoffset); + + // size of thing being read out + DataSpace memspace(1, rankedextent); + + const DataType dataType = dataset.getDataType(); + if (getType() == dataType) { // no conversion necessary + output.resize(static_cast(filespace.getSelectNpoints())); + dataset.read(output.data(), dataType, memspace, filespace); + } else if (PredType::NATIVE_INT32 == dataType) { + convertingRead(dataset, dataType, output, memspace, filespace); + } else if (PredType::NATIVE_UINT32 == dataType) { + convertingRead(dataset, dataType, output, memspace, filespace); + } else if (PredType::NATIVE_INT64 == dataType) { + convertingRead(dataset, dataType, output, memspace, filespace); + } else if (PredType::NATIVE_UINT64 == dataType) { + convertingRead(dataset, dataType, output, memspace, filespace); + } else if (PredType::NATIVE_FLOAT == dataType) { + convertingRead(dataset, dataType, output, memspace, filespace); + } else if (PredType::NATIVE_DOUBLE == dataType) { + convertingRead(dataset, dataType, output, memspace, filespace); + } else { + // not a supported type + throw DataTypeIException(); + } + } +} + /// Test if a group exists in an HDF5 file or parent group. bool groupExists(H5::H5Object &h5, const std::string &groupPath) { bool status = true; @@ -600,4 +646,17 @@ template MANTID_NEXUS_DLL void readArray1DCoerce(const DataSet &dataset, std::ve template MANTID_NEXUS_DLL void readArray1DCoerce(const DataSet &dataset, std::vector &output); template MANTID_NEXUS_DLL void readArray1DCoerce(const DataSet &dataset, std::vector &output); template MANTID_NEXUS_DLL void readArray1DCoerce(const DataSet &dataset, std::vector &output); + +template MANTID_NEXUS_DLL void readArray1DCoerce(const DataSet &dataset, std::vector &output, + const size_t length, const size_t offset); +template MANTID_NEXUS_DLL void readArray1DCoerce(const DataSet &dataset, std::vector &output, + const size_t length, const size_t offset); +template MANTID_NEXUS_DLL void readArray1DCoerce(const DataSet &dataset, std::vector &output, + const size_t length, const size_t offset); +template MANTID_NEXUS_DLL void readArray1DCoerce(const DataSet &dataset, std::vector &output, + const size_t length, const size_t offset); +template MANTID_NEXUS_DLL void readArray1DCoerce(const DataSet &dataset, std::vector &output, + const size_t length, const size_t offset); +template MANTID_NEXUS_DLL void readArray1DCoerce(const DataSet &dataset, std::vector &output, + const size_t length, const size_t offset); } // namespace Mantid::NeXus::H5Util diff --git a/Framework/Nexus/test/H5UtilTest.h b/Framework/Nexus/test/H5UtilTest.h index 6228582842788..fe06916690e33 100644 --- a/Framework/Nexus/test/H5UtilTest.h +++ b/Framework/Nexus/test/H5UtilTest.h @@ -135,7 +135,7 @@ class H5UtilTest : public CxxTest::TestSuite { TS_ASSERT(std::filesystem::exists(FILENAME)); - { // read tests + { // ---------- simple read tests H5File file(FILENAME, H5F_ACC_RDONLY); auto group = file.openGroup(GRP_NAME); @@ -151,6 +151,63 @@ class H5UtilTest : public CxxTest::TestSuite { const boost::numeric::positive_overflow &); TS_ASSERT_THROWS_NOTHING(H5Util::readArray1DCoerce(group, "array1d_int32")); + // ---------- slicing read tests + auto dataSetFloat = group.openDataSet("array1d_float"); + auto dataSetDouble = group.openDataSet("array1d_double"); + + std::vector output; + // full dataset + output.clear(); + H5Util::readArray1DCoerce(dataSetFloat, output, array1d_double.size(), 0); + TS_ASSERT_EQUALS(output, array1d_double); // whole thing w/ coercion + output.clear(); + H5Util::readArray1DCoerce(dataSetDouble, output, array1d_double.size(), 0); + TS_ASSERT_EQUALS(output, array1d_double); // whole thing w/o coercion + output.clear(); + H5Util::readArray1DCoerce(dataSetFloat, output, array1d_double.size() + 1, 0); + TS_ASSERT_EQUALS(output, array1d_double); // more than the whole thing w/ coercion + output.clear(); + H5Util::readArray1DCoerce(dataSetDouble, output, array1d_double.size() + 1, 0); + TS_ASSERT_EQUALS(output, array1d_double); // more than the whole thing w/o coercion + + { // partial dataset from front 1->end + const std::vector expected({1, 2, 3, 4}); + output.clear(); + H5Util::readArray1DCoerce(dataSetFloat, output, array1d_double.size() - 1, 1); + TS_ASSERT_EQUALS(output, expected); // w/ coercion + output.clear(); + H5Util::readArray1DCoerce(dataSetDouble, output, array1d_double.size() - 1, 1); + TS_ASSERT_EQUALS(output, expected); // w/o coercion + } + + { // partial dataset from front 0->end-1 + const std::vector expected({0, 1, 2, 3}); + output.clear(); + H5Util::readArray1DCoerce(dataSetFloat, output, array1d_double.size() - 1, 0); + TS_ASSERT_EQUALS(output, expected); // w/ coercion + output.clear(); + H5Util::readArray1DCoerce(dataSetDouble, output, array1d_double.size() - 1, 0); + TS_ASSERT_EQUALS(output, expected); // w/o coercion + } + { // partial dataset from front 1->end-1 + const std::vector expected({1, 2, 3}); + output.clear(); + H5Util::readArray1DCoerce(dataSetFloat, output, array1d_double.size() - 2, 1); + TS_ASSERT_EQUALS(output, expected); // w/ coercion + output.clear(); + H5Util::readArray1DCoerce(dataSetDouble, output, array1d_double.size() - 2, 1); + TS_ASSERT_EQUALS(output, expected); // w/o coercion + } + { // from 1->end+1 + const std::vector expected({1, 2, 3, 4}); + output.clear(); + H5Util::readArray1DCoerce(dataSetFloat, output, array1d_double.size() + 1, 1); + TS_ASSERT_EQUALS(output, expected); // w/ coercion + output.clear(); + H5Util::readArray1DCoerce(dataSetDouble, output, array1d_double.size() + 1, 1); + TS_ASSERT_EQUALS(output, expected); // w/o coercion + } + file.close(); }