Skip to content

Commit

Permalink
Merge pull request #38546 from peterfpeterson/38332_h5util_improvements
Browse files Browse the repository at this point in the history 
H5Util read 1d slices of data
  • Loading branch information
@AndreiSavici
AndreiSavici authored Jan 8, 2025
2 parents a203844 + 325fb4a commit 17a7a92
Show file tree
Hide file tree
Showing 3 changed files with 114 additions and 28 deletions.
7 changes: 6 additions & 1 deletion Framework/Nexus/inc/MantidNexus/H5Util.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@

#include "MantidNexus/DllConfig.h"

#include <limits>
#include <map>
#include <string>
#include <vector>
Expand Down Expand Up @@ -91,9 +92,13 @@ std::vector<NumT> readNumArrayAttributeCoerce(const H5::H5Object &object, const

template <typename NumT>
void readArray1DCoerce(const H5::Group &group, const std::string &name, std::vector<NumT> &output);

template <typename NumT> std::vector<NumT> readArray1DCoerce(const H5::Group &group, const std::string &name);

template <typename NumT> void readArray1DCoerce(const H5::DataSet &dataset, std::vector<NumT> &output);
template <typename NumT>
void readArray1DCoerce(const H5::DataSet &dataset, std::vector<NumT> &output,
const size_t length = std::numeric_limits<size_t>::max(),
const size_t offset = static_cast<size_t>(0));

/// Test if a group already exists within an HDF5 file or parent group.
MANTID_NEXUS_DLL bool groupExists(H5::H5Object &h5, const std::string &groupPath);
Expand Down
76 changes: 50 additions & 26 deletions Framework/Nexus/src/H5Util.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -282,16 +282,15 @@ template <typename NumT> std::vector<NumT> readArray1DCoerce(const H5::Group &gr

namespace {
template <typename InputNumT, typename OutputNumT>
void convertingRead(const DataSet &dataset, const DataType &dataType, std::vector<OutputNumT> &output) {
DataSpace dataSpace = dataset.getSpace();

std::vector<InputNumT> temp(dataSpace.getSelectNpoints());
dataset.read(temp.data(), dataType, dataSpace);
void convertingRead(const DataSet &dataset, const DataType &dataType, std::vector<OutputNumT> &output,
const DataSpace &memspace, const DataSpace &filespace) {
std::vector<InputNumT> temp(filespace.getSelectNpoints());
dataset.read(temp.data(), dataType, memspace, filespace);

output.resize(temp.size());

std::transform(temp.begin(), temp.end(), output.begin(),
[](const InputNumT a) { // lambda
[](const auto &a) { // lambda
return boost::numeric_cast<OutputNumT>(a);
});
}
Expand Down Expand Up @@ -379,27 +378,46 @@ std::vector<NumT> readNumArrayAttributeCoerce(const H5::H5Object &object, const
return value;
}

template <typename NumT> void readArray1DCoerce(const DataSet &dataset, std::vector<NumT> &output) {
DataType dataType = dataset.getDataType();

if (getType<NumT>() == dataType) { // no conversion necessary
DataSpace dataSpace = dataset.getSpace();
output.resize(dataSpace.getSelectNpoints());
dataset.read(output.data(), dataType, dataSpace);
template <typename NumT>
void readArray1DCoerce(const H5::DataSet &dataset, std::vector<NumT> &output, const size_t length,
const size_t offset) {
DataSpace filespace = dataset.getSpace();
const auto length_actual = static_cast<size_t>(filespace.getSelectNpoints());

if (offset >= length_actual) {
std::stringstream msg;
msg << "Tried to read offset=" << offset << " into array that is only lenght=" << length_actual << " long";
throw std::runtime_error(msg.str());
}

if (PredType::NATIVE_INT32 == dataType) {
convertingRead<int32_t>(dataset, dataType, output);
// set extent and offset in DataSpace
const hsize_t rankedoffset[1] = {static_cast<hsize_t>(offset)};
const hsize_t rankedextent[1] = {
static_cast<hsize_t>(std::min(length, length_actual - offset))}; // don't read past the end
// select a part of filespace if appropriate
if (rankedextent[0] < length_actual)
filespace.selectHyperslab(H5S_SELECT_SET, rankedextent, rankedoffset);

// size of thing being read out
DataSpace memspace(1, rankedextent);

// do the actual read
const DataType dataType = dataset.getDataType();
if (getType<NumT>() == dataType) { // no conversion necessary
output.resize(static_cast<size_t>(filespace.getSelectNpoints()));
dataset.read(output.data(), dataType, memspace, filespace);
} else if (PredType::NATIVE_INT32 == dataType) {
convertingRead<int32_t>(dataset, dataType, output, memspace, filespace);
} else if (PredType::NATIVE_UINT32 == dataType) {
convertingRead<uint32_t>(dataset, dataType, output);
convertingRead<uint32_t>(dataset, dataType, output, memspace, filespace);
} else if (PredType::NATIVE_INT64 == dataType) {
convertingRead<int64_t>(dataset, dataType, output);
convertingRead<int64_t>(dataset, dataType, output, memspace, filespace);
} else if (PredType::NATIVE_UINT64 == dataType) {
convertingRead<uint64_t>(dataset, dataType, output);
convertingRead<uint64_t>(dataset, dataType, output, memspace, filespace);
} else if (PredType::NATIVE_FLOAT == dataType) {
convertingRead<float>(dataset, dataType, output);
convertingRead<float>(dataset, dataType, output, memspace, filespace);
} else if (PredType::NATIVE_DOUBLE == dataType) {
convertingRead<double>(dataset, dataType, output);
convertingRead<double>(dataset, dataType, output, memspace, filespace);
} else {
// not a supported type
throw DataTypeIException();
Expand Down Expand Up @@ -596,10 +614,16 @@ template MANTID_NEXUS_DLL std::vector<uint32_t> readArray1DCoerce(const H5::Grou
template MANTID_NEXUS_DLL std::vector<int64_t> readArray1DCoerce(const H5::Group &group, const std::string &name);
template MANTID_NEXUS_DLL std::vector<uint64_t> readArray1DCoerce(const H5::Group &group, const std::string &name);

template MANTID_NEXUS_DLL void readArray1DCoerce(const DataSet &dataset, std::vector<float> &output);
template MANTID_NEXUS_DLL void readArray1DCoerce(const DataSet &dataset, std::vector<double> &output);
template MANTID_NEXUS_DLL void readArray1DCoerce(const DataSet &dataset, std::vector<int32_t> &output);
template MANTID_NEXUS_DLL void readArray1DCoerce(const DataSet &dataset, std::vector<uint32_t> &output);
template MANTID_NEXUS_DLL void readArray1DCoerce(const DataSet &dataset, std::vector<int64_t> &output);
template MANTID_NEXUS_DLL void readArray1DCoerce(const DataSet &dataset, std::vector<uint64_t> &output);
template MANTID_NEXUS_DLL void readArray1DCoerce(const DataSet &dataset, std::vector<float> &output,
const size_t length, const size_t offset);
template MANTID_NEXUS_DLL void readArray1DCoerce(const DataSet &dataset, std::vector<double> &output,
const size_t length, const size_t offset);
template MANTID_NEXUS_DLL void readArray1DCoerce(const DataSet &dataset, std::vector<int32_t> &output,
const size_t length, const size_t offset);
template MANTID_NEXUS_DLL void readArray1DCoerce(const DataSet &dataset, std::vector<uint32_t> &output,
const size_t length, const size_t offset);
template MANTID_NEXUS_DLL void readArray1DCoerce(const DataSet &dataset, std::vector<int64_t> &output,
const size_t length, const size_t offset);
template MANTID_NEXUS_DLL void readArray1DCoerce(const DataSet &dataset, std::vector<uint64_t> &output,
const size_t length, const size_t offset);
} // namespace Mantid::NeXus::H5Util
59 changes: 58 additions & 1 deletion Framework/Nexus/test/H5UtilTest.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -135,7 +135,7 @@ class H5UtilTest : public CxxTest::TestSuite {

TS_ASSERT(std::filesystem::exists(FILENAME));

{ // read tests
{ // ---------- simple read tests
H5File file(FILENAME, H5F_ACC_RDONLY);
auto group = file.openGroup(GRP_NAME);

Expand All @@ -151,6 +151,63 @@ class H5UtilTest : public CxxTest::TestSuite {
const boost::numeric::positive_overflow &);
TS_ASSERT_THROWS_NOTHING(H5Util::readArray1DCoerce<uint32_t>(group, "array1d_int32"));

// ---------- slicing read tests
auto dataSetFloat = group.openDataSet("array1d_float");
auto dataSetDouble = group.openDataSet("array1d_double");

std::vector<double> output;
// full dataset
output.clear();
H5Util::readArray1DCoerce(dataSetFloat, output, array1d_double.size(), 0);
TS_ASSERT_EQUALS(output, array1d_double); // whole thing w/ coercion
output.clear();
H5Util::readArray1DCoerce(dataSetDouble, output, array1d_double.size(), 0);
TS_ASSERT_EQUALS(output, array1d_double); // whole thing w/o coercion
output.clear();
H5Util::readArray1DCoerce(dataSetFloat, output, array1d_double.size() + 1, 0);
TS_ASSERT_EQUALS(output, array1d_double); // more than the whole thing w/ coercion
output.clear();
H5Util::readArray1DCoerce(dataSetDouble, output, array1d_double.size() + 1, 0);
TS_ASSERT_EQUALS(output, array1d_double); // more than the whole thing w/o coercion

{ // partial dataset from front 1->end
const std::vector<double> expected({1, 2, 3, 4});
output.clear();
H5Util::readArray1DCoerce(dataSetFloat, output, array1d_double.size() - 1, 1);
TS_ASSERT_EQUALS(output, expected); // w/ coercion
output.clear();
H5Util::readArray1DCoerce(dataSetDouble, output, array1d_double.size() - 1, 1);
TS_ASSERT_EQUALS(output, expected); // w/o coercion
}

{ // partial dataset from front 0->end-1
const std::vector<double> expected({0, 1, 2, 3});
output.clear();
H5Util::readArray1DCoerce(dataSetFloat, output, array1d_double.size() - 1, 0);
TS_ASSERT_EQUALS(output, expected); // w/ coercion
output.clear();
H5Util::readArray1DCoerce(dataSetDouble, output, array1d_double.size() - 1, 0);
TS_ASSERT_EQUALS(output, expected); // w/o coercion
}
{ // partial dataset from front 1->end-1
const std::vector<double> expected({1, 2, 3});
output.clear();
H5Util::readArray1DCoerce(dataSetFloat, output, array1d_double.size() - 2, 1);
TS_ASSERT_EQUALS(output, expected); // w/ coercion
output.clear();
H5Util::readArray1DCoerce(dataSetDouble, output, array1d_double.size() - 2, 1);
TS_ASSERT_EQUALS(output, expected); // w/o coercion
}
{ // from 1->end+1
const std::vector<double> expected({1, 2, 3, 4});
output.clear();
H5Util::readArray1DCoerce(dataSetFloat, output, array1d_double.size() + 1, 1);
TS_ASSERT_EQUALS(output, expected); // w/ coercion
output.clear();
H5Util::readArray1DCoerce(dataSetDouble, output, array1d_double.size() + 1, 1);
TS_ASSERT_EQUALS(output, expected); // w/o coercion
}

file.close();
}

Expand Down

0 comments on commit 17a7a92

Please sign in to comment.