Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

H5Util read 1d slices of data - ornl-next #38576

Merged
merged 3 commits into from
Jan 8, 2025
Merged

H5Util read 1d slices of data - ornl-next #38576

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
5dc6ce1
Do not double read when actual matches requested
peterfpeterson Dec 26, 2024
3c6a54a
Add ability to read portions of the data
peterfpeterson Dec 27, 2024
325fb4a
Combine functions by using default values
peterfpeterson Jan 6, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
7 changes: 6 additions & 1 deletion Framework/Nexus/inc/MantidNexus/H5Util.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@

#include "MantidNexus/DllConfig.h"

#include <limits>
#include <map>
#include <string>
#include <vector>
Expand Down Expand Up @@ -91,9 +92,13 @@ std::vector<NumT> readNumArrayAttributeCoerce(const H5::H5Object &object, const

template <typename NumT>
void readArray1DCoerce(const H5::Group &group, const std::string &name, std::vector<NumT> &output);

template <typename NumT> std::vector<NumT> readArray1DCoerce(const H5::Group &group, const std::string &name);

template <typename NumT> void readArray1DCoerce(const H5::DataSet &dataset, std::vector<NumT> &output);
template <typename NumT>
void readArray1DCoerce(const H5::DataSet &dataset, std::vector<NumT> &output,
const size_t length = std::numeric_limits<size_t>::max(),
const size_t offset = static_cast<size_t>(0));

/// Test if a group already exists within an HDF5 file or parent group.
MANTID_NEXUS_DLL bool groupExists(H5::H5Object &h5, const std::string &groupPath);
Expand Down
76 changes: 50 additions & 26 deletions Framework/Nexus/src/H5Util.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -282,16 +282,15 @@ template <typename NumT> std::vector<NumT> readArray1DCoerce(const H5::Group &gr

namespace {
template <typename InputNumT, typename OutputNumT>
void convertingRead(const DataSet &dataset, const DataType &dataType, std::vector<OutputNumT> &output) {
DataSpace dataSpace = dataset.getSpace();

std::vector<InputNumT> temp(dataSpace.getSelectNpoints());
dataset.read(temp.data(), dataType, dataSpace);
void convertingRead(const DataSet &dataset, const DataType &dataType, std::vector<OutputNumT> &output,
const DataSpace &memspace, const DataSpace &filespace) {
std::vector<InputNumT> temp(filespace.getSelectNpoints());
dataset.read(temp.data(), dataType, memspace, filespace);

output.resize(temp.size());

std::transform(temp.begin(), temp.end(), output.begin(),
[](const InputNumT a) { // lambda
[](const auto &a) { // lambda
return boost::numeric_cast<OutputNumT>(a);
});
}
Expand Down Expand Up @@ -379,27 +378,46 @@ std::vector<NumT> readNumArrayAttributeCoerce(const H5::H5Object &object, const
return value;
}

template <typename NumT> void readArray1DCoerce(const DataSet &dataset, std::vector<NumT> &output) {
DataType dataType = dataset.getDataType();

if (getType<NumT>() == dataType) { // no conversion necessary
DataSpace dataSpace = dataset.getSpace();
output.resize(dataSpace.getSelectNpoints());
dataset.read(output.data(), dataType, dataSpace);
template <typename NumT>
void readArray1DCoerce(const H5::DataSet &dataset, std::vector<NumT> &output, const size_t length,
const size_t offset) {
DataSpace filespace = dataset.getSpace();
const auto length_actual = static_cast<size_t>(filespace.getSelectNpoints());

if (offset >= length_actual) {
std::stringstream msg;
msg << "Tried to read offset=" << offset << " into array that is only lenght=" << length_actual << " long";
throw std::runtime_error(msg.str());
}

if (PredType::NATIVE_INT32 == dataType) {
convertingRead<int32_t>(dataset, dataType, output);
// set extent and offset in DataSpace
const hsize_t rankedoffset[1] = {static_cast<hsize_t>(offset)};
const hsize_t rankedextent[1] = {
static_cast<hsize_t>(std::min(length, length_actual - offset))}; // don't read past the end
// select a part of filespace if appropriate
if (rankedextent[0] < length_actual)
filespace.selectHyperslab(H5S_SELECT_SET, rankedextent, rankedoffset);

// size of thing being read out
DataSpace memspace(1, rankedextent);

// do the actual read
const DataType dataType = dataset.getDataType();
if (getType<NumT>() == dataType) { // no conversion necessary
output.resize(static_cast<size_t>(filespace.getSelectNpoints()));
dataset.read(output.data(), dataType, memspace, filespace);
} else if (PredType::NATIVE_INT32 == dataType) {
convertingRead<int32_t>(dataset, dataType, output, memspace, filespace);
} else if (PredType::NATIVE_UINT32 == dataType) {
convertingRead<uint32_t>(dataset, dataType, output);
convertingRead<uint32_t>(dataset, dataType, output, memspace, filespace);
} else if (PredType::NATIVE_INT64 == dataType) {
convertingRead<int64_t>(dataset, dataType, output);
convertingRead<int64_t>(dataset, dataType, output, memspace, filespace);
} else if (PredType::NATIVE_UINT64 == dataType) {
convertingRead<uint64_t>(dataset, dataType, output);
convertingRead<uint64_t>(dataset, dataType, output, memspace, filespace);
} else if (PredType::NATIVE_FLOAT == dataType) {
convertingRead<float>(dataset, dataType, output);
convertingRead<float>(dataset, dataType, output, memspace, filespace);
} else if (PredType::NATIVE_DOUBLE == dataType) {
convertingRead<double>(dataset, dataType, output);
convertingRead<double>(dataset, dataType, output, memspace, filespace);
} else {
// not a supported type
throw DataTypeIException();
Expand Down Expand Up @@ -596,10 +614,16 @@ template MANTID_NEXUS_DLL std::vector<uint32_t> readArray1DCoerce(const H5::Grou
template MANTID_NEXUS_DLL std::vector<int64_t> readArray1DCoerce(const H5::Group &group, const std::string &name);
template MANTID_NEXUS_DLL std::vector<uint64_t> readArray1DCoerce(const H5::Group &group, const std::string &name);

template MANTID_NEXUS_DLL void readArray1DCoerce(const DataSet &dataset, std::vector<float> &output);
template MANTID_NEXUS_DLL void readArray1DCoerce(const DataSet &dataset, std::vector<double> &output);
template MANTID_NEXUS_DLL void readArray1DCoerce(const DataSet &dataset, std::vector<int32_t> &output);
template MANTID_NEXUS_DLL void readArray1DCoerce(const DataSet &dataset, std::vector<uint32_t> &output);
template MANTID_NEXUS_DLL void readArray1DCoerce(const DataSet &dataset, std::vector<int64_t> &output);
template MANTID_NEXUS_DLL void readArray1DCoerce(const DataSet &dataset, std::vector<uint64_t> &output);
template MANTID_NEXUS_DLL void readArray1DCoerce(const DataSet &dataset, std::vector<float> &output,
const size_t length, const size_t offset);
template MANTID_NEXUS_DLL void readArray1DCoerce(const DataSet &dataset, std::vector<double> &output,
const size_t length, const size_t offset);
template MANTID_NEXUS_DLL void readArray1DCoerce(const DataSet &dataset, std::vector<int32_t> &output,
const size_t length, const size_t offset);
template MANTID_NEXUS_DLL void readArray1DCoerce(const DataSet &dataset, std::vector<uint32_t> &output,
const size_t length, const size_t offset);
template MANTID_NEXUS_DLL void readArray1DCoerce(const DataSet &dataset, std::vector<int64_t> &output,
const size_t length, const size_t offset);
template MANTID_NEXUS_DLL void readArray1DCoerce(const DataSet &dataset, std::vector<uint64_t> &output,
const size_t length, const size_t offset);
} // namespace Mantid::NeXus::H5Util
59 changes: 58 additions & 1 deletion Framework/Nexus/test/H5UtilTest.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -135,7 +135,7 @@ class H5UtilTest : public CxxTest::TestSuite {

TS_ASSERT(std::filesystem::exists(FILENAME));

{ // read tests
{ // ---------- simple read tests
H5File file(FILENAME, H5F_ACC_RDONLY);
auto group = file.openGroup(GRP_NAME);

Expand All @@ -151,6 +151,63 @@ class H5UtilTest : public CxxTest::TestSuite {
const boost::numeric::positive_overflow &);
TS_ASSERT_THROWS_NOTHING(H5Util::readArray1DCoerce<uint32_t>(group, "array1d_int32"));

// ---------- slicing read tests
auto dataSetFloat = group.openDataSet("array1d_float");
auto dataSetDouble = group.openDataSet("array1d_double");

std::vector<double> output;
// full dataset
output.clear();
H5Util::readArray1DCoerce(dataSetFloat, output, array1d_double.size(), 0);
TS_ASSERT_EQUALS(output, array1d_double); // whole thing w/ coercion
output.clear();
H5Util::readArray1DCoerce(dataSetDouble, output, array1d_double.size(), 0);
TS_ASSERT_EQUALS(output, array1d_double); // whole thing w/o coercion
output.clear();
H5Util::readArray1DCoerce(dataSetFloat, output, array1d_double.size() + 1, 0);
TS_ASSERT_EQUALS(output, array1d_double); // more than the whole thing w/ coercion
output.clear();
H5Util::readArray1DCoerce(dataSetDouble, output, array1d_double.size() + 1, 0);
TS_ASSERT_EQUALS(output, array1d_double); // more than the whole thing w/o coercion

{ // partial dataset from front 1->end
const std::vector<double> expected({1, 2, 3, 4});
output.clear();
H5Util::readArray1DCoerce(dataSetFloat, output, array1d_double.size() - 1, 1);
TS_ASSERT_EQUALS(output, expected); // w/ coercion
output.clear();
H5Util::readArray1DCoerce(dataSetDouble, output, array1d_double.size() - 1, 1);
TS_ASSERT_EQUALS(output, expected); // w/o coercion
}

{ // partial dataset from front 0->end-1
const std::vector<double> expected({0, 1, 2, 3});
output.clear();
H5Util::readArray1DCoerce(dataSetFloat, output, array1d_double.size() - 1, 0);
TS_ASSERT_EQUALS(output, expected); // w/ coercion
output.clear();
H5Util::readArray1DCoerce(dataSetDouble, output, array1d_double.size() - 1, 0);
TS_ASSERT_EQUALS(output, expected); // w/o coercion
}
{ // partial dataset from front 1->end-1
const std::vector<double> expected({1, 2, 3});
output.clear();
H5Util::readArray1DCoerce(dataSetFloat, output, array1d_double.size() - 2, 1);
TS_ASSERT_EQUALS(output, expected); // w/ coercion
output.clear();
H5Util::readArray1DCoerce(dataSetDouble, output, array1d_double.size() - 2, 1);
TS_ASSERT_EQUALS(output, expected); // w/o coercion
}
{ // from 1->end+1
const std::vector<double> expected({1, 2, 3, 4});
output.clear();
H5Util::readArray1DCoerce(dataSetFloat, output, array1d_double.size() + 1, 1);
TS_ASSERT_EQUALS(output, expected); // w/ coercion
output.clear();
H5Util::readArray1DCoerce(dataSetDouble, output, array1d_double.size() + 1, 1);
TS_ASSERT_EQUALS(output, expected); // w/o coercion
}

file.close();
}

Expand Down
Loading