Merge pull request #38309 from mantidproject/EWM6549_SaveNexusESS_gro…

…ups-main

`SaveNexusESS` append mode.

Framework/Beamline/src/ComponentInfo.cpp

@@ -210,7 +210,7 @@ Eigen::Quaterniond ComponentInfo::rotation(const std::pair<size_t, size_t> &inde
 /**
  * Extract the position of a component relative to it's parent
  *
- * The parent rotatation is unwound prior to establishing the offset. This means
+ * The parent rotation is unwound prior to establishing the offset. This means
  *that
  * recorded relative positions are independent of changes in rotation.
  *

Framework/DataHandling/inc/MantidDataHandling/H5Util.h

@@ -20,6 +20,7 @@ class DSetCreatPropList;
 class DataType;
 class Group;
 class H5File;
+class H5Object;
 } // namespace H5
 
 namespace Mantid {
@@ -94,6 +95,22 @@ template <typename NumT> std::vector<NumT> readArray1DCoerce(H5::Group &group, c
 
 template <typename NumT> std::vector<NumT> readArray1DCoerce(H5::DataSet &dataset);
 
+/// Test if a group already exists within an HDF5 file or parent group.
+MANTID_DATAHANDLING_DLL bool groupExists(H5::H5Object &h5, const std::string &groupPath);
+
+/// Test if an attribute is present and has a specific string value for an HDF5 group or dataset.
+MANTID_DATAHANDLING_DLL bool keyHasValue(H5::H5Object &h5, const std::string &key, const std::string &value);
+
+/// Copy a group and all of its contents, between the same or different HDF5 files or groups.
+MANTID_DATAHANDLING_DLL void copyGroup(H5::H5Object &dest, const std::string &destGroupPath, H5::H5Object &src,
+                                       const std::string &srcGroupPath);
+
+/**
+ * Delete a target link for a group or dataset from a parent group.
+ * If this is the last link to the target in the HDF5 graph, then it will be removed from the file.
+ */
+MANTID_DATAHANDLING_DLL void deleteObjectLink(H5::H5Object &h5, const std::string &target);
+
 } // namespace H5Util
 } // namespace DataHandling
 } // namespace Mantid

Framework/DataHandling/inc/MantidDataHandling/LoadNexusProcessed.h

@@ -93,10 +93,12 @@ class MANTID_DATAHANDLING_DLL LoadNexusProcessed : public API::NexusFileLoader {
   std::string loadWorkspaceName(Mantid::NeXus::NXRoot &root, const std::string &entry_name);
 
   /// Load nexus geometry and apply to workspace
-  virtual bool loadNexusGeometry(Mantid::API::Workspace &, const int, Kernel::Logger &,
-                                 const std::string &) { /*do nothing*/
-    return false;
+  virtual bool loadNexusGeometry(Mantid::API::Workspace & /* ws */, size_t /* entryNumber */,
+                                 Kernel::Logger & /* logger */,
+                                 const std::string & /* filePath */) { /* args not used */
+    return false;                                                      /*do nothing*/
   }
+
   /// Load a single entry
   API::Workspace_sptr loadEntry(Mantid::NeXus::NXRoot &root, const std::string &entry_name, const double &progressStart,
                                 const double &progressRange);

Framework/DataHandling/inc/MantidDataHandling/LoadNexusProcessed2.h

@@ -13,6 +13,7 @@
 #include "MantidIndexing/SpectrumNumber.h"
 #include "MantidKernel/NexusDescriptor.h"
 #include <string>
+#include <unordered_map>
 
 namespace Mantid {
 namespace API {
@@ -31,26 +32,37 @@ enum class InstrumentLayout { Mantid, NexusFormat, NotRecognised };
  * Processed files.
  *
  * The majority of the implementation consists of function overrides for
- * specific virtual hooks make in the base Algorithm LoadNexusProcessed
+ * specific virtual functions in the base Algorithm LoadNexusProcessed
  */
 class MANTID_DATAHANDLING_DLL LoadNexusProcessed2 : public LoadNexusProcessed {
 public:
-  const std::string name() const override;
+  // algorithm "name" is still "LoadNexusProcessed" (not "LoadNexusProcessed2"):
+  //   `cppcheck` has an issue with any "useless" override.
+  // const std::string name() const override;
+
   int version() const override;
   int confidence(Kernel::NexusHDF5Descriptor &descriptor) const override;
 
 private:
   void readSpectraToDetectorMapping(Mantid::NeXus::NXEntry &mtd_entry, Mantid::API::MatrixWorkspace &ws) override;
-  bool loadNexusGeometry(Mantid::API::Workspace &ws, const int nWorkspaceEntries, Kernel::Logger &logger,
-                         const std::string &filename) override;
+
+  /// Load nexus geometry and apply to workspace
+  bool loadNexusGeometry(Mantid::API::Workspace &ws, size_t entryNumber, Kernel::Logger &logger,
+                         const std::string &filePath) override;
+
   /// Extract mapping information where it is build across NXDetectors
   void extractMappingInfoNew(const Mantid::NeXus::NXEntry &mtd_entry);
-  /// Load nexus geometry and apply to workspace
-  /// Local caches
+
   InstrumentLayout m_instrumentLayout = InstrumentLayout::Mantid;
-  std::vector<Indexing::SpectrumNumber> m_spectrumNumbers;
-  std::vector<Mantid::detid_t> m_detectorIds;
-  std::vector<int> m_detectorCounts;
+
+  // Local cache vectors:
+  //   spectral mapping information is accumulated before
+  //   the instrument geometry has been completely loaded.
+  //
+  // The key is the NXentry-group name (in order to allow for group workspaces).
+  std::unordered_map<std::string, std::vector<Indexing::SpectrumNumber>> m_spectrumNumberss;
+  std::unordered_map<std::string, std::vector<Mantid::detid_t>> m_detectorIdss;
+  std::unordered_map<std::string, std::vector<int>> m_detectorCountss;
 };
 
 } // namespace DataHandling

Framework/DataHandling/inc/MantidDataHandling/SaveNexusESS.h

@@ -31,9 +31,13 @@ class MANTID_DATAHANDLING_DLL SaveNexusESS : public Mantid::DataHandling::SaveNe
   bool processGroups() override;
 
 private:
-  void saveNexusGeometry(const Mantid::API::MatrixWorkspace &ws, const std::string &filename);
+  void saveNexusGeometry(const Mantid::API::MatrixWorkspace &ws, const std::string &filename,
+                         std::optional<size_t> entryNumber = std::optional<size_t>());
+
   virtual bool saveLegacyInstrument() override;
+
   void init() override;
+
   void exec() override;
 };
 

Framework/DataHandling/src/H5Util.cpp

@@ -12,6 +12,8 @@
 #include <algorithm>
 #include <array>
 #include <boost/numeric/conversion/cast.hpp>
+#include <stdexcept>
+#include <string>
 
 using namespace H5;
 
@@ -175,10 +177,10 @@ std::string readString(H5::H5File &file, const std::string &path) {
   try {
     auto data = file.openDataSet(path);
     return readString(data);
-  } catch (H5::FileIException &e) {
+  } catch (const H5::FileIException &e) {
     UNUSED_ARG(e);
     return "";
-  } catch (H5::GroupIException &e) {
+  } catch (const H5::GroupIException &e) {
     UNUSED_ARG(e);
     return "";
   }
@@ -188,7 +190,7 @@ std::string readString(H5::Group &group, const std::string &name) {
   try {
     auto data = group.openDataSet(name);
     return readString(data);
-  } catch (H5::GroupIException &e) {
+  } catch (const H5::GroupIException &e) {
     UNUSED_ARG(e);
     return "";
   }
@@ -252,10 +254,10 @@ template <typename NumT> std::vector<NumT> readArray1DCoerce(H5::Group &group, c
   try {
     DataSet dataset = group.openDataSet(name);
     result = readArray1DCoerce<NumT>(dataset);
-  } catch (H5::GroupIException &e) {
+  } catch (const H5::GroupIException &e) {
     UNUSED_ARG(e);
     g_log.information("Failed to open dataset \"" + name + "\"\n");
-  } catch (H5::DataTypeIException &e) {
+  } catch (const H5::DataTypeIException &e) {
     UNUSED_ARG(e);
     g_log.information("DataSet \"" + name + "\" should be double" + "\n");
   }
@@ -395,6 +397,66 @@ template <typename NumT> std::vector<NumT> readArray1DCoerce(DataSet &dataset) {
   throw DataTypeIException();
 }
 
+/// Test if a group exists in an HDF5 file or parent group.
+bool groupExists(H5::H5Object &h5, const std::string &groupPath) {
+  bool status = true;
+  // Unfortunately, this is actually the approach recommended by the HDF Group.
+  try {
+    h5.openGroup(groupPath);
+  } catch (const H5::Exception &e) {
+    UNUSED_ARG(e);
+    status = false;
+  }
+  return status;
+}
+
+/// Test if an attribute is present on an HDF5 group or dataset and has a specific string value.
+bool keyHasValue(H5::H5Object &h5, const std::string &key, const std::string &value) {
+  bool status = true;
+  try {
+    Attribute attr = h5.openAttribute(key);
+    std::string value_;
+    attr.read(attr.getDataType(), value_);
+    if (value_ != value)
+      status = false;
+  } catch (const H5::Exception &e) {
+    UNUSED_ARG(e);
+    status = false;
+  }
+  return status;
+}
+
+void copyGroup(H5::H5Object &dest, const std::string &destGroupPath, H5::H5Object &src,
+               const std::string &srcGroupPath) {
+  // Source group must exist, and destination group must not exist.
+  if (!groupExists(src, srcGroupPath) || groupExists(dest, destGroupPath))
+    throw std::invalid_argument(std::string("H5Util::copyGroup: source group '") + srcGroupPath + "' must exist and " +
+                                "destination group '" + destGroupPath + "' must not exist.");
+
+  // TODO: check that source file must have at least read access and destination file must have write access.
+
+  // Note that in the HDF5 API:
+  //   C++ API support for these HDF5 methods does not yet exist.
+
+  // Create intermediate groups, if necessary
+  hid_t lcpl = H5Pcreate(H5P_LINK_CREATE);
+  if (H5Pset_create_intermediate_group(lcpl, 1) < 0)
+    throw std::runtime_error("H5Util::copyGroup: 'H5Pset_create_intermediate_group' error return.");
+
+  if (H5Ocopy(src.getId(), srcGroupPath.c_str(), dest.getId(), destGroupPath.c_str(), H5P_DEFAULT, lcpl) < 0)
+    throw std::runtime_error("H5Util::copyGroup: 'H5Ocopy' error return.");
+  H5Pclose(lcpl);
+}
+
+void deleteObjectLink(H5::H5Object &h5, const std::string &target) {
+  // Note that in the HDF5 API:
+  //   C++ API support for this HDF5 method does not yet exist.
+
+  // Target object must exist
+  if (H5Ldelete(h5.getId(), target.c_str(), H5P_DEFAULT) < 0)
+    throw std::runtime_error("H5Util::deleteObjectLink: 'H5Ldelete' error return.");
+}
+
 // -------------------------------------------------------------------
 // instantiations for writeStrAttribute
 // -------------------------------------------------------------------

Framework/DataHandling/src/LoadNexusProcessed.cpp

@@ -375,6 +375,12 @@ void LoadNexusProcessed::execLoader() {
 
   API::Workspace_sptr tempWS;
   size_t nWorkspaceEntries = 0;
+
+  // Check for an entry number property
+  int entryNumber = getProperty("EntryNumber");
+  Property const *const entryNumberProperty = this->getProperty("EntryNumber");
+  bool bDefaultEntryNumber = entryNumberProperty->isDefault();
+
   // Start scoped block
   {
     progress(0, "Opening file...");
@@ -390,13 +396,9 @@ void LoadNexusProcessed::execLoader() {
     nWorkspaceEntries = std::count_if(root.groups().cbegin(), root.groups().cend(),
                                       [](const auto &g) { return g.nxclass == "NXentry"; });
 
-    // Check for an entry number property
-    int entrynumber = getProperty("EntryNumber");
-    Property const *const entryNumberProperty = this->getProperty("EntryNumber");
-    bool bDefaultEntryNumber = entryNumberProperty->isDefault();
-
-    if (!bDefaultEntryNumber && entrynumber > static_cast<int>(nWorkspaceEntries)) {
-      g_log.error() << "Invalid entry number specified. File only contains " << nWorkspaceEntries << " entries.\n";
+    if (!bDefaultEntryNumber && static_cast<size_t>(entryNumber) > nWorkspaceEntries) {
+      g_log.error() << "Invalid entry number: " << entryNumber
+                    << " specified. File only contains: " << nWorkspaceEntries << " entries.\n";
       throw std::invalid_argument("Invalid entry number specified.");
     }
 
@@ -405,9 +407,9 @@ void LoadNexusProcessed::execLoader() {
     std::ostringstream os;
     if (bDefaultEntryNumber) {
       // Set the entry number to 1 if not provided.
-      entrynumber = 1;
+      entryNumber = 1;
     }
-    os << basename << entrynumber;
+    os << basename << entryNumber;
     const std::string targetEntryName = os.str();
 
     // Take the first real workspace obtainable. We need it even if loading
@@ -502,12 +504,28 @@ void LoadNexusProcessed::execLoader() {
     }
 
     root.close();
-  } // All file resources should be scoped to here. All previous file handles
-  // must be cleared to release locks
-  loadNexusGeometry(*tempWS, static_cast<int>(nWorkspaceEntries), g_log, std::string(getProperty("Filename")));
+  }
+
+  // All file resources should be scoped to here. All previous file handles
+  //   must be cleared to release locks.
+
+  // NexusGeometry uses direct HDF5 access, and not the `NexusFileIO` methods.
+  // For this reason, a separate section is required to load the instrument[s] into the output workspace[s].
+
+  if (nWorkspaceEntries == 1 || !bDefaultEntryNumber)
+    loadNexusGeometry(*getValue<API::Workspace_sptr>("OutputWorkspace"), static_cast<size_t>(entryNumber), g_log,
+                      std::string(getProperty("Filename")));
+  else {
+    for (size_t nEntry = 1; nEntry <= static_cast<size_t>(nWorkspaceEntries); ++nEntry) {
+      std::ostringstream wsPropertyName;
+      wsPropertyName << "OutputWorkspace_" << nEntry;
+      loadNexusGeometry(*getValue<API::Workspace_sptr>(wsPropertyName.str()), nEntry, g_log,
+                        std::string(getProperty("Filename")));
+    }
+  }
 
   m_axis1vals.clear();
-} // namespace DataHandling
+}
 
 /**
  * Decides what to call a child of a group workspace.
@@ -1890,14 +1908,16 @@ API::Workspace_sptr LoadNexusProcessed::loadEntry(NXRoot &root, const std::strin
     progress(progressStart + 0.11 * progressRange, "Reading the parameter maps...");
     local_workspace->readParameterMap(parameterStr);
   } catch (std::exception &e) {
-    // TODO. For workspaces saved via SaveNexusESS, these warnings are not
-    // relevant. Unfortunately we need to close all file handles before we can
-    // attempt loading the new way see loadNexusGeometry function . A better
-    // solution should be found
-    g_log.warning("Error loading Instrument section of nxs file");
-    g_log.warning(e.what());
-    g_log.warning("Try running LoadInstrument Algorithm on the Workspace to "
-                  "update the geometry");
+    // For workspaces saved via SaveNexusESS, these warnings are not
+    // relevant. Such workspaces will contain an `NXinstrument` entry
+    // with the name of the instrument.
+    const auto &entries = getFileInfo()->getAllEntries();
+    if (version() < 2 || entries.find("NXinstrument") == entries.end()) {
+      g_log.warning("Error loading Instrument section of nxs file");
+      g_log.warning(e.what());
+      g_log.warning("Try running LoadInstrument Algorithm on the Workspace to "
+                    "update the geometry");
+    }
   }
 
   readSpectraToDetectorMapping(mtd_entry, *local_workspace);