diff --git a/src/axom/sina/CMakeLists.txt b/src/axom/sina/CMakeLists.txt
index c7cb0c0827..b0d6f20ea0 100644
--- a/src/axom/sina/CMakeLists.txt
+++ b/src/axom/sina/CMakeLists.txt
@@ -45,15 +45,30 @@ set(sina_sources
core/Run.cpp
)
+# Set CMake Policy
+cmake_policy(SET CMP0115 NEW)
+
# Add Adiak header and source
blt_list_append( TO sina_headers ELEMENTS core/AdiakWriter.hpp IF AXOM_USE_ADIAK )
blt_list_append( TO sina_sources ELEMENTS core/AdiakWriter.cpp IF AXOM_USE_ADIAK )
# Add fortran interface for Sina
+if(AXOM_USE_HDF5 AND ENABLE_FORTRAN)
+ set(AXOM_USE_HDF5_FORTRAN ".true.")
+else()
+ set(AXOM_USE_HDF5_FORTRAN ".false.")
+endif()
+
if (ENABLE_FORTRAN)
- blt_list_append( TO sina_headers ELEMENTS interface/sina_fortran_interface.h)
- blt_list_append( TO sina_sources
- ELEMENTS interface/sina_fortran_interface.cpp interface/sina_fortran_interface.f)
+ blt_list_append(TO sina_headers
+ ELEMENTS interface/sina_fortran_interface.h)
+ blt_list_append(TO sina_sources
+ ELEMENTS interface/sina_fortran_interface.cpp interface/sina_fortran_interface.f)
+ configure_file(interface/sina_fortran_interface.f.in
+ interface/sina_fortran_interface.f
+ @ONLY)
+ install(FILES ${CMAKE_CURRENT_BINARY_DIR}/interface/sina_fortran_interface.f
+ DESTINATION interface)
endif()
#------------------------------------------------------------------------------
@@ -87,4 +102,4 @@ endif()
if(AXOM_ENABLE_EXAMPLES)
add_subdirectory(examples)
-endif()
+endif()
\ No newline at end of file
diff --git a/src/axom/sina/core/Document.cpp b/src/axom/sina/core/Document.cpp
index 83495e3a1f..dc98593a1c 100644
--- a/src/axom/sina/core/Document.cpp
+++ b/src/axom/sina/core/Document.cpp
@@ -12,9 +12,18 @@
*
******************************************************************************
*/
-
#include "axom/sina/core/Document.hpp"
+#include "axom/config.hpp"
+#include "axom/core/Path.hpp"
+#include "axom/core/utilities/StringUtilities.hpp"
+
+#include "conduit.hpp"
+#ifdef AXOM_USE_HDF5
+ #include "conduit_relay.hpp"
+ #include "conduit_relay_io.hpp"
+#endif
+
#include <cstdio>
#include <fstream>
#include <ios>
@@ -22,6 +31,7 @@
#include <utility>
#include <sstream>
#include <stdexcept>
+#include <algorithm>
namespace axom
{
@@ -35,6 +45,41 @@ char const RELATIONSHIPS_KEY[] = "relationships";
char const SAVE_TMP_FILE_EXTENSION[] = ".sina.tmp";
} // namespace
+void protocolWarn(std::string const protocol, std::string const &name)
+{
+ std::unordered_map<std::string, std::string> protocolMessages = {
+ {".json", ".json extension not found, did you mean to save to this format?"},
+ {".hdf5",
+ ".hdf5 extension not found, did you use one of its other supported types? "
+ "(h5, hdf, ...)"}};
+
+ Path path(name, '.');
+
+ if(protocol != '.' + path.baseName())
+ {
+ auto messageIt = protocolMessages.find(protocol);
+ if(messageIt != protocolMessages.end())
+ {
+ std::cerr << messageIt->second;
+ }
+ }
+}
+
+std::string get_supported_file_types()
+{
+ std::string types = "[";
+ for(size_t i = 0; i < supported_types.size(); ++i)
+ {
+ types += supported_types[i];
+ if(i < supported_types.size() - 1)
+ {
+ types += ", ";
+ }
+ }
+ types += "]";
+ return types;
+}
+
void Document::add(std::unique_ptr<Record> record)
{
records.emplace_back(std::move(record));
@@ -63,63 +108,184 @@ conduit::Node Document::toNode() const
return document;
}
-void Document::createFromNode(conduit::Node const &asNode,
- RecordLoader const &recordLoader)
+void Document::createFromNode(const conduit::Node &asNode,
+ const RecordLoader &recordLoader)
{
- if(asNode.has_child(RECORDS_KEY))
- {
- conduit::Node record_nodes = asNode[RECORDS_KEY];
- if(record_nodes.dtype().is_list())
+ conduit::Node nodeCopy = asNode;
+
+ auto processChildNodes = [&](const char *key,
+ std::function<void(conduit::Node &)> addFunc) {
+ if(nodeCopy.has_child(key))
{
- auto recordIter = record_nodes.children();
- while(recordIter.has_next())
+ conduit::Node &childNodes = nodeCopy[key];
+
+ // -- 1. Check if this node is a primitive leaf (throw immediately if so)
+ // Customize these checks to match exactly what you consider "primitive."
+ if(childNodes.dtype().is_number() || childNodes.dtype().is_char8_str() ||
+ childNodes.dtype().is_string())
+ {
+ std::ostringstream message;
+ message << "The '" << key
+ << "' element of a document cannot be a primitive value.";
+ throw std::invalid_argument(message.str());
+ }
+
+ // -- 2. Not a primitive. Check if it has no children.
+ if(childNodes.number_of_children() == 0)
{
- auto record = recordIter.next();
- add(recordLoader.load(record));
+ // Turn it into an empty list
+ childNodes.set(conduit::DataType::list());
+ }
+
+ // -- 3. If it's still not a list, throw
+ if(!childNodes.dtype().is_list())
+ {
+ std::ostringstream message;
+ message << "The '" << key
+ << "' element of a document must be an array/list.";
+ throw std::invalid_argument(message.str());
+ }
+
+ // -- 4. Now it's guaranteed to be a list, so iterate
+ auto childIter = childNodes.children();
+ while(childIter.has_next())
+ {
+ conduit::Node child = childIter.next();
+ addFunc(child);
}
}
- else
+ };
+ processChildNodes(RECORDS_KEY, [&](conduit::Node &record) {
+ add(recordLoader.load(record));
+ });
+
+ processChildNodes(RELATIONSHIPS_KEY, [&](conduit::Node &relationship) {
+ add(Relationship {relationship});
+ });
+}
+
+Document::Document(conduit::Node const &asNode, RecordLoader const &recordLoader)
+{
+ this->createFromNode(asNode, recordLoader);
+}
+
+Document::Document(std::string const &asJson, RecordLoader const &recordLoader)
+{
+ conduit::Node asNode;
+ asNode.parse(asJson, "json");
+ this->createFromNode(asNode, recordLoader);
+}
+
+#ifdef AXOM_USE_HDF5
+void removeSlashes(const conduit::Node &originalNode, conduit::Node &modifiedNode)
+{
+ for(auto it = originalNode.children(); it.has_next();)
+ {
+ it.next();
+ std::string key = it.name();
+ std::string modifiedKey =
+ axom::utilities::string::replaceAllInstances(key, "/", slashSubstitute);
+
+ modifiedNode[modifiedKey] = it.node();
+
+ if(it.node().dtype().is_object())
{
- std::ostringstream message;
- message << "The '" << RECORDS_KEY
- << "' element of a document must be an array";
- throw std::invalid_argument(message.str());
+ conduit::Node nestedNode;
+ removeSlashes(it.node(), nestedNode);
+ modifiedNode[modifiedKey].set(nestedNode);
}
}
+}
- if(asNode.has_child(RELATIONSHIPS_KEY))
+void restoreSlashes(const conduit::Node &modifiedNode, conduit::Node &restoredNode)
+{
+ // Check if List or Object, if its a list the else statement would turn it into an object
+ // which breaks the Document
+
+ if(modifiedNode.dtype().is_list())
{
- conduit::Node relationship_nodes = asNode[RELATIONSHIPS_KEY];
- if(relationship_nodes.dtype().is_list())
+ // If its empty with no children it's the end of a tree
+
+ for(auto it = modifiedNode.children(); it.has_next();)
{
- auto relationshipsIter = relationship_nodes.children();
- while(relationshipsIter.has_next())
+ it.next();
+ conduit::Node &newChild = restoredNode.append();
+
+ // Leaves empty nodes empty, if null data is set the
+ // Document breaks
+
+ if(it.node().dtype().is_string() || it.node().dtype().is_number())
{
- auto &relationship = relationshipsIter.next();
- add(Relationship {relationship});
+ newChild.set(it.node()); // Lists need .set
+ }
+
+ // Recursive Call
+ if(it.node().number_of_children() > 0)
+ {
+ restoreSlashes(it.node(), newChild);
}
}
- else
+ }
+ else
+ {
+ for(auto it = modifiedNode.children(); it.has_next();)
{
- std::ostringstream message;
- message << "The '" << RELATIONSHIPS_KEY
- << "' element of a document must be an array";
- throw std::invalid_argument(message.str());
+ it.next();
+ std::string key = it.name();
+ std::string restoredKey =
+ axom::utilities::string::replaceAllInstances(key, slashSubstitute, "/");
+
+ // Initialize a new node for the restored key
+ conduit::Node &newChild = restoredNode.add_child(restoredKey);
+
+ // Leaves empty keys empty but continues recursive call if its a list
+ if(it.node().dtype().is_string() || it.node().dtype().is_number() ||
+ it.node().dtype().is_object())
+ {
+ newChild.set(it.node());
+ }
+ else if(it.node().dtype().is_list())
+ {
+ restoreSlashes(it.node(), newChild); // Handle nested lists
+ }
+
+ // If the node has children, recursively restore them
+ if(it.node().number_of_children() > 0)
+ {
+ conduit::Node nestedNode;
+ restoreSlashes(it.node(), nestedNode);
+ newChild.set(nestedNode);
+ }
}
}
}
-Document::Document(conduit::Node const &asNode, RecordLoader const &recordLoader)
+void Document::toHDF5(const std::string &filename) const
{
- this->createFromNode(asNode, recordLoader);
-}
+ conduit::Node node;
+ conduit::Node &recordsNode = node["records"];
+ conduit::Node &relationshipsNode = node["relationships"];
-Document::Document(std::string const &asJson, RecordLoader const &recordLoader)
-{
- conduit::Node asNode;
- asNode.parse(asJson, "json");
- this->createFromNode(asNode, recordLoader);
+ for(const auto &record : getRecords())
+ {
+ conduit::Node recordNode = record->toNode();
+
+ removeSlashes(recordNode, recordsNode.append());
+ }
+
+ // Process relationships
+ for(const auto &relationship : getRelationships())
+ {
+ conduit::Node relationshipNode = relationship.toNode();
+
+ removeSlashes(relationshipNode, relationshipsNode.append());
+ }
+
+ conduit::relay::io::save(node, filename, "hdf5");
}
+#endif
+
+//
std::string Document::toJson(conduit::index_t indent,
conduit::index_t depth,
@@ -129,7 +295,9 @@ std::string Document::toJson(conduit::index_t indent,
return this->toNode().to_json("json", indent, depth, pad, eoe);
}
-void saveDocument(Document const &document, std::string const &fileName)
+void saveDocument(Document const &document,
+ std::string const &fileName,
+ Protocol protocol)
{
// It is a common use case for users to want to overwrite their files as
// the simulation progresses. However, this operation should be atomic so
@@ -138,12 +306,33 @@ void saveDocument(Document const &document, std::string const &fileName)
// file is in the same directory to ensure that it is part of the same
// file system as the destination file so that the move operation is
// atomic.
+
std::string tmpFileName = fileName + SAVE_TMP_FILE_EXTENSION;
- auto asJson = document.toJson();
- std::ofstream fout {tmpFileName};
- fout.exceptions(std::ostream::failbit | std::ostream::badbit);
- fout << asJson;
- fout.close();
+
+ if(protocol == Protocol::JSON)
+ {
+ protocolWarn(".json", fileName);
+ auto asJson = document.toJson();
+ std::ofstream fout {tmpFileName};
+ fout.exceptions(std::ostream::failbit | std::ostream::badbit);
+ fout << asJson;
+ fout.close();
+ }
+#ifdef AXOM_USE_HDF5
+ else if(protocol == Protocol::HDF5)
+ {
+ protocolWarn(".hdf5", fileName);
+ document.toHDF5(tmpFileName);
+ }
+#endif
+ else
+ {
+ std::ostringstream message;
+ message << "Invalid format choice. Please choose from one of the supported "
+ "protocols: "
+ << get_supported_file_types();
+ throw std::invalid_argument(message.str());
+ }
if(rename(tmpFileName.c_str(), fileName.c_str()) != 0)
{
@@ -154,20 +343,42 @@ void saveDocument(Document const &document, std::string const &fileName)
}
}
-Document loadDocument(std::string const &path)
+Document loadDocument(std::string const &path, Protocol protocol)
{
- return loadDocument(path, createRecordLoaderWithAllKnownTypes());
+ return loadDocument(path, createRecordLoaderWithAllKnownTypes(), protocol);
}
-Document loadDocument(std::string const &path, RecordLoader const &recordLoader)
+Document loadDocument(std::string const &path,
+ RecordLoader const &recordLoader,
+ Protocol protocol)
{
- conduit::Node nodeFromJson;
- std::ifstream file_in {path};
+ conduit::Node node, modifiedNode;
std::ostringstream file_contents;
- file_contents << file_in.rdbuf();
- file_in.close();
- nodeFromJson.parse(file_contents.str(), "json");
- return Document {nodeFromJson, recordLoader};
+ std::ifstream file_in {path};
+
+ // Load the file depending on the protocol
+ switch(protocol)
+ {
+ case Protocol::JSON:
+ file_contents << file_in.rdbuf();
+ file_in.close();
+ node.parse(file_contents.str(), "json");
+ return Document {node, recordLoader};
+#ifdef AXOM_USE_HDF5
+ case Protocol::HDF5:
+ file_in.close();
+ conduit::relay::io::load(path, "hdf5", node);
+ restoreSlashes(node, modifiedNode);
+ return Document {modifiedNode, recordLoader};
+#endif
+ default:
+ std::ostringstream message;
+ message << "Invalid format choice. Please choose from one of the supported "
+ "protocols: "
+ << get_supported_file_types();
+ throw std::invalid_argument(message.str());
+ break;
+ }
}
} // namespace sina
diff --git a/src/axom/sina/core/Document.hpp b/src/axom/sina/core/Document.hpp
index 04e19c5e7f..d179982448 100644
--- a/src/axom/sina/core/Document.hpp
+++ b/src/axom/sina/core/Document.hpp
@@ -16,13 +16,14 @@
******************************************************************************
*/
-#include <memory>
-#include <vector>
+#include "axom/config.hpp"
+#include "axom/sina/core/Record.hpp"
+#include "axom/sina/core/Relationship.hpp"
#include "conduit.hpp"
-#include "axom/sina/core/Record.hpp"
-#include "axom/sina/core/Relationship.hpp"
+#include <memory>
+#include <vector>
#define SINA_FILE_FORMAT_VERSION_MAJOR 1
#define SINA_FILE_FORMAT_VERSION_MINOR 0
@@ -32,12 +33,30 @@ namespace axom
namespace sina
{
+enum class Protocol
+{
+ JSON,
+ HDF5
+};
+
+const std::vector<std::string> supported_types = {"JSON",
+#ifdef AXOM_USE_HDF5
+ "HDF5"
+#endif
+};
+
+/**
+ * \brief The string used to replace '/' in parent node names when saving to HDF5.
+ */
+const std::string slashSubstitute = "__SINA_SLASHREPLACE__";
+
/**
- * \brief An object representing the top-level object of a Sina JSON file
+ * \brief An object representing the top-level object of a Sina file
*
- * A Document represents the top-level object of a JSON file conforming to the
+ * A Document represents the top-level object of a file conforming to the
* Sina schema. When serialized, these documents can be ingested into a
- * Sina database and used with the Sina tool.
+ * Sina database and used with the Sina tool. Sina files are defaulted to
+ * JSON but optionally support HDF5.
*
* Documents contain at most two objects: a list of Records and a list of Relationships. A simple, empty document:
* \code{.json}
@@ -75,6 +94,13 @@ namespace sina
* \code
* axom::sina::saveDocument(myDocument, "path/to/outfile.json")
* \endcode
+ *
+ * Loading and Saving documents will default to the JSON file type, but if an optional file type is
+ * loaded the Protocol parameter will control your file type. For example with HDF5:
+ * \code
+ * axom::sina::Document myDocument = axom::sina::loadDocument("path/to/infile.hdf5, Protocol::HDF5");
+ * axom::sina::saveDocument(myDocument, "path/to/outfile.hdf5", Protocol::HDF5)
+ * \endcode
*
* Check the Sina file format version with:
* \code
@@ -177,6 +203,15 @@ class Document
*/
conduit::Node toNode() const;
+#ifdef AXOM_USE_HDF5
+ /**
+ * \brief Dump this document as an HDF5 File
+ *
+ * \param filename the location of which to save the file
+ */
+ void toHDF5(const std::string &filename) const;
+#endif
+
/**
* \brief Convert this document to a JSON string.
*
@@ -187,6 +222,13 @@ class Document
const std::string &pad = "",
const std::string &eoe = "") const;
+ /**
+ * \brief Get the list of file types currently supported by the implementation.
+ *
+ * \return a string of supported file types
+ */
+ std::string get_supported_file_types();
+
private:
/**
* Constructor helper method, extracts info from a conduit Node.
@@ -200,12 +242,16 @@ class Document
/**
* \brief Save the given Document to the specified location. If the given file exists,
* it will be overwritten.
- *
+ *
* \param document the Document to save
- * \param fileName the location to which to save the file
+ * \param fileName the location of which to save the file
+ * \param protocol the file type requested to save as contained in supported_types, default = JSON
* \throws std::ios::failure if there are any IO errors
+ * std::invalid_argument if the protocol given is an undefined, optional protocol
*/
-void saveDocument(Document const &document, std::string const &fileName);
+void saveDocument(Document const &document,
+ std::string const &fileName,
+ Protocol protocol = Protocol::JSON);
/**
* \brief Get the current file format version.
@@ -223,9 +269,11 @@ inline std::string getSinaFileFormatVersion()
* knows about will be able to be loaded.
*
* \param path the file system path from which to load the document
+ * \param protocol the type of file being loaded, default = JSON
* \return the loaded Document
*/
-Document loadDocument(std::string const &path);
+Document loadDocument(std::string const &path,
+ Protocol protocol = Protocol::JSON);
/**
* \brief Load a document from the given path.
@@ -233,9 +281,13 @@ Document loadDocument(std::string const &path);
* \param path the file system path from which to load the document
* \param recordLoader the RecordLoader to use to load the different types
* of records
+ * \param protocol the type of file being loaded, default = JSON
+ * \throws std::invalid_argument if the protocol given is an undefined, optional protocol
* \return the loaded Document
*/
-Document loadDocument(std::string const &path, RecordLoader const &recordLoader);
+Document loadDocument(std::string const &path,
+ RecordLoader const &recordLoader,
+ Protocol protocol = Protocol::JSON);
} // namespace sina
} // namespace axom
diff --git a/src/axom/sina/docs/imgs/JSON_vs_HDF5_size.png b/src/axom/sina/docs/imgs/JSON_vs_HDF5_size.png
new file mode 100644
index 0000000000..c028463e9c
Binary files /dev/null and b/src/axom/sina/docs/imgs/JSON_vs_HDF5_size.png differ
diff --git a/src/axom/sina/docs/imgs/JSON_vs_HDF5_speed.png b/src/axom/sina/docs/imgs/JSON_vs_HDF5_speed.png
new file mode 100644
index 0000000000..b8c8f26448
Binary files /dev/null and b/src/axom/sina/docs/imgs/JSON_vs_HDF5_speed.png differ
diff --git a/src/axom/sina/docs/sphinx/documents.rst b/src/axom/sina/docs/sphinx/documents.rst
index 64fa8ed775..2a0d270a32 100644
--- a/src/axom/sina/docs/sphinx/documents.rst
+++ b/src/axom/sina/docs/sphinx/documents.rst
@@ -106,6 +106,30 @@ of the ``Document`` that way:
std::cout << myDocument.toJson() << std::endl;
}
+
+------------------------------
+Generating Documents From HDF5
+------------------------------
+
+In addition to assembling ``Document`` instances from existing JSON files, it
+is possible to generate ``Document`` objects from existing HDF5 files using
+conduit.
+
+Sina's ``saveDocument()`` and ``loadDocument()`` functions support HDF5 assembly if we both
+build axom with HDF5 support and provide it with the optional Protocol variable set to HDF5.
+If Protocol::HDF5 is attempted while axom does not have HDF5 support, the functions will
+recognize they won't have the support for what you are attempting and will return a list of
+supported types instead as a runtime error.
+
+.. code:: cpp
+
+ #include "axom/sina.hpp"
+
+ int main (void) {
+ axom::sina::Document myDocument = axom::sina::loadDocument("MySinaData.hdf5", axom::sina::Protocol::HDF5);
+ }
+
+
---------------------------------------------------------
Obtaining Records & Relationships from Existing Documents
---------------------------------------------------------
diff --git a/src/axom/sina/docs/sphinx/hdf5_vs_json.rst b/src/axom/sina/docs/sphinx/hdf5_vs_json.rst
new file mode 100644
index 0000000000..f33679fb36
--- /dev/null
+++ b/src/axom/sina/docs/sphinx/hdf5_vs_json.rst
@@ -0,0 +1,55 @@
+.. ## Copyright (c) 2017-2025, Lawrence Livermore National Security, LLC and
+.. ## other Axom Project Developers. See the top-level LICENSE file for details.
+.. ##
+.. ## SPDX-License-Identifier: (BSD-3-Clause)
+
+.. _curvesets-label:
+
+==========
+HDF5 and JSON
+==========
+
+Sina's C++ and Python Code supports loading both JSON and HDF5 files as Documents,
+and saving Documents to either file type.
+
+Both options offer differing strengths and weaknesses when compared to each other,
+the following data on the size and speed has been compiled to give a better idea
+of which file type fits your requirements.
+
+==========
+Why You Should Use JSON
+==========
+
+* JSON is more universally parsable
+* JSON code is human readable allowing you more, easier flexibility with searching
+for relevent data. This is most applicable with smaller files since large amounts
+of data or curve sets quickly become more efficient to navigate digitally
+* JSON outperforms HDF5 in speed and size efficiency when dealing with non-curve set data
+and outperforms at smaller curve set sizes (just before :math:`10^2` Curve Sets for Size and
+around :math:`10^3.25` for Speed)
+
+==========
+Why You Should Use HDF5
+==========
+
+* HDF5 offers better size and speed efficiency when dealing with larger files/curve sets
+and only outperforms more dramatically as size increases
+* Hierarchical structure leads to being 2.5x more size efficient and 5x faster at our largest
+tested files
+
+==========
+Images
+==========
+
+JSON vs HDF5 Size
+--------
+
+.. image:: ../imgs/JSON_vs_HDF5_size.png
+ :alt: JSON Size / HDF5 Size when dealing with 10 - :math:`10^4` curves with 10 - :math:`10^4` items
+
+
+JSON vs HDF5 Speed
+--------
+
+.. image:: ../imgs/JSON_vs_HDF5_speed.png
+ :alt: JSON Size / HDF5 Size when dealing with 10 - :math:`10^4` curves with 10 - :math:`10^4` items
\ No newline at end of file
diff --git a/src/axom/sina/docs/sphinx/tutorial.rst b/src/axom/sina/docs/sphinx/tutorial.rst
index 6be1fd62e9..da9c2af7f1 100644
--- a/src/axom/sina/docs/sphinx/tutorial.rst
+++ b/src/axom/sina/docs/sphinx/tutorial.rst
@@ -53,7 +53,7 @@ Adding Data
Once we have a Record, we can add different types of data to it. Any Datum
object that is added will end up in the "data" section of the record in
-the JSON file.
+the output file.
.. literalinclude:: ../../examples/sina_tutorial.cpp
:language: cpp
@@ -146,9 +146,12 @@ users will be able to search for "temperature" (value = 450),
Input and Output
----------------
-Once you have a document, it is easy to save it to a file. After executing
-the below, your will output a file named "my_output.json" which you can ingest
-into a Sina datastore.
+Once you have a document, it is easy to save it to a file. To save to a JSON, we
+run the saveDocument() with the optional argument Protocol set to JSON or set as
+nothing. Alternatively if you wish to save the document to an HDF5 file: Configure
+axom for HDF5 support then you can set saveDocument()'s optional Protocol parameter
+to HDF5. After executing the below, you will output a file named "my_output.json"
+and a file named "my_output.hdf5", both of which you can ingest into a Sina datastore.
.. literalinclude:: ../../examples/sina_tutorial.cpp
:language: cpp
@@ -157,7 +160,15 @@ into a Sina datastore.
If needed, you can also load a document from a file. This can be useful,
for example, if you wrote a document when writing a restart and you want to
-continue from where you left off.
+continue from where you left off. To load from a JSON file simply run loadDocument()
+with the optional argument Protocol set to JSON or set as nothing. If you've configured
+for, and wish to load from an HDF5 simply set the Protocol to HDF5.
+
+Note that due to HDF5's handling of '/' as indicators for nested structures,
+parent nodes will have '/' changed to the ``slashSubstitute`` variable located in
+``axom/sina/core/Document.hpp`` as an HDF5 with saveDocument(). loadDocument()
+will restore them to normal:
+
.. literalinclude:: ../../examples/sina_tutorial.cpp
:language: cpp
@@ -184,3 +195,4 @@ convert to and from JSON. The user-defined section is exposed as a
:language: cpp
:start-after: //! [begin user defined]
:end-before: //! [end user defined]
+
diff --git a/src/axom/sina/examples/CMakeLists.txt b/src/axom/sina/examples/CMakeLists.txt
index 93e307e99f..f9904282a7 100644
--- a/src/axom/sina/examples/CMakeLists.txt
+++ b/src/axom/sina/examples/CMakeLists.txt
@@ -27,7 +27,7 @@ set(sina_example_sources
sina_view_datum_values.cpp
)
-set(sina_example_depends sina conduit slic)
+set(sina_example_depends sina conduit::conduit slic)
if (ENABLE_FORTRAN)
blt_list_append( TO sina_example_sources ELEMENTS sina_fortran.f)
@@ -38,16 +38,26 @@ endif()
#------------------------------------------------------------------------------
foreach(src ${sina_example_sources})
get_filename_component(exe_name ${src} NAME_WE)
+
+ if (exe_name STREQUAL "sina_fortran")
+ set(sina_mod_src ${CMAKE_CURRENT_BINARY_DIR}/../interface/sina_fortran_interface.f)
+ else()
+ set(sina_mod_src "")
+ endif()
+
axom_add_executable(
NAME ${exe_name}_ex
- SOURCES ${src}
+ SOURCES ${src} ${sina_mod_src}
OUTPUT_DIR ${EXAMPLE_OUTPUT_DIRECTORY}
DEPENDS_ON ${sina_example_depends}
FOLDER axom/sina/examples
- )
+ )
+
+ if (exe_name STREQUAL "sina_fortran")
+ target_include_directories(${exe_name}_ex PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/../interface)
+ endif()
- # Need to add this flag so XL will ignore trailing underscores in fortran function names
if (${exe_name}_ex STREQUAL "sina_fortran_ex" AND CMAKE_Fortran_COMPILER_ID STREQUAL "XL")
target_compile_options(${exe_name}_ex PRIVATE -qextname)
endif()
-endforeach()
+endforeach()
\ No newline at end of file
diff --git a/src/axom/sina/examples/sina_basic.cpp b/src/axom/sina/examples/sina_basic.cpp
index eba21c0913..4e2e937b61 100644
--- a/src/axom/sina/examples/sina_basic.cpp
+++ b/src/axom/sina/examples/sina_basic.cpp
@@ -3,6 +3,7 @@
//
// SPDX-License-Identifier: (BSD-3-Clause)
+#include "axom/config.hpp"
#include "axom/sina.hpp"
int main(void)
@@ -17,5 +18,12 @@ int main(void)
// Add the run to the document
document.add(std::move(run));
// Save the document directly to a file.
- axom::sina::saveDocument(document, "MySinaData.json");
-}
\ No newline at end of file
+ // by specifying Protocol::JSON, we set the file to save as a JSON.
+ // if we wished, we could provide no third argument since saveDocument defaults to JSON.
+ axom::sina::saveDocument(document, "MySinaData.json", axom::sina::Protocol::JSON);
+
+#ifdef AXOM_USE_HDF5
+ // by specifying Protocol::HDF5, we also save a copy as an HDF5 file.
+ axom::sina::saveDocument(document, "MySinaData.hdf5", axom::sina::Protocol::HDF5);
+#endif
+}
diff --git a/src/axom/sina/examples/sina_document_assembly.cpp b/src/axom/sina/examples/sina_document_assembly.cpp
index b8ceb44548..6dc4002254 100644
--- a/src/axom/sina/examples/sina_document_assembly.cpp
+++ b/src/axom/sina/examples/sina_document_assembly.cpp
@@ -3,6 +3,7 @@
//
// SPDX-License-Identifier: (BSD-3-Clause)
+#include "axom/config.hpp"
#include "axom/sina.hpp"
int main(void)
@@ -32,5 +33,12 @@ int main(void)
document.add(relationship);
// Save the document directly to a file.
+ // since we gave saveDocument no optional protocol parameter, it will default to JSON
axom::sina::saveDocument(document, "MySinaData.json");
-}
\ No newline at end of file
+
+#ifdef AXOM_USE_HDF5
+ // We will also save a copy of the document as an HDF5 file
+ // which can be done by passing the protocol as HDF5
+ axom::sina::saveDocument(document, "MySinaData.hdf5", axom::sina::Protocol::HDF5);
+#endif
+}
diff --git a/src/axom/sina/examples/sina_fortran.f b/src/axom/sina/examples/sina_fortran.f
index 8dd2a5e0d4..021e3cd0ac 100644
--- a/src/axom/sina/examples/sina_fortran.f
+++ b/src/axom/sina/examples/sina_fortran.f
@@ -1,5 +1,6 @@
program example
use sina_functions
+ use hdf5_config
implicit none
! data types
@@ -28,6 +29,7 @@ program example
character(:), allocatable :: tag
character(:), allocatable :: units
character(20) :: json_fn
+ character(20) :: hdf5_fn
character(15) :: name
character(25) :: curve
@@ -56,6 +58,9 @@ program example
full_path = make_cstring(wrk_dir//''//fle_nme)
ofull_path = make_cstring(wrk_dir//''//ofle_nme)
json_fn = make_cstring('sina_dump.json')
+ if (use_hdf5) then
+ hdf5_fn = make_cstring('sina_dump.hdf5')
+ end if
mime_type = make_cstring('')
@@ -149,6 +154,9 @@ program example
! write out the Sina Document
print *,'Writing out the Sina Document'
call write_sina_document(json_fn)
+ if (use_hdf5) then
+ call write_sina_document(hdf5_fn, 1)
+ end if
contains
diff --git a/src/axom/sina/examples/sina_tutorial.cpp b/src/axom/sina/examples/sina_tutorial.cpp
index e03be73513..92e5ccac11 100644
--- a/src/axom/sina/examples/sina_tutorial.cpp
+++ b/src/axom/sina/examples/sina_tutorial.cpp
@@ -3,6 +3,7 @@
//
// SPDX-License-Identifier: (BSD-3-Clause)
+#include "axom/config.hpp"
#include "axom/sina.hpp"
#include <utility>
@@ -130,13 +131,20 @@ void gatherAllData(axom::sina::Record &record)
void save(axom::sina::Document const &doc)
{
axom::sina::saveDocument(doc, "my_output.json");
+#ifdef AXOM_USE_HDF5
+ axom::sina::saveDocument(doc, "my_output.hdf5", axom::sina::Protocol::HDF5);
+#endif
}
//! [end io write]
//! [begin io read]
void load()
{
- axom::sina::Document doc = axom::sina::loadDocument("my_output.json");
+ axom::sina::Document doc1 = axom::sina::loadDocument("my_output.json");
+#ifdef AXOM_USE_HDF5
+ axom::sina::Document doc2 =
+ axom::sina::loadDocument("my_output.hdf5", axom::sina::Protocol::HDF5);
+#endif
}
//! [end io read]
diff --git a/src/axom/sina/interface/sina_fortran_interface.cpp b/src/axom/sina/interface/sina_fortran_interface.cpp
index 6b15d94a93..9ac01a6abb 100644
--- a/src/axom/sina/interface/sina_fortran_interface.cpp
+++ b/src/axom/sina/interface/sina_fortran_interface.cpp
@@ -258,7 +258,18 @@ extern "C" void sina_add_file_(char *filename, char *mime_type)
}
}
-extern "C" void write_sina_document_(char *input_fn)
+extern "C" void write_sina_document_protocol_(char *input_fn, int *protocol)
+{
+ std::string filename(input_fn);
+ axom::sina::Protocol proto = static_cast<axom::sina::Protocol>(*protocol);
+ // Save everything
+ if(sina_document)
+ {
+ axom::sina::saveDocument(*sina_document, filename.c_str(), proto);
+ }
+}
+
+extern "C" void write_sina_document_noprotocol_(char *input_fn)
{
std::string filename(input_fn);
// Save everything
diff --git a/src/axom/sina/interface/sina_fortran_interface.f b/src/axom/sina/interface/sina_fortran_interface.f.in
similarity index 74%
rename from src/axom/sina/interface/sina_fortran_interface.f
rename to src/axom/sina/interface/sina_fortran_interface.f.in
index 56fe05ebe6..320c8ada8a 100644
--- a/src/axom/sina/interface/sina_fortran_interface.f
+++ b/src/axom/sina/interface/sina_fortran_interface.f.in
@@ -1,5 +1,8 @@
module sina_functions
+ integer, parameter :: JSON = 0
+ integer, parameter :: HDF5 = 1
+
interface
subroutine create_document_and_record(id)
@@ -17,14 +20,6 @@ end subroutine sina_add_file
end interface
- interface
-
- subroutine write_sina_document(file_nm)
- character(*) file_nm
- end subroutine write_sina_document
-
- end interface
-
interface sina_add
subroutine sina_add_long(key, value, units, tags)
@@ -114,5 +109,44 @@ subroutine sina_add_curve_long(name, curve, values, n, independent)
end subroutine sina_add_curve_long
end interface
+
+ interface
+
+ subroutine write_sina_document_protocol(file_nm, protocol)
+ character(*) file_nm
+ integer protocol
+ end subroutine write_sina_document_protocol
+
+ end interface
+
+ interface
+
+ subroutine write_sina_document_noprotocol(file_nm)
+ character(*) file_nm
+ end subroutine write_sina_document_noprotocol
+
+ end interface
+
+
+ interface write_sina_document
+ module procedure save_with_protocol
+ module procedure save_without_protocol
+ end interface
+
+contains
+ subroutine save_with_protocol(fname, proto)
+ character(*) fname
+ integer proto
+ call write_sina_document_protocol(fname, proto)
+ end subroutine save_with_protocol
+ subroutine save_without_protocol(fname)
+ character(*) fname
+ call write_sina_document_noprotocol(fname)
+ end subroutine save_without_protocol
+
+end module
-end module
\ No newline at end of file
+module hdf5_config
+ implicit none
+ logical, parameter :: use_hdf5 = @AXOM_USE_HDF5_FORTRAN@
+end module hdf5_config
\ No newline at end of file
diff --git a/src/axom/sina/interface/sina_fortran_interface.h b/src/axom/sina/interface/sina_fortran_interface.h
index 3d06d7d7f9..0a11010c9c 100644
--- a/src/axom/sina/interface/sina_fortran_interface.h
+++ b/src/axom/sina/interface/sina_fortran_interface.h
@@ -3,9 +3,6 @@
//
// SPDX-License-Identifier: (BSD-3-Clause)
-#include "axom/sina/core/Document.hpp"
-#include "axom/sina/core/Record.hpp"
-#include "axom/sina/core/Run.hpp"
#include "axom/sina.hpp"
extern "C" char *Get_File_Extension(char *);
@@ -13,7 +10,8 @@ extern "C" void create_document_and_run_(char *);
extern "C" axom::sina::Record *Sina_Get_Run();
extern "C" void sina_add_file_to_record_(char *);
extern "C" void sina_add_file_with_mimetype_to_record_(char *, char *);
-extern "C" void write_sina_document_(char *);
+extern "C" void write_sina_document_protocol_(char *, int *);
+extern "C" void write_sina_document_noprotocol_(char *);
extern "C" void sina_add_long_(char *, long long int *, char *, char *);
extern "C" void sina_add_int_(char *, int *, char *, char *);
extern "C" void sina_add_float_(char *, float *, char *, char *);
diff --git a/src/axom/sina/tests/CMakeLists.txt b/src/axom/sina/tests/CMakeLists.txt
index 109572e3b2..34cb313402 100644
--- a/src/axom/sina/tests/CMakeLists.txt
+++ b/src/axom/sina/tests/CMakeLists.txt
@@ -110,6 +110,20 @@ if (ENABLE_FORTRAN AND DEFINED PYTHON_EXECUTABLE)
COPYONLY
)
+ # Define Python-friendly boolean for HDF5 guarding in Fortran Test.
+ if(AXOM_USE_HDF5)
+ set(AXOM_USE_HDF5_PY "True")
+ else()
+ set(AXOM_USE_HDF5_PY "False")
+ endif()
+
+ # Generate config.py in the test output directory
+ configure_file(
+ ${CMAKE_CURRENT_SOURCE_DIR}/config.py.in
+ ${TEST_OUTPUT_DIRECTORY}/config.py
+ @ONLY
+ )
+
axom_add_test( NAME sina_fortran_integration_test
COMMAND ${PYTHON_EXECUTABLE} ${TEST_OUTPUT_DIRECTORY}/test_fortran_integration.py -bd ${PROJECT_BINARY_DIR}
)
diff --git a/src/axom/sina/tests/config.py.in b/src/axom/sina/tests/config.py.in
new file mode 100644
index 0000000000..b6d0a41e78
--- /dev/null
+++ b/src/axom/sina/tests/config.py.in
@@ -0,0 +1,6 @@
+# Copyright (c) 2017-2025, Lawrence Livermore National Security, LLC and
+# other Axom Project Developers. See the top-level LICENSE file for details.
+#
+# SPDX-License-Identifier: (BSD-3-Clause)
+
+AXOM_USE_HDF5 = @AXOM_USE_HDF5_PY@
\ No newline at end of file
diff --git a/src/axom/sina/tests/sina_Document.cpp b/src/axom/sina/tests/sina_Document.cpp
index 81cf0caa41..74f8278005 100644
--- a/src/axom/sina/tests/sina_Document.cpp
+++ b/src/axom/sina/tests/sina_Document.cpp
@@ -3,21 +3,26 @@
//
// SPDX-License-Identifier: (BSD-3-Clause)
+#include "axom/config.hpp"
+#include "axom/core/utilities/FileUtilities.hpp"
+
+#include "axom/sina/core/Document.hpp"
+#include "axom/sina/core/Run.hpp"
+#include "axom/sina/tests/TestRecord.hpp"
+
+#include "conduit.hpp"
+#include "conduit_relay.hpp"
+#include "conduit_relay_io.hpp"
+
+#include "gtest/gtest.h"
+#include "gmock/gmock.h"
+
#include <cstdio>
#include <fstream>
#include <iostream>
#include <type_traits>
#include <utility>
-#include "gtest/gtest.h"
-#include "gmock/gmock.h"
-
-#include "axom/core/utilities/FileUtilities.hpp"
-#include "axom/sina/core/Document.hpp"
-#include "axom/sina/core/Run.hpp"
-
-#include "axom/sina/tests/TestRecord.hpp"
-
namespace axom
{
namespace sina
@@ -34,6 +39,115 @@ char const TEST_RECORD_TYPE[] = "test type";
char const EXPECTED_RECORDS_KEY[] = "records";
char const EXPECTED_RELATIONSHIPS_KEY[] = "relationships";
+// Large JSONs Used For JSON and HDF5 Save Tests
+std::string data_json = R"(
+{
+ "records": [
+ {
+ "type": "run",
+ "application": "test",
+ "id": "test_1",
+ "data": {
+ "int": {
+ "value": 500,
+ "units": "miles"
+ },
+ "str/ings": {
+ "value": ["z", "o", "o"]
+ }
+ },
+ "files": {
+ "test/test.png": {}
+ }
+ }
+ ]
+}
+)";
+
+std::string long_json = R"(
+{
+ "records": [
+ {
+ "type": "foo",
+ "id": "test_1",
+ "user_defined": {
+ "name": "bob"
+ },
+ "files": {
+ "foo/bar.png": {
+ "mimetype": "image"
+ }
+ },
+ "data": {
+ "scalar": {
+ "value": 500,
+ "units": "miles"
+ }
+ }
+ },
+ {
+ "type": "bar",
+ "id": "test_2",
+ "data": {
+ "scalar_list": {
+ "value": [1, 2, 3]
+ },
+ "string_list": {
+ "value": ["a", "wonderful", "world"],
+ "tags": ["observation"]
+ }
+ }
+ },
+ {
+ "type": "run",
+ "application": "sina_test",
+ "id": "test_3",
+ "data": {
+ "scalar": {
+ "value": 12.3,
+ "units": "g/s",
+ "tags": ["hi"]
+ },
+ "scalar_list": {
+ "value": [1, 2, 3.0, 4]
+ }
+ }
+ },
+ {
+ "type": "bar",
+ "id": "test_4",
+ "data": {
+ "string": {
+ "value": "yarr"
+ },
+ "string_list": {
+ "value": ["y", "a", "r"]
+ }
+ },
+ "files": {
+ "test/test.png": {}
+ },
+ "user_defined": {
+ "hello": "there"
+ }
+ }
+ ],
+ "relationships": [
+ {
+ "predicate": "completes",
+ "subject": "test_2",
+ "object": "test_1"
+ },
+ {
+ "subject": "test_3",
+ "predicate": "overrides",
+ "object": "test_4"
+ }
+ ]
+}
+)";
+
+// Tests
TEST(Document, create_fromNode_empty)
{
conduit::Node documentAsNode;
@@ -109,129 +223,6 @@ TEST(Document, create_fromNode_withRelationships)
EXPECT_EQ("is related to", relationships[0].getPredicate());
}
-TEST(Document, create_fromJson_roundtrip)
-{
- std::string orig_json =
- "{\"records\": [{\"type\": \"test_rec\",\"id\": "
- "\"test\"}],\"relationships\": []}";
- axom::sina::Document myDocument =
- Document(orig_json, createRecordLoaderWithAllKnownTypes());
- EXPECT_EQ(0, myDocument.getRelationships().size());
- ASSERT_EQ(1, myDocument.getRecords().size());
- EXPECT_EQ("test_rec", myDocument.getRecords()[0]->getType());
- std::string returned_json = myDocument.toJson(0, 0, "", "");
- EXPECT_EQ(orig_json, returned_json);
-}
-
-TEST(Document, create_fromJson_full)
-{
- std::string long_json = R"({
- "records": [
- {
- "type": "foo",
- "id": "test_1",
- "user_defined": {
- "name": "bob"
- },
- "files": {
- "foo/bar.png": {
- "mimetype": "image"
- }
- },
- "data": {
- "scalar": {
- "value": 500,
- "units": "miles"
- }
- }
- },
- {
- "type": "bar",
- "id": "test_2",
- "data": {
- "scalar_list": {
- "value": [1, 2, 3]
- },
- "string_list": {
- "value": ["a", "wonderful", "world"],
- "tags": ["observation"]
- }
- }
- },
- {
- "type": "run",
- "application": "sina_test",
- "id": "test_3",
- "data": {
- "scalar": {
- "value": 12.3,
- "units": "g/s",
- "tags": ["hi"]
- },
- "scalar_list": {
- "value": [1, 2, 3.0, 4]
- }
- }
- },
- {
- "type": "bar",
- "id": "test_4",
- "data": {
- "string": {
- "value": "yarr"
- },
- "string_list": {
- "value": ["y", "a", "r"]
- }
- },
- "files": {
- "test/test.png": {}
- },
- "user_defined": {
- "hello": "there"
- }
- }
- ],
- "relationships": [
- {
- "predicate": "completes",
- "subject": "test_2",
- "object": "test_1"
- },
- {
- "subject": "test_3",
- "predicate": "overrides",
- "object": "test_4"
- }
- ]
- })";
- axom::sina::Document myDocument =
- Document(long_json, createRecordLoaderWithAllKnownTypes());
- EXPECT_EQ(2, myDocument.getRelationships().size());
- auto &records = myDocument.getRecords();
- EXPECT_EQ(4, records.size());
-}
-
-TEST(Document, create_fromJson_value_check)
-{
- std::string data_json =
- "{\"records\": [{\"type\": \"run\", \"application\":\"test\", \"id\": "
- "\"test_1\",\"data\":{\"int\": {\"value\": 500,\"units\": \"miles\"}, "
- "\"str/ings\": {\"value\":[\"z\", \"o\", \"o\"]}}, "
- "\"files\":{\"test/test.png\":{}}}]}";
- axom::sina::Document myDocument =
- Document(data_json, createRecordLoaderWithAllKnownTypes());
- EXPECT_EQ(0, myDocument.getRelationships().size());
- auto &records = myDocument.getRecords();
- EXPECT_EQ(1, records.size());
- EXPECT_EQ(records[0]->getType(), "run");
- auto &data = records[0]->getData();
- EXPECT_EQ(data.at("int").getScalar(), 500.0);
- std::vector<std::string> expected_string_vals = {"z", "o", "o"};
- EXPECT_EQ(data.at("str/ings").getStringArray(), expected_string_vals);
- EXPECT_EQ(records[0]->getFiles().count(File {"test/test.png"}), 1);
-}
-
TEST(Document, toNode_empty)
{
// A sina document should always have, at minimum, both records and
@@ -349,7 +340,45 @@ NamedTempFile::~NamedTempFile()
axom::utilities::filesystem::removeFile(fileName.data());
}
-TEST(Document, saveDocument)
+TEST(Document, create_fromJson_roundtrip_json)
+{
+ std::string orig_json =
+ "{\"records\": [{\"type\": \"test_rec\",\"id\": "
+ "\"test\"}],\"relationships\": []}";
+ axom::sina::Document myDocument =
+ Document(orig_json, createRecordLoaderWithAllKnownTypes());
+ EXPECT_EQ(0, myDocument.getRelationships().size());
+ ASSERT_EQ(1, myDocument.getRecords().size());
+ EXPECT_EQ("test_rec", myDocument.getRecords()[0]->getType());
+ std::string returned_json1 = myDocument.toJson(0, 0, "", "");
+ EXPECT_EQ(orig_json, returned_json1);
+}
+
+TEST(Document, create_fromJson_full_json)
+{
+ axom::sina::Document myDocument =
+ Document(long_json, createRecordLoaderWithAllKnownTypes());
+ EXPECT_EQ(2, myDocument.getRelationships().size());
+ auto &records1 = myDocument.getRecords();
+ EXPECT_EQ(4, records1.size());
+}
+
+TEST(Document, create_fromJson_value_check_json)
+{
+ axom::sina::Document myDocument =
+ Document(data_json, createRecordLoaderWithAllKnownTypes());
+ EXPECT_EQ(0, myDocument.getRelationships().size());
+ auto &records1 = myDocument.getRecords();
+ EXPECT_EQ(1, records1.size());
+ EXPECT_EQ(records1[0]->getType(), "run");
+ auto &data1 = records1[0]->getData();
+ EXPECT_EQ(data1.at("int").getScalar(), 500.0);
+ std::vector<std::string> expected_string_vals = {"z", "o", "o"};
+ EXPECT_EQ(data1.at("str/ings").getStringArray(), expected_string_vals);
+ EXPECT_EQ(records1[0]->getFiles().count(File {"test/test.png"}), 1);
+}
+
+TEST(Document, saveDocument_json)
{
NamedTempFile tmpFile;
@@ -433,6 +462,79 @@ TEST(Document, load_defaultRecordLoaders)
EXPECT_NE(nullptr, loadedRun);
}
+#ifdef AXOM_USE_HDF5
+TEST(Document, create_fromJson_roundtrip_hdf5)
+{
+ std::string orig_json =
+ "{\"records\": [{\"type\": \"test_rec\",\"id\": "
+ "\"test\"}],\"relationships\": []}";
+ axom::sina::Document myDocument =
+ Document(orig_json, createRecordLoaderWithAllKnownTypes());
+ saveDocument(myDocument, "round_json.hdf5", Protocol::HDF5);
+ Document loadedDocument = loadDocument("round_json.hdf5", Protocol::HDF5);
+ EXPECT_EQ(0, loadedDocument.getRelationships().size());
+ ASSERT_EQ(1, loadedDocument.getRecords().size());
+ EXPECT_EQ("test_rec", loadedDocument.getRecords()[0]->getType());
+ std::string returned_json2 = loadedDocument.toJson(0, 0, "", "");
+ EXPECT_EQ(orig_json, returned_json2);
+}
+
+TEST(Document, create_fromJson_full_hdf5)
+{
+ axom::sina::Document myDocument =
+ Document(long_json, createRecordLoaderWithAllKnownTypes());
+ saveDocument(myDocument, "long_json.hdf5", Protocol::HDF5);
+ Document loadedDocument = loadDocument("long_json.hdf5", Protocol::HDF5);
+ EXPECT_EQ(2, loadedDocument.getRelationships().size());
+ auto &records2 = loadedDocument.getRecords();
+ EXPECT_EQ(4, records2.size());
+}
+
+TEST(Document, create_fromJson_value_check_hdf5)
+{
+ axom::sina::Document myDocument =
+ Document(data_json, createRecordLoaderWithAllKnownTypes());
+ std::vector<std::string> expected_string_vals = {"z", "o", "o"};
+ saveDocument(myDocument, "data_json.hdf5", Protocol::HDF5);
+ Document loadedDocument = loadDocument("data_json.hdf5", Protocol::HDF5);
+ EXPECT_EQ(0, loadedDocument.getRelationships().size());
+ auto &records2 = loadedDocument.getRecords();
+ EXPECT_EQ(1, records2.size());
+ EXPECT_EQ(records2[0]->getType(), "run");
+ auto &data2 = records2[0]->getData();
+ EXPECT_EQ(data2.at("int").getScalar(), 500.0);
+ EXPECT_EQ(data2.at("str/ings").getStringArray(), expected_string_vals);
+ EXPECT_EQ(records2[0]->getFiles().count(File {"test/test.png"}), 1);
+}
+
+TEST(Document, saveDocument_hdf5)
+{
+ NamedTempFile tmpFile;
+
+ // First, write some random stuff to the temp file to make sure it is
+ // overwritten.
+ {
+ std::ofstream fout {tmpFile.getName()};
+ fout << "Initial contents";
+ }
+
+ Document document;
+ document.add(
+ std::make_unique<Record>(ID {"the id", IDType::Global}, "the type"));
+
+ saveDocument(document, tmpFile.getName(), Protocol::HDF5);
+
+ conduit::Node readContents;
+ conduit::relay::io::load(tmpFile.getName(), "hdf5", readContents);
+
+ ASSERT_TRUE(readContents[EXPECTED_RECORDS_KEY].dtype().is_list());
+ EXPECT_EQ(1, readContents[EXPECTED_RECORDS_KEY].number_of_children());
+ auto &readRecord = readContents[EXPECTED_RECORDS_KEY][0];
+ EXPECT_EQ("the id", readRecord["id"].as_string());
+ EXPECT_EQ("the type", readRecord["type"].as_string());
+}
+#endif
+
} // namespace
} // namespace testing
} // namespace sina
diff --git a/src/axom/sina/tests/test_fortran_integration.py b/src/axom/sina/tests/test_fortran_integration.py
index 08cb20cf15..a40e102867 100644
--- a/src/axom/sina/tests/test_fortran_integration.py
+++ b/src/axom/sina/tests/test_fortran_integration.py
@@ -4,6 +4,10 @@
import os
import subprocess
import unittest
+import config
+
+if (config.AXOM_USE_HDF5):
+ import h5py
def parse_args():
@@ -14,28 +18,33 @@ def parse_args():
# Add other arguments as needed
return parser.parse_args()
-
-class TestFortranExampleIntegration(unittest.TestCase):
+# JSON Tests: Will always run
+class TestFortranExampleIntegrationJSON(unittest.TestCase):
@classmethod
def setUpClass(cls):
"""
- Obtain the binary directory from the CLI.
+ Obtain the binary directory from the CLI and compile the sina fortran
+ example needed for these tests if necessary.
"""
+ cwd = os.getcwd()
+
args = parse_args()
cls.binary_dir = args.binary_dir
if cls.binary_dir is None:
- # Assume we're at /path/to/build_dir/axom/sina/tests so move up to build_dir
- cls.binary_dir = os.path.join(os.getcwd(), "..", "..", "..")
+ # Move up three levels and resolve an absolute path
+ cls.binary_dir = os.path.abspath(os.path.join(cwd, "../"))
+
+ os.chdir(cls.binary_dir)
+
+ if not os.path.exists(os.path.join(cls.binary_dir, "examples/sina_fortran_ex")):
+ subprocess.run(["make", "sina_fortran_ex"])
+
+ os.chdir(cwd)
def setUp(self):
""" Invoke example Fortran application to dump a sina file """
- sina_fortran_ex_path = os.path.join(self.binary_dir, "examples", "sina_fortran_ex")
- if not os.path.exists(sina_fortran_ex_path):
- raise FileNotFoundError(
- f"The sina_fortran_ex needed for running fortran tests could not be found at path '{sina_fortran_ex_path}'"
- )
- subprocess.run([sina_fortran_ex_path])
+ subprocess.run([os.path.join(self.binary_dir, "examples/sina_fortran_ex")])
self.dump_file = "sina_dump.json"
def tearDown(self):
@@ -47,6 +56,7 @@ def test_file_validity(self):
try:
import jsonschema
schema_file = os.path.join(self.binary_dir, "tests", "sina_schema.json")
+
with io.open(schema_file, "r", encoding="utf-8") as schema:
schema = json.load(schema)
with io.open(self.dump_file, "r", encoding="utf-8") as loaded_test:
@@ -55,7 +65,6 @@ def test_file_validity(self):
except ModuleNotFoundError:
print("jsonschema module not found. Skipping test_file_validity.")
pass
-
def test_validate_contents_of_record(self):
""" Ensure that the record written out matches what we expect """
@@ -69,12 +78,9 @@ def test_validate_contents_of_record(self):
self.assertEqual("my_type", record["type"])
# Test the files
- path_to_my_file = os.path.join("/path", "to", "my", "file")
- my_file = os.path.join(path_to_my_file, "my_file.txt")
- other_file = os.path.join(path_to_my_file, "my_other_file.txt")
- self.assertEqual(list(record["files"].keys()), [other_file, my_file])
- self.assertEqual(record["files"][other_file]["mimetype"], "png")
- self.assertEqual(record["files"][my_file]["mimetype"], "txt")
+ self.assertEqual(list(record["files"].keys()), ["/path/to/my/file/my_other_file.txt", "/path/to/my/file/my_file.txt"])
+ self.assertEqual(record["files"]["/path/to/my/file/my_other_file.txt"]["mimetype"], "png")
+ self.assertEqual(record["files"]["/path/to/my/file/my_file.txt"]["mimetype"], "txt")
# Test the signed variants
self.assertEqual("A", record["data"]["char"]["value"])
@@ -116,8 +122,153 @@ def test_validate_contents_of_record(self):
self.assertEqual(double_arr, record["curve_sets"][curveset]["dependent"][double_2_name]["value"])
+#HDF5 Test
+@unittest.skipUnless(config.AXOM_USE_HDF5, "Requires h5py for HDF5-dependent tests")
+class TestFortranExampleIntegrationHDF5(unittest.TestCase):
+
+ @classmethod
+ def setUpClass(cls):
+ """
+ Obtain the binary directory from the CLI and compile the sina fortran
+ example needed for these tests if necessary.
+ """
+ cwd = os.getcwd()
+
+ args = parse_args()
+ cls.binary_dir = args.binary_dir
+ if cls.binary_dir is None:
+ # Move up three levels and resolve an absolute path
+ cls.binary_dir = os.path.abspath(os.path.join(cwd, "../"))
+
+ os.chdir(cls.binary_dir)
+
+ if not os.path.exists(os.path.join(cls.binary_dir, "examples/sina_fortran_ex")):
+ subprocess.run(["make", "sina_fortran_ex"])
+
+ os.chdir(cwd)
+
+ def setUp(self):
+ """ Invoke example Fortran application to dump a sina file """
+ subprocess.run([os.path.join(self.binary_dir, "examples/sina_fortran_ex")])
+ self.dump_file = "sina_dump.hdf5"
+
+ def tearDown(self):
+ os.remove(self.dump_file)
+
+ def extract_hdf5_value(self, value):
+ # If the value is an h5py.Dataset, retrieve its underlying data.
+ if isinstance(value, h5py.Dataset):
+ value = value[()]
+
+ # If the value is a bytes instance, decode it.
+ if isinstance(value, bytes):
+ return value.decode("utf-8").strip("\0").strip()
+
+ # If the value is a list or tuple of bytes, join them into a single bytes object and decode.
+ if isinstance(value, (list, tuple)) and value and all(isinstance(item, bytes) for item in value):
+ joined = b"".join(value)
+ return joined.decode("utf-8").strip("\0").strip()
+
+ # If the value has a 'tolist' method (e.g., from an array-like object), convert it and process.
+ if hasattr(value, "tolist"):
+ converted = value.tolist()
+ if isinstance(converted, (list, tuple)) and converted and all(isinstance(item, bytes) for item in converted):
+ joined = b"".join(converted)
+ return joined.decode("utf-8").strip("\0").strip()
+ if isinstance(converted, list) and len(converted) == 1:
+ return converted[0]
+ return converted
+
+ # Otherwise, return the value as-is.
+ return value
+
+ def test_validate_contents_of_record(self):
+ with h5py.File(self.dump_file, "r") as f:
+ record = f["records"]["0"]
+
+ # Validate metadata
+ self.assertEqual("my_rec_id", self.extract_hdf5_value(record["id"]))
+ self.assertEqual("my_type", self.extract_hdf5_value(record["type"]))
+
+ # Validate Files
+ files_group = record["files"]
+ expected_file_keys = [
+ "__SINA_SLASHREPLACE__path__SINA_SLASHREPLACE__to__SINA_SLASHREPLACE__my__SINA_SLASHREPLACE__file__SINA_SLASHREPLACE__my_other_file.txt",
+ "__SINA_SLASHREPLACE__path__SINA_SLASHREPLACE__to__SINA_SLASHREPLACE__my__SINA_SLASHREPLACE__file__SINA_SLASHREPLACE__my_file.txt"
+ ]
+ self.assertEqual(sorted(list(files_group.keys())), sorted(expected_file_keys))
+ self.assertEqual(self.extract_hdf5_value(
+ files_group["__SINA_SLASHREPLACE__path__SINA_SLASHREPLACE__to__SINA_SLASHREPLACE__my__SINA_SLASHREPLACE__file__SINA_SLASHREPLACE__my_other_file.txt"]["mimetype"]),
+ "png")
+ self.assertEqual(self.extract_hdf5_value(
+ files_group["__SINA_SLASHREPLACE__path__SINA_SLASHREPLACE__to__SINA_SLASHREPLACE__my__SINA_SLASHREPLACE__file__SINA_SLASHREPLACE__my_file.txt"]["mimetype"]),
+ "txt")
+
+ # Validate Data
+ data_group = record["data"]
+ self.assertEqual(self.extract_hdf5_value(data_group["char"]["value"]), "A")
+ self.assertEqual(self.extract_hdf5_value(data_group["int"]["value"]), 10)
+ self.assertEqual(self.extract_hdf5_value(data_group["logical"]["value"]), 0)
+ self.assertEqual(self.extract_hdf5_value(data_group["long"]["value"]), 1000000000.0)
+ self.assertAlmostEqual(self.extract_hdf5_value(data_group["real"]["value"]), 1.23456704616547)
+ self.assertAlmostEqual(self.extract_hdf5_value(data_group["double"]["value"]), 0.810000002384186)
+
+ self.assertEqual(self.extract_hdf5_value(data_group["u_char"]["value"]), "A")
+ self.assertEqual(self.extract_hdf5_value(data_group["u_char"]["units"]), "kg")
+ self.assertEqual(self.extract_hdf5_value(data_group["u_int"]["value"]), 10)
+ self.assertEqual(self.extract_hdf5_value(data_group["u_int"]["units"]), "kg")
+ self.assertEqual(self.extract_hdf5_value(data_group["u_logical"]["value"]), 1.0)
+ self.assertEqual(self.extract_hdf5_value(data_group["u_logical"]["units"]), "kg")
+ self.assertEqual(self.extract_hdf5_value(data_group["u_long"]["value"]), 1000000000.0)
+ self.assertEqual(self.extract_hdf5_value(data_group["u_long"]["units"]), "kg")
+ self.assertAlmostEqual(self.extract_hdf5_value(data_group["u_real"]["value"]), 1.23456704616547)
+ self.assertEqual(self.extract_hdf5_value(data_group["u_real"]["units"]), "kg")
+ self.assertAlmostEqual(self.extract_hdf5_value(data_group["u_double"]["value"]), 0.810000002384186)
+ self.assertEqual(self.extract_hdf5_value(data_group["u_double"]["units"]), "kg")
+ self.assertAlmostEqual(self.extract_hdf5_value(data_group["u_double_w_tag"]["value"]), 0.810000002384186)
+ self.assertEqual(self.extract_hdf5_value(data_group["u_double_w_tag"]["units"]), "kg")
+
+ tags_value = data_group["u_double_w_tag"]["tags"]
+ if isinstance(tags_value, h5py.Group):
+ if len(tags_value) == 1:
+ inner_dataset = list(tags_value.values())[0]
+ tags_value = self.extract_hdf5_value(inner_dataset)
+ if isinstance(tags_value, str):
+ tags_value = [tag.strip() for tag in tags_value.split(',')]
+ self.assertEqual(tags_value, ["new_fancy_tag"])
+
+ # Validate Curves
+ curveset_group = record["curve_sets"]["my_curveset"]
+ independent_group = curveset_group["independent"]
+ dependent_group = curveset_group["dependent"]
+
+ nums = list(range(1, 21))
+ real_arr = [i for i in nums]
+ double_arr = [i * 2 for i in nums]
+ int_arr = [i * 3 for i in nums]
+ long_arr = [i * 4 for i in nums]
+
+ for kind, grp in (("indep", independent_group), ("dep", dependent_group)):
+ for val_type, target in (("real", real_arr),
+ ("double", double_arr),
+ ("int", int_arr),
+ ("long", long_arr)):
+ curve_name = f"my_{kind}_curve_{val_type}"
+ self.assertIn(curve_name, grp)
+ curve_val = self.extract_hdf5_value(grp[curve_name]["value"])
+ self.assertEqual(curve_val, target)
+
+ double_2_name = "my_dep_curve_double_2"
+ self.assertIn(double_2_name, dependent_group)
+ curve_double_2 = self.extract_hdf5_value(dependent_group[double_2_name]["value"])
+ self.assertEqual(curve_double_2, double_arr)
+
+
if __name__ == "__main__":
# Doing the below instead of unittest.main() so that we can print to stdout
- suite = unittest.TestLoader().loadTestsFromTestCase(TestFortranExampleIntegration)
+ suite = unittest.TestSuite()
+ suite.addTests(unittest.defaultTestLoader.loadTestsFromTestCase(TestFortranExampleIntegrationJSON))
+ if config.AXOM_USE_HDF5:
+ suite.addTests(unittest.defaultTestLoader.loadTestsFromTestCase(TestFortranExampleIntegrationHDF5))
runner = unittest.TextTestRunner(buffer=False)
runner.run(suite)
\ No newline at end of file