Skip to content

Commit

Permalink
Better error messages and some documentation inside example
Browse files Browse the repository at this point in the history
  • Loading branch information
@franzpoeschel
franzpoeschel committed Jul 10, 2024
1 parent a191f83 commit 6dc4560
Show file tree
Hide file tree
Showing 2 changed files with 125 additions and 50 deletions.
33 changes: 30 additions & 3 deletions examples/13_write_dynamic_configuration.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,7 @@ using namespace openPMD;

int main()
{
if (!getVariants()["adios2"])
if (!getVariants()["hdf5"])
{
// Example configuration below selects the ADIOS2 backend
return 0;
Expand Down Expand Up @@ -57,13 +57,40 @@ parameters.clevel = 5
# type = "some other parameter"
# # ...
# Sometimes, dataset configurations should not affect all datasets, but only
# specific ones, e.g. only particle data.
# Dataset configurations can be given as a list, here at the example of HDF5.
# In such lists, each entry is an object with two keys:
#
# 1. 'cfg': Mandatory key, this is the actual dataset configuration.
# 2. 'select': A Regex or a list of Regexes to match against the dataset name.
#
# This makes it possible to give dataset-specific configurations.
# The dataset name is the same as returned
# by `Attributable::myPath().openPMDPath()`.
# The regex must match against either the full path (e.g. "/data/1/meshes/E/x")
# or against the path within the iteration (e.g. "meshes/E/x").
# Example:
# Let HDF5 datasets be automatically chunked by default
[[hdf5.dataset]]
cfg.chunks = "auto"
# For particles, we can specify the chunking explicitly
[[hdf5.dataset]]
# Multiple selection regexes can be given as a list.
# They will be fused into a single regex '($^)|(regex1)|(regex2)|(regex3)|...'.
select = ["/data/1/particles/e/.*", "/data/2/particles/e/.*"]
cfg.chunks = [5]
# Selecting a match works top-down, the order of list entries is important.
[[hdf5.dataset]]
# Specifying only a single regex.
# The regex can match against the full dataset path
# or against the path within the Iteration.
# Capitalization is irrelevant.
select = "particles/e/.*"
cfg.chunks = [10]
cfg.chornks = []
CFG.CHUNKS = [10]
)END";

// open file for writing
Expand Down
142 changes: 95 additions & 47 deletions src/auxiliary/JSONMatcher.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,8 +2,10 @@
#include "openPMD/Error.hpp"
#include "openPMD/auxiliary/JSON_internal.hpp"

#include <iostream>
#include <nlohmann/json.hpp>
#include <sstream>
#include <stdexcept>

namespace openPMD::json
{
Expand All @@ -16,6 +18,8 @@ namespace
* The "select" key is optional, indicating the default configuration if it
* is missing.
*
* @param backend_name For error messages.
* @param index_in_list For error messages.
* @param patterns Output parameter: Emplace a parsed pattern into this
* list.
* @param defaultConfig Output parameter: If the pattern was the default
Expand All @@ -24,6 +28,8 @@ namespace
* @return Whether the pattern was the default configuration or not.
*/
auto readPattern(
std::string const &backend_name,
size_t index_in_list,
std::vector<Pattern> &patterns,
std::optional<nlohmann::json> &defaultConfig,
nlohmann::json object) -> void;
Expand Down Expand Up @@ -53,19 +59,23 @@ void MatcherPerBackend::init(TracingJSON tracing_config)
{
std::optional<nlohmann::json> defaultConfig;
// enhanced PIConGPU-defined layout
for (auto &value : config)
for (size_t i = 0; i < config.size(); ++i)
{
readPattern(m_patterns, defaultConfig, std::move(value));
readPattern(
backendName,
i,
m_patterns,
defaultConfig,
std::move(config.at(i)));
}
// now replace the pattern list with the default config
tracing_config.json() =
std::move(defaultConfig).value_or(nlohmann::json::object());
}
else
{
throw std::runtime_error(
"[openPMD plugin] Expecting an object or an array as JSON "
"configuration.");
throw error::BackendConfigSchema(
{backendName, "dataset"}, "Expecting an object or an array.");
}
}

Expand Down Expand Up @@ -171,66 +181,104 @@ auto JsonMatcher::getDefault() -> TracingJSON
namespace
{
auto readPattern(
std::string const &backend_name,
size_t index_in_list,
std::vector<Pattern> &patterns,
std::optional<nlohmann::json> &defaultConfig,
nlohmann::json object) -> void
{
constexpr char const *errorMsg = &R"END(
Each single pattern in an extended JSON configuration must be a JSON object
with keys 'select' and 'cfg'.
The key 'select' is optional, indicating a default configuration if it is
not set.
The key 'select' must point to either a single string or an array of strings.)END"
[1];
constexpr char const *errorMsg = R"END(
Each single pattern in an dataset-specific JSON/TOML configuration must be
an object with mandatory key 'cfg' and optional key 'select'.
When the key 'select' is not specified, the given configuration is used
for setting up the default dataset configuration upon backend initialization.
The key 'select' must point to either a single string or an array of strings
and is interpreted as a regular expression against which the dataset name
(full path or path within an iteration) must match.)END";
auto throw_up = [&](std::string const &additional_info,
auto &&...additional_path) {
throw error::BackendConfigSchema(
{backend_name,
"dataset",
std::to_string(index_in_list),
additional_path...},
additional_info + errorMsg);
};

if (!object.is_object())
{
throw std::runtime_error(errorMsg);
throw_up("Not an object!");
}
if (!object.contains("cfg"))
{
throw_up("Mandatory key missing: 'cfg'!");
}
try
{
nlohmann::json &cfg = object.at("cfg");
if (!object.contains("select"))
std::vector<std::string> unrecognized_keys;
for (auto it = object.begin(); it != object.end(); ++it)
{
if (defaultConfig.has_value())
if (it.key() == "select" || it.key() == "cfg")
{
throw std::runtime_error(
"Specified more than one default configuration.");
continue;
}
defaultConfig.emplace(std::move(cfg));
return;
unrecognized_keys.emplace_back(it.key());
}
else
if (!unrecognized_keys.empty())
{
nlohmann::json const &pattern = object.at("select");
std::string pattern_str = [&]() -> std::string {
if (pattern.is_string())
{
return pattern.get<std::string>();
}
else if (pattern.is_array())
std::cerr << "[Warning] JSON/TOML config at '" << backend_name
<< ".dataset." << index_in_list
<< "' has unrecognized keys:";
for (auto const &item : unrecognized_keys)
{
std::cerr << " '" << item << '\'';
}
std::cerr << '.' << std::endl;
}
}

nlohmann::json &cfg = object.at("cfg");
if (!object.contains("select"))
{
if (defaultConfig.has_value())
{
throw_up("Specified more than one default configuration!");
}
defaultConfig.emplace(std::move(cfg));
return;
}
else
{
nlohmann::json const &pattern = object.at("select");
std::string pattern_str = [&]() -> std::string {
if (pattern.is_string())
{
return pattern.get<std::string>();
}
else if (pattern.is_array())
{
std::stringstream res;
res << "($^)";
for (auto const &sub_pattern : pattern)
{
std::stringstream res;
res << "($^)";
for (auto const &sub_pattern : pattern)
if (!sub_pattern.is_string())
{
res << "|(" << sub_pattern.get<std::string>()
<< ")";
throw_up(
"Must be a string or an array of string!",
"select");
}
return res.str();
}
else
{
throw std::runtime_error(errorMsg);
res << "|(" << sub_pattern.get<std::string>() << ")";
}
}();
patterns.emplace_back(pattern_str, std::move(cfg));
return;
}
}
catch (nlohmann::json::out_of_range const &)
{
throw std::runtime_error(errorMsg);
return res.str();
}
else
{
throw_up(
"Must be a string or an array of string!", "select");
throw std::runtime_error("Unreachable!");
}
}();
patterns.emplace_back(pattern_str, std::move(cfg));
return;
}
}
} // namespace
Expand Down

0 comments on commit 6dc4560

Please sign in to comment.