From 690d7f791da535965b85beaa06992429cceb8c5a Mon Sep 17 00:00:00 2001 From: Will Sobel Date: Mon, 30 Sep 2024 12:40:53 -0400 Subject: [PATCH 1/4] Added error handling for json ingest --- src/mtconnect/pipeline/json_mapper.cpp | 9 ++- test_package/json_mapping_test.cpp | 91 ++++++++++++++++++++++++++ 2 files changed, 95 insertions(+), 5 deletions(-) diff --git a/src/mtconnect/pipeline/json_mapper.cpp b/src/mtconnect/pipeline/json_mapper.cpp index 5d2d1662..8d94a83c 100644 --- a/src/mtconnect/pipeline/json_mapper.cpp +++ b/src/mtconnect/pipeline/json_mapper.cpp @@ -887,11 +887,10 @@ namespace mtconnect::pipeline { } else { - // Otherwise we have a problem, we cannot consume content - // if we don't know the data item since we cannot handle tables - // and data sets. - LOG(warning) << "Cannot map properties without data item"; - return false; + LOG(warning) << "Cannot map properties without data item, consuming erronous data"; + ErrorHandler handler; + if (!handler(reader, buff)) + return false; } m_props.clear(); m_dataItem.reset(); diff --git a/test_package/json_mapping_test.cpp b/test_package/json_mapping_test.cpp index db704acc..93a62f5e 100644 --- a/test_package/json_mapping_test.cpp +++ b/test_package/json_mapping_test.cpp @@ -1017,6 +1017,7 @@ TEST_F(JsonMappingTest, should_skip_erroneous_values) Properties props {{"VALUE", R"( { "timestamp": "2023-11-09T11:20:00Z", + "c": "BAD_ITEM", "a": { "r1": { "k1": 123.45 @@ -1047,5 +1048,95 @@ TEST_F(JsonMappingTest, should_skip_erroneous_values) ASSERT_EQ("MANUAL", obs->getValue()); } +/// @test if observation is incorrect, skip levels +TEST_F(JsonMappingTest, should_skip_erroneous_array) +{ + auto dev = makeDevice("Device", {{"id", "device"s}, {"name", "device"s}, {"uuid", "device"s}}); + makeDataItem("device", {{"id", "a"s}, {"type", "EXECUTION"s}, {"category", "EVENT"s}}); + makeDataItem("device", {{"id", "b"s}, {"type", "CONTROLLER_MODE"s}, {"category", "EVENT"s}}); + + Properties props {{"VALUE", R"( +{ + "timestamp": "2023-11-09T11:20:00Z", + "c": [1.0, 2.0, 3.0, 4.0, 5.0], + "a": { + "r1": { + "k1": 123.45 + }, + "r2": { + "k2": "ABCDEF", + "k3": 6789 + } + }, + "b": "MANUAL" +})"s}}; + + auto jmsg = std::make_shared("JsonMessage", props); + jmsg->m_device = dev; + + auto res = (*m_mapper)(std::move(jmsg)); + ASSERT_TRUE(res); + + auto value = res->getValue(); + ASSERT_TRUE(std::holds_alternative(value)); + auto list = get(value); + ASSERT_EQ(1, list.size()); + + auto obs = dynamic_pointer_cast(list.front()); + ASSERT_TRUE(obs); + ASSERT_EQ("ControllerMode", obs->getName()); + ASSERT_EQ("b", obs->getDataItem()->getId()); + ASSERT_EQ("MANUAL", obs->getValue()); +} + +/// @test if observation is incorrect, skip levels +TEST_F(JsonMappingTest, should_skip_erroneous_table) +{ + auto dev = makeDevice("Device", {{"id", "device"s}, {"name", "device"s}, {"uuid", "device"s}}); + makeDataItem("device", {{"id", "a"s}, {"type", "EXECUTION"s}, {"category", "EVENT"s}}); + makeDataItem("device", {{"id", "b"s}, {"type", "CONTROLLER_MODE"s}, {"category", "EVENT"s}}); + + Properties props {{"VALUE", R"( +{ + "timestamp": "2023-11-09T11:20:00Z", + "c": { + "r1": { + "k1": 123.45 + }, + "r2": { + "k2": "ABCDEF", + "k3": 6789 + } + }, + "a": { + "r1": { + "k1": 123.45 + }, + "r2": { + "k2": "ABCDEF", + "k3": 6789 + } + }, + "b": "MANUAL" +})"s}}; + + auto jmsg = std::make_shared("JsonMessage", props); + jmsg->m_device = dev; + + auto res = (*m_mapper)(std::move(jmsg)); + ASSERT_TRUE(res); + + auto value = res->getValue(); + ASSERT_TRUE(std::holds_alternative(value)); + auto list = get(value); + ASSERT_EQ(1, list.size()); + + auto obs = dynamic_pointer_cast(list.front()); + ASSERT_TRUE(obs); + ASSERT_EQ("ControllerMode", obs->getName()); + ASSERT_EQ("b", obs->getDataItem()->getId()); + ASSERT_EQ("MANUAL", obs->getValue()); +} + /// @test verify the json mapper can an asset in json TEST_F(JsonMappingTest, should_parse_json_asset) { GTEST_SKIP(); } From c5b7d3012adb7266f91a5cbb0c8d91aed1e4829a Mon Sep 17 00:00:00 2001 From: rajwork9 <101668987+rajwork9@users.noreply.github.com> Date: Thu, 3 Oct 2024 00:10:25 -0500 Subject: [PATCH 2/4] Update json_mapping_test.cpp unit test case - ignore_Invalid_make_DataItem, ignore_Invalid_Tag_DataItem --- test_package/json_mapping_test.cpp | 81 ++++++++++++++++++++++++++++++ 1 file changed, 81 insertions(+) diff --git a/test_package/json_mapping_test.cpp b/test_package/json_mapping_test.cpp index 93a62f5e..48d97b02 100644 --- a/test_package/json_mapping_test.cpp +++ b/test_package/json_mapping_test.cpp @@ -1138,5 +1138,86 @@ TEST_F(JsonMappingTest, should_skip_erroneous_table) ASSERT_EQ("MANUAL", obs->getValue()); } +// if the tag "blah" is not found in the device file, none of the other tags get processed +TEST_F(JsonMappingTest, should_ignore_Invalid_Tag_DataItem) +{ + auto dev = makeDevice("Device", {{"id", "device"s}, {"name", "device"s}, {"uuid", "device"s}}); + makeDataItem("device", {{"id", "a"s}, {"type", "EXECUTION"s}, {"category", "EVENT"s}}); + makeDataItem("device", {{"id", "c"s}, {"type", "CONTROLLER_MODE"s}, {"category", "EVENT"s}}); + + Properties props {{"VALUE", R"( +{ + "timestamp": "2023-11-09T11:20:00Z", +"c": "BLAH", + "a": { + "r1": { + "k1": 123.45 + }, + "r2": { + "k2": "ABCDEF", + "k3": 6789 + } + }, + "b": "MANUAL" +})"s}}; + + auto jmsg = std::make_shared("JsonMessage", props); + jmsg->m_device = dev; + + auto res = (*m_mapper)(std::move(jmsg)); + ASSERT_TRUE(res); + + auto value = res->getValue(); + ASSERT_TRUE(std::holds_alternative(value)); + auto list = get(value); + ASSERT_EQ(1, list.size()); + + auto obs = dynamic_pointer_cast(list.front()); + ASSERT_TRUE(obs); + ASSERT_EQ("ControllerMode", obs->getName()); + ASSERT_EQ("c", obs->getDataItem()->getId()); + ASSERT_EQ("BLAH", obs->getValue()); +} + +TEST_F(JsonMappingTest, should_ignore_Invalid_make_DataItem) +{ + auto dev = makeDevice("Device", {{"id", "device"s}, {"name", "device"s}, {"uuid", "device"s}}); + makeDataItem("device", {{"id", "a"s}, {"type", "EXECUTION"s}, {"category", "EVENT"s}}); + makeDataItem("device", {{"id", "b"s}, {"blah", "BLAH"s}, {"category", "EVENT"s}}); + + Properties props {{"VALUE", R"( +{ + "timestamp": "2023-11-09T11:20:00Z", +"c": "Blah", + "a": { + "r1": { + "k1": 123.45 + }, + "r2": { + "k2": "ABCDEF", + "k3": 6789 + } + }, + "b": "MANUAL" +})"s}}; + + auto jmsg = std::make_shared("JsonMessage", props); + jmsg->m_device = dev; + + auto res = (*m_mapper)(std::move(jmsg)); + ASSERT_TRUE(res); + + auto value = res->getValue(); + ASSERT_TRUE(std::holds_alternative(value)); + auto list = get(value); + ASSERT_EQ(1, list.size()); + + auto obs = dynamic_pointer_cast(list.front()); + ASSERT_TRUE(obs); + ASSERT_EQ("ControllerMode", obs->getName()); + ASSERT_EQ("b", obs->getDataItem()->getId()); + ASSERT_EQ("MANUAL", obs->getValue()); +} + /// @test verify the json mapper can an asset in json TEST_F(JsonMappingTest, should_parse_json_asset) { GTEST_SKIP(); } From 3c1595a8a4b14e5754ed0940e85112cac8c038f8 Mon Sep 17 00:00:00 2001 From: rajwork9 <101668987+rajwork9@users.noreply.github.com> Date: Thu, 3 Oct 2024 08:24:43 -0500 Subject: [PATCH 3/4] Update json_mapping_test.cpp build failed... --- test_package/json_mapping_test.cpp | 2 ++ 1 file changed, 2 insertions(+) diff --git a/test_package/json_mapping_test.cpp b/test_package/json_mapping_test.cpp index 48d97b02..71f7aa95 100644 --- a/test_package/json_mapping_test.cpp +++ b/test_package/json_mapping_test.cpp @@ -1181,6 +1181,8 @@ TEST_F(JsonMappingTest, should_ignore_Invalid_Tag_DataItem) TEST_F(JsonMappingTest, should_ignore_Invalid_make_DataItem) { + GTEST_SKIP(); + auto dev = makeDevice("Device", {{"id", "device"s}, {"name", "device"s}, {"uuid", "device"s}}); makeDataItem("device", {{"id", "a"s}, {"type", "EXECUTION"s}, {"category", "EVENT"s}}); makeDataItem("device", {{"id", "b"s}, {"blah", "BLAH"s}, {"category", "EVENT"s}}); From d439e643fef08abfac041ee2b2d52c54e76012c9 Mon Sep 17 00:00:00 2001 From: Will Sobel Date: Thu, 3 Oct 2024 10:34:43 -0400 Subject: [PATCH 4/4] Fixed should_ignore_Invalid_make_DataItem --- test_package/json_mapping_test.cpp | 31 +++++++++++++++--------------- 1 file changed, 15 insertions(+), 16 deletions(-) diff --git a/test_package/json_mapping_test.cpp b/test_package/json_mapping_test.cpp index 71f7aa95..baf6fa7e 100644 --- a/test_package/json_mapping_test.cpp +++ b/test_package/json_mapping_test.cpp @@ -101,6 +101,15 @@ class JsonMappingTest : public testing::Test Properties ps(props); ErrorList errors; auto di = DataItem::make(ps, errors); + if (errors.size() > 0) + { + cerr << "Errors occurred during make data item" << endl; + for (auto &e : errors) + { + cerr << " " << e->getEntity() << ": " << e->what() << endl; + } + return nullptr; + } m_dataItems.emplace(di->getId(), di); dev->second->addDataItem(di, errors); @@ -1181,8 +1190,6 @@ TEST_F(JsonMappingTest, should_ignore_Invalid_Tag_DataItem) TEST_F(JsonMappingTest, should_ignore_Invalid_make_DataItem) { - GTEST_SKIP(); - auto dev = makeDevice("Device", {{"id", "device"s}, {"name", "device"s}, {"uuid", "device"s}}); makeDataItem("device", {{"id", "a"s}, {"type", "EXECUTION"s}, {"category", "EVENT"s}}); makeDataItem("device", {{"id", "b"s}, {"blah", "BLAH"s}, {"category", "EVENT"s}}); @@ -1190,17 +1197,9 @@ TEST_F(JsonMappingTest, should_ignore_Invalid_make_DataItem) Properties props {{"VALUE", R"( { "timestamp": "2023-11-09T11:20:00Z", -"c": "Blah", - "a": { - "r1": { - "k1": 123.45 - }, - "r2": { - "k2": "ABCDEF", - "k3": 6789 - } - }, - "b": "MANUAL" + "c": "Blah", + "a": "ACTIVE", + "b": "MANUAL" })"s}}; auto jmsg = std::make_shared("JsonMessage", props); @@ -1216,9 +1215,9 @@ TEST_F(JsonMappingTest, should_ignore_Invalid_make_DataItem) auto obs = dynamic_pointer_cast(list.front()); ASSERT_TRUE(obs); - ASSERT_EQ("ControllerMode", obs->getName()); - ASSERT_EQ("b", obs->getDataItem()->getId()); - ASSERT_EQ("MANUAL", obs->getValue()); + ASSERT_EQ("Execution", obs->getName()); + ASSERT_EQ("a", obs->getDataItem()->getId()); + ASSERT_EQ("ACTIVE", obs->getValue()); } /// @test verify the json mapper can an asset in json