Merge branch 'main' into feature/issue-421-alert-noise-reduction-on-h…

…wstatusstate=present
sentrysoftware · Sep 25, 2024 · cbe103b · cbe103b
2 parents 5b94741 + 9ef5209
commit cbe103b
Show file tree

Hide file tree

Showing 10 changed files with 235 additions and 29 deletions.
diff --git a/metricshub-agent/src/main/java/org/sentrysoftware/metricshub/agent/helper/ConfigHelper.java b/metricshub-agent/src/main/java/org/sentrysoftware/metricshub/agent/helper/ConfigHelper.java
@@ -823,24 +823,18 @@ private static void updateResourceGroupTelemetryManagers(
 			// Validate protocols and update the configuration's hostname if required.
 			validateAndNormalizeProtocols(resourceKey, resourceConfig, hostConfiguration.getHostname());
 
+			// Read the configured connector for the current resource
 			addConfiguredConnector(resourceConnectorStore, resourceConfig.getConnector());
 
-			// Retrieve connectors variables map from the resource configuration
-			final Map<String, ConnectorVariables> connectorVariablesMap = resourceConfig.getVariables();
-
-			// If connectors variables exist then merge the existing connector store with a new one containing custom connectors
-			if (connectorVariablesMap != null && !connectorVariablesMap.isEmpty()) {
-				// Call ConnectorTemplateLibraryParser and parse the custom connectors
-				final ConnectorTemplateLibraryParser connectorTemplateLibraryParser = new ConnectorTemplateLibraryParser();
-
-				final Map<String, Connector> customConnectors = connectorTemplateLibraryParser.parse(
-					ConfigHelper.getSubDirectory("connectors", false),
-					connectorVariablesMap
-				);
+			// Read connectors with configuration variables safely
+			final Map<String, Connector> connectorsWithConfigVariables = readConnectorsWithConfigurationVariablesSafe(
+				resourceGroupKey,
+				resourceKey,
+				resourceConfig
+			);
 
-				// Overwrite resourceConnectorStore
-				updateConnectorStore(resourceConnectorStore, customConnectors);
-			}
+			// Overwrite resourceConnectorStore
+			updateConnectorStore(resourceConnectorStore, connectorsWithConfigVariables);
 
 			resourceGroupTelemetryManagers.putIfAbsent(
 				resourceKey,
@@ -858,6 +852,43 @@ private static void updateResourceGroupTelemetryManagers(
 		}
 	}
 
+	/**
+	 * Parse the connectors having configuration variables.
+	 *
+	 * @param resourceGroupKey The resource group key under which the resource is configured for logging purposes.
+	 * @param resourceKey      The resource key for logging purposes.
+	 * @param resourceConfig   The resource configuration.
+	 * @return Map of connectors with configuration variables
+	 */
+	private static Map<String, Connector> readConnectorsWithConfigurationVariablesSafe(
+		final String resourceGroupKey,
+		final String resourceKey,
+		final ResourceConfig resourceConfig
+	) {
+		// Retrieve connectors variables map from the resource configuration
+		final Map<String, ConnectorVariables> connectorVariablesMap = resourceConfig.getVariables();
+
+		// Call ConnectorTemplateLibraryParser and parse the custom connectors
+		final ConnectorTemplateLibraryParser connectorTemplateLibraryParser = new ConnectorTemplateLibraryParser();
+
+		try {
+			return connectorTemplateLibraryParser.parse(
+				ConfigHelper.getSubDirectory("connectors", false),
+				connectorVariablesMap
+			);
+		} catch (Exception e) {
+			log.warn(
+				"Resource {} - Under the resource group configuration {}, the resource configuration {} will not load connectors with configuration variables." +
+				" Reason: {}.",
+				resourceKey,
+				resourceGroupKey,
+				resourceKey,
+				e.getMessage()
+			);
+			return new HashMap<>();
+		}
+	}
+
 	/**
 	 * Add the custom connectors to the resource's connector store.
 	 *

diff --git a/.../main/java/org/sentrysoftware/metricshub/agent/helper/ConnectorTemplateLibraryParser.java b/.../main/java/org/sentrysoftware/metricshub/agent/helper/ConnectorTemplateLibraryParser.java
@@ -30,12 +30,15 @@
 import java.nio.file.Path;
 import java.nio.file.SimpleFileVisitor;
 import java.nio.file.attribute.BasicFileAttributes;
+import java.util.HashMap;
 import java.util.Map;
+import java.util.Map.Entry;
 import java.util.TreeMap;
 import lombok.NonNull;
 import lombok.extern.slf4j.Slf4j;
 import org.sentrysoftware.metricshub.engine.configuration.ConnectorVariables;
 import org.sentrysoftware.metricshub.engine.connector.model.Connector;
+import org.sentrysoftware.metricshub.engine.connector.model.identity.ConnectorDefaultVariable;
 import org.sentrysoftware.metricshub.engine.connector.parser.ConnectorParser;
 
 /**
@@ -84,17 +87,41 @@ public FileVisitResult visitFile(Path path, BasicFileAttributes attrs) throws IO
 			final String filename = path.getFileName().toString();
 			final String connectorId = filename.substring(0, filename.lastIndexOf('.'));
 
-			if (!connectorVariablesMap.containsKey(connectorId)) {
+			if (!connectorNode.toString().contains("${var::")) {
 				return FileVisitResult.CONTINUE;
 			}
 
+			// User connector variables
+			final ConnectorVariables connectorUserVariables = connectorVariablesMap.computeIfAbsent(
+				connectorId,
+				id -> new ConnectorVariables()
+			);
+
+			// Retrieve the default connector variables that have been specified in this connector.
+			final Map<String, ConnectorDefaultVariable> connectorDefaultVariables = getConnectorVariables(connectorNode);
+
+			// User didn't configure variables for this connector, and no connector default variables are configured
+			if (connectorUserVariables.getVariableValues().isEmpty() && connectorDefaultVariables.isEmpty()) {
+				return FileVisitResult.CONTINUE;
+			}
+
+			// For each configured default connector variable, if the user didn't specify a value to that variable, user default value.
+			for (final Entry<String, ConnectorDefaultVariable> entry : connectorDefaultVariables.entrySet()) {
+				connectorUserVariables.getVariableValues().putIfAbsent(entry.getKey(), entry.getValue().getDefaultValue());
+			}
+
 			final ConnectorParser connectorParser = ConnectorParser.withNodeProcessorAndUpdateChain(
 				path.getParent(),
 				connectorVariablesMap.get(connectorId).getVariableValues()
 			);
 
 			// Put in the custom connectorsMap
-			customConnectorsMap.put(connectorId, connectorParser.parse(path.toFile()));
+			try {
+				customConnectorsMap.put(connectorId, connectorParser.parse(path.toFile()));
+			} catch (Exception e) {
+				log.error("Error while parsing connector with template variables {}: {}", filename, e.getMessage());
+				log.debug("Exception: ", e);
+			}
 
 			return FileVisitResult.CONTINUE;
 		}
@@ -124,6 +151,45 @@ private boolean isConnector(final JsonNode connector) {
 		private boolean isYamlFile(final String name) {
 			return name.toLowerCase().endsWith(".yaml");
 		}
+
+		/**
+		 * Converts the "variables" section of a {@link JsonNode} into a {@link Map} where each entry consists of
+		 * a variable name as the key and a {@link ConnectorDefaultVariable} as the value.
+		 * Each {@link ConnectorDefaultVariable} contains a description and a default value extracted from the JSON node.
+		 *
+		 * @param connectorNode the {@link JsonNode} representing the connector, which includes a "variables" section.
+		 * @return a map where the key is the variable name, and the value is a {@link ConnectorDefaultVariable} object
+		 *         containing the description and defaultValue for that variable. If the "variables" section is not present,
+		 *         an empty map is returned.
+		 */
+		private static Map<String, ConnectorDefaultVariable> getConnectorVariables(final JsonNode connectorNode) {
+			final JsonNode variablesNode = connectorNode.get("connector").get("variables");
+			if (variablesNode == null) {
+				return new HashMap<>();
+			}
+
+			final Map<String, ConnectorDefaultVariable> connectorVariablesMap = new HashMap<>();
+
+			// Iterate over the variables and extract description and defaultValue
+			variablesNode
+				.fields()
+				.forEachRemaining(entry -> {
+					final String variableName = entry.getKey();
+					final JsonNode variableValue = entry.getValue();
+
+					final String description = variableValue.get("description").asText();
+					final String defaultValue = variableValue.get("defaultValue").asText();
+
+					// Create a ConnectorDefaultVariable object and put it into the map
+					final ConnectorDefaultVariable connectorDefaultVariable = new ConnectorDefaultVariable(
+						description,
+						defaultValue
+					);
+					connectorVariablesMap.put(variableName, connectorDefaultVariable);
+				});
+
+			return connectorVariablesMap;
+		}
 	}
 
 	/**
@@ -141,7 +207,10 @@ public Map<String, Connector> parse(
 		final long startTime = System.currentTimeMillis();
 		final ConnectorFileVisitor connectorFileVisitor = new ConnectorFileVisitor(connectorVariablesMap);
 		Files.walkFileTree(yamlParentDirectory, connectorFileVisitor);
-		log.info("Yaml loading duration: {} seconds", (System.currentTimeMillis() - startTime) / 1000);
+		log.info(
+			"Connectors with template variables parsing duration: {} seconds",
+			(System.currentTimeMillis() - startTime) / 1000
+		);
 		return connectorFileVisitor.getCustomConnectorsMap();
 	}
 }
diff --git a/...t/java/org/sentrysoftware/metricshub/agent/helper/ConnectorTemplateLibraryParserTest.java b/...t/java/org/sentrysoftware/metricshub/agent/helper/ConnectorTemplateLibraryParserTest.java
@@ -1,14 +1,20 @@
 package org.sentrysoftware.metricshub.agent.helper;
 
 import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.sentrysoftware.metricshub.engine.connector.model.identity.ConnectionType.LOCAL;
+import static org.sentrysoftware.metricshub.engine.connector.model.identity.ConnectionType.REMOTE;
 
 import java.io.IOException;
 import java.nio.file.Path;
 import java.nio.file.Paths;
+import java.util.HashMap;
 import java.util.Map;
+import java.util.Set;
 import org.junit.jupiter.api.Test;
 import org.sentrysoftware.metricshub.engine.configuration.ConnectorVariables;
 import org.sentrysoftware.metricshub.engine.connector.model.Connector;
+import org.sentrysoftware.metricshub.engine.connector.model.identity.Detection;
+import org.sentrysoftware.metricshub.engine.connector.model.identity.criterion.SnmpGetNextCriterion;
 
 class ConnectorTemplateLibraryParserTest {
 
@@ -21,17 +27,34 @@ void testParse() throws IOException {
 
 		// Call ConnectorTemplateLibraryParser to parse the custom connectors files using the connectorVariables map and the connector id
 		final ConnectorTemplateLibraryParser connectorTemplateLibraryParser = new ConnectorTemplateLibraryParser();
-		final ConnectorVariables connectorVariables = new ConnectorVariables(Map.of("snmp-get-next", "snmpGetNext"));
+
+		final ConnectorVariables connectorVariables = new ConnectorVariables(new HashMap<>());
+		connectorVariables.addVariableValue("snmp-get-next", "snmpGetNext");
+		connectorVariables.addVariableValue("local-variable", "local");
+		final Map<String, ConnectorVariables> connectorVariablesMap = new HashMap<>();
+		connectorVariablesMap.put(CONNECTOR_ID, connectorVariables);
 		final Map<String, Connector> customConnectorsMap = connectorTemplateLibraryParser.parse(
 			yamlTestPath,
-			Map.of(CONNECTOR_ID, connectorVariables)
+			connectorVariablesMap
 		);
 
 		// Check that only the connector containing variables is returned in the map
 		assertEquals(1, customConnectorsMap.size());
 
-		// Check that the connector variable value was successfully replaced
+		// Check that the connector variable value was successfully replaced.
 		final Connector customConnector = customConnectorsMap.get(CONNECTOR_ID);
-		assertEquals("snmpGetNext", customConnector.getConnectorIdentity().getDetection().getCriteria().get(0).getType());
+		final Detection detection = customConnector.getConnectorIdentity().getDetection();
+		// Both user's and default values are set, the priority is for the user's variable value.
+		assertEquals("snmpGetNext", detection.getCriteria().get(0).getType());
+		// User's value set, no default value.
+		assertEquals(Set.of(REMOTE, LOCAL), detection.getConnectionTypes());
+		// User's value not set, default value is set.
+		final SnmpGetNextCriterion criterion = (SnmpGetNextCriterion) detection.getCriteria().get(0);
+		assertEquals("1.3.6.1.4.1.795.10.1.1.3.1.1", criterion.getOid());
+		// User's value not set, default value not set, variable remains unchanged.
+		assertEquals(
+			"${var::oid-description}",
+			customConnector.getConnectorIdentity().getVariables().get("oid").getDescription().trim()
+		);
 	}
 }
diff --git a/metricshub-agent/src/test/resources/connectorTemplateLibraryParser/templateVariable.yaml b/metricshub-agent/src/test/resources/connectorTemplateLibraryParser/templateVariable.yaml
@@ -8,13 +8,20 @@ connector:
   detection:
     connectionTypes:
     - remote
-    - local
+    - ${var::local-variable}
     appliesTo:
     - NT
     - Linux
     criteria:
     - type: ${var::snmp-get-next}
-      oid: 1.3.6.1.4.1.795.10.1.1.3.1.1
+      oid: ${var::oid}
+  variables:
+    oid:
+      description: ${var::oid-description}
+      defaultValue: 1.3.6.1.4.1.795.10.1.1.3.1.1
+    snmp-get-next:
+      description: the detection SNMP operation, for the sake of testing, it is different from the user's specified value in the test. The purpose is to ensure that the user's values are more of a priority than the default values.
+      defaultValue: wrongSnmpGetNextValue
 monitors:
   disk_controller:
     discovery:

diff --git a/...g/sentrysoftware/metricshub/engine/connector/model/identity/ConnectorDefaultVariable.java b/...g/sentrysoftware/metricshub/engine/connector/model/identity/ConnectorDefaultVariable.java
@@ -0,0 +1,46 @@
+package org.sentrysoftware.metricshub.engine.connector.model.identity;
+
+import lombok.AllArgsConstructor;
+import lombok.Builder;
+import lombok.Data;
+import lombok.NoArgsConstructor;
+
+/*-
+ * ╱╲╱╲╱╲╱╲╱╲╱╲╱╲╱╲╱╲╱╲╱╲╱╲╱╲╱╲╱╲╱╲╱╲╱╲╱╲╱╲
+ * MetricsHub Engine
+ * ჻჻჻჻჻჻
+ * Copyright 2023 - 2024 Sentry Software
+ * ჻჻჻჻჻჻
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ * ╲╱╲╱╲╱╲╱╲╱╲╱╲╱╲╱╲╱╲╱╲╱╲╱╲╱╲╱╲╱╲╱╲╱╲╱╲╱╲╱
+ */
+
+/**
+ * Represents the connector default variables that can be defined on a Connector Template.
+ */
+@NoArgsConstructor
+@AllArgsConstructor
+@Data
+@Builder
+public class ConnectorDefaultVariable {
+
+	/**
+	 * The default connector variable description.
+	 */
+	private String description;
+	/**
+	 * The default connector variable default value.
+	 */
+	private String defaultValue;
+}
diff --git a/...java/org/sentrysoftware/metricshub/engine/connector/model/identity/ConnectorIdentity.java b/...java/org/sentrysoftware/metricshub/engine/connector/model/identity/ConnectorIdentity.java
@@ -22,8 +22,11 @@
  */
 
 import java.io.Serializable;
+import java.util.HashMap;
+import java.util.Map;
 import lombok.AllArgsConstructor;
 import lombok.Builder;
+import lombok.Builder.Default;
 import lombok.Data;
 import lombok.NoArgsConstructor;
 
@@ -77,4 +80,10 @@ public class ConnectorIdentity implements Serializable {
 	 * The detection information of the connector.
 	 */
 	private Detection detection;
+
+	/**
+	 * The connector default variables that can be specified.
+	 */
+	@Default
+	private Map<String, ConnectorDefaultVariable> variables = new HashMap<>();
 }
diff --git a/...in/java/org/sentrysoftware/metricshub/engine/connector/parser/ConnectorLibraryParser.java b/...in/java/org/sentrysoftware/metricshub/engine/connector/parser/ConnectorLibraryParser.java
@@ -203,7 +203,7 @@ public Map<String, Connector> parseConnectorsFromAllYamlFiles(Path yamlParentDir
 		final long startTime = System.currentTimeMillis();
 		final ConnectorFileVisitor fileVisitor = new ConnectorFileVisitor();
 		Files.walkFileTree(yamlParentDirectory, fileVisitor);
-		log.info("Yaml loading duration: {} seconds", (System.currentTimeMillis() - startTime) / 1000);
+		log.info("Connectors parsing duration: {} seconds", (System.currentTimeMillis() - startTime) / 1000);
 		return fileVisitor.getConnectorsMap();
 	}
 }