Skip to content

Commit

Permalink
Merge pull request #378 from sentrysoftware/feature/issue-316-feature…
Browse files Browse the repository at this point in the history
…-request-develop-a-new-metricshub-awk-source

Issue #316: Develop a new metricshub awk source
  • Loading branch information
NassimBtk authored Sep 10, 2024
2 parents a168d99 + cbdf1a3 commit de3e67c
Show file tree
Hide file tree
Showing 36 changed files with 1,633 additions and 24 deletions.
1 change: 1 addition & 0 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,7 @@ This is a multi-module project:
* **metricshub-winrm-extension**: Enables the use of Windows Remote Management (WinRM) for remote management and monitoring of Windows-based systems.
* **metricshub-wbem-extension**: Supports the Web-Based Enterprise Management (WBEM) standard for accessing management information.
* **metricshub-ping-extension**: Enables testing the reachability of hosts using ICMP-based ping commands.
* **metricshub-jawk-extension**: Allows execution of Jawk scripts.
* **metricshub-hardware**: Hardware Energy and Sustainability module, dedicated to managing and monitoring hardware-related metrics, focusing on energy consumption and sustainability aspects.
* **metricshub-it-common**: Contains common code and utilities used by integration tests across various modules.
* **metricshub-windows**: Builds the `.zip` package for MetricsHub on Windows platforms.
Expand Down
6 changes: 6 additions & 0 deletions metricshub-agent/pom.xml
Original file line number Diff line number Diff line change
Expand Up @@ -183,6 +183,12 @@
<version>${project.version}</version>
<scope>test</scope>
</dependency>
<dependency>
<groupId>${project.groupId}</groupId>
<artifactId>metricshub-jawk-extension</artifactId>
<version>${project.version}</version>
<scope>test</scope>
</dependency>

<!-- Log4j2 -->
<dependency>
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -41,6 +41,7 @@
import org.sentrysoftware.metricshub.agent.deserialization.ConnectorVariablesDeserializer;
import org.sentrysoftware.metricshub.agent.deserialization.ExtensionProtocolsDeserializer;
import org.sentrysoftware.metricshub.agent.deserialization.MonitorJobsDeserializer;
import org.sentrysoftware.metricshub.engine.configuration.ConnectorVariables;
import org.sentrysoftware.metricshub.engine.configuration.IConfiguration;
import org.sentrysoftware.metricshub.engine.connector.model.Connector;
import org.sentrysoftware.metricshub.engine.connector.model.monitor.MonitorJob;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -28,7 +28,7 @@
import java.io.IOException;
import java.util.Map;
import java.util.TreeMap;
import org.sentrysoftware.metricshub.agent.config.ConnectorVariables;
import org.sentrysoftware.metricshub.engine.configuration.ConnectorVariables;

/**
* Custom JSON deserializer for deserializing a JSON object into a TreeMap with
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -63,7 +63,6 @@
import org.apache.logging.log4j.ThreadContext;
import org.sentrysoftware.metricshub.agent.config.AgentConfig;
import org.sentrysoftware.metricshub.agent.config.AlertingSystemConfig;
import org.sentrysoftware.metricshub.agent.config.ConnectorVariables;
import org.sentrysoftware.metricshub.agent.config.ResourceConfig;
import org.sentrysoftware.metricshub.agent.config.ResourceGroupConfig;
import org.sentrysoftware.metricshub.agent.context.MetricDefinitions;
Expand All @@ -73,6 +72,7 @@
import org.sentrysoftware.metricshub.engine.common.helpers.LocalOsHandler;
import org.sentrysoftware.metricshub.engine.common.helpers.MetricsHubConstants;
import org.sentrysoftware.metricshub.engine.common.helpers.ResourceHelper;
import org.sentrysoftware.metricshub.engine.configuration.ConnectorVariables;
import org.sentrysoftware.metricshub.engine.configuration.HostConfiguration;
import org.sentrysoftware.metricshub.engine.configuration.IConfiguration;
import org.sentrysoftware.metricshub.engine.connector.model.Connector;
Expand Down Expand Up @@ -1015,6 +1015,7 @@ static HostConfiguration buildHostConfiguration(
.hostType(hostType)
.sequential(Boolean.TRUE.equals(resourceConfig.getSequential()))
.configuredConnectorId(configuredConnectorId)
.connectorVariables(resourceConfig.getVariables())
.build();
}

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -34,7 +34,7 @@
import java.util.TreeMap;
import lombok.NonNull;
import lombok.extern.slf4j.Slf4j;
import org.sentrysoftware.metricshub.agent.config.ConnectorVariables;
import org.sentrysoftware.metricshub.engine.configuration.ConnectorVariables;
import org.sentrysoftware.metricshub.engine.connector.model.Connector;
import org.sentrysoftware.metricshub.engine.connector.parser.ConnectorParser;

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -31,11 +31,11 @@
import java.util.Set;
import org.junit.jupiter.api.Test;
import org.sentrysoftware.metricshub.agent.config.AgentConfig;
import org.sentrysoftware.metricshub.agent.config.ConnectorVariables;
import org.sentrysoftware.metricshub.agent.config.ResourceConfig;
import org.sentrysoftware.metricshub.agent.config.ResourceGroupConfig;
import org.sentrysoftware.metricshub.agent.helper.OtelSdkConfigConstants;
import org.sentrysoftware.metricshub.engine.common.helpers.MapHelper;
import org.sentrysoftware.metricshub.engine.configuration.ConnectorVariables;
import org.sentrysoftware.metricshub.engine.connector.model.common.HttpMethod;
import org.sentrysoftware.metricshub.engine.connector.model.common.ResultContent;
import org.sentrysoftware.metricshub.engine.connector.model.monitor.SimpleMonitorJob;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,7 @@
import java.nio.file.Paths;
import java.util.Map;
import org.junit.jupiter.api.Test;
import org.sentrysoftware.metricshub.agent.config.ConnectorVariables;
import org.sentrysoftware.metricshub.engine.configuration.ConnectorVariables;
import org.sentrysoftware.metricshub.engine.connector.model.Connector;

class ConnectorTemplateLibraryParserTest {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -59,7 +59,7 @@ public static AwkTuples getIntermediateCode(final String script) throws ParseExc
// All scripts need to be prefixed with an extra statement that sets the Record Separator (RS)
// to the "normal" end-of-line (\n), because Jawk uses line.separator System property, which
// is \r\n on Windows, thus preventing it from splitting lines properly.
final ScriptSource awkHeader = new ScriptSource("Header", new StringReader("BEGIN { RS = \"\\n\"; }"), false);
final ScriptSource awkHeader = new ScriptSource("Header", new StringReader("BEGIN { ORS = RS = \"\\n\"; }"), false);
final ScriptSource awkSource = new ScriptSource("Body", new StringReader(script), false);
final List<ScriptSource> sourceList = new ArrayList<>();
sourceList.add(awkHeader);
Expand All @@ -68,11 +68,10 @@ public static AwkTuples getIntermediateCode(final String script) throws ParseExc
// Awk Setup
final AwkSettings settings = new AwkSettings();
settings.setCatchIllegalFormatExceptions(false);
settings.setUseStdIn(false);

// Parse the Awk script
final AwkTuples tuples = new AwkTuples();
final AwkParser parser = new AwkParser(false, false, false, Collections.emptyMap());
final AwkParser parser = new AwkParser(false, false, Collections.emptyMap());
final AwkSyntaxTree ast;
try {
ast = parser.parse(sourceList);
Expand All @@ -85,7 +84,7 @@ public static AwkTuples getIntermediateCode(final String script) throws ParseExc
// 2nd pass to tie actual parameters to forward-referenced formal parameters
ast.semanticAnalysis();
if (ast.populateTuples(tuples) != 0) {
throw new ParseException("Syntax problem with the Awk script", 0);
throw new RuntimeException("Syntax problem with the Awk script");
}
tuples.postProcess();
parser.populateGlobalVariableNameToOffsetMappings(tuples);
Expand Down Expand Up @@ -129,6 +128,11 @@ public static String interpret(final String input, final AwkTuples intermediateC
try {
avm.interpret(intermediateCode);
} catch (ExitException e) {
// ExitException code 0 means exit OK
if (e.getCode() != 0) {
throw new RuntimeException(e.getMessage());
}
} catch (Exception e) {
throw new RuntimeException(e.getMessage());
}

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -139,10 +139,33 @@ public String executeAwkScript(
* @return The CSV representation of the JSON.
* @throws TimeoutException If the execution times out.
* @throws ExecutionException If an execution exception occurs.
* @throws InterruptedException If the execution is interrupted.
* @throws InterruptedException If the execution is interrupted.
*/
public String executeJson2Csv(String jsonSource, String jsonEntryKey, List<String> propertyList, String separator)
throws InterruptedException, ExecutionException, TimeoutException {
return executeJson2Csv(jsonSource, jsonEntryKey, propertyList, separator, telemetryManager.getHostname());
}

/**
* Execute JSON to CSV operation.
*
* @param jsonSource The JSON source string.
* @param jsonEntryKey The JSON entry key.
* @param propertyList The list of properties.
* @param separator The separator for CSV.
* @param hostname The hostname, for logging purpose.
* @return The CSV representation of the JSON.
* @throws TimeoutException If the execution times out.
* @throws ExecutionException If an execution exception occurs.
* @throws InterruptedException If the execution is interrupted.
*/
public static String executeJson2Csv(
String jsonSource,
String jsonEntryKey,
List<String> propertyList,
String separator,
String hostname
) throws InterruptedException, ExecutionException, TimeoutException {
LoggingHelper.trace(() ->
log.trace(
"Executing JSON to CSV conversion:\n- Json-source:\n{}\n- Json-entry-key: {}\n" + // NOSONAR
Expand All @@ -154,8 +177,6 @@ public String executeJson2Csv(String jsonSource, String jsonEntryKey, List<Strin
)
);

final String hostname = telemetryManager.getHostname();

final Callable<String> jflatToCSV = () -> {
try {
JFlat jsonFlat = new JFlat(jsonSource);
Expand Down
Original file line number Diff line number Diff line change
@@ -1,8 +1,8 @@
package org.sentrysoftware.metricshub.agent.config;
package org.sentrysoftware.metricshub.engine.configuration;

/*-
* ╱╲╱╲╱╲╱╲╱╲╱╲╱╲╱╲╱╲╱╲╱╲╱╲╱╲╱╲╱╲╱╲╱╲╱╲╱╲╱╲
* MetricsHub Agent
* MetricsHub Engine
* ჻჻჻჻჻჻
* Copyright 2023 - 2024 Sentry Software
* ჻჻჻჻჻჻
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -60,7 +60,9 @@ public class HostConfiguration {
private boolean sequential;
private Consumer<AlertInfo> alertTrigger;
private long retryDelay;
private Map<String, String> connectorVariables;

// The map of connector variables. The key is the connector ID.
private Map<String, ConnectorVariables> connectorVariables;

@Default
private Map<Class<? extends IConfiguration>, IConfiguration> configurations = new HashMap<>();
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,147 @@
package org.sentrysoftware.metricshub.engine.connector.model.monitor.task.source;

/*-
* ╱╲╱╲╱╲╱╲╱╲╱╲╱╲╱╲╱╲╱╲╱╲╱╲╱╲╱╲╱╲╱╲╱╲╱╲╱╲╱╲
* MetricsHub Engine
* ჻჻჻჻჻჻
* Copyright 2023 - 2024 Sentry Software
* ჻჻჻჻჻჻
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
* ╲╱╲╱╲╱╲╱╲╱╲╱╲╱╲╱╲╱╲╱╲╱╲╱╲╱╲╱╲╱╲╱╲╱╲╱╲╱╲╱
*/

import static com.fasterxml.jackson.annotation.Nulls.FAIL;
import static org.sentrysoftware.metricshub.engine.common.helpers.MetricsHubConstants.NEW_LINE;
import static org.sentrysoftware.metricshub.engine.common.helpers.StringHelper.addNonNull;

import com.fasterxml.jackson.annotation.JsonCreator;
import com.fasterxml.jackson.annotation.JsonProperty;
import com.fasterxml.jackson.annotation.JsonSetter;
import com.fasterxml.jackson.databind.annotation.JsonDeserialize;
import java.util.ArrayList;
import java.util.List;
import java.util.StringJoiner;
import java.util.function.UnaryOperator;
import lombok.AllArgsConstructor;
import lombok.Builder;
import lombok.Data;
import lombok.EqualsAndHashCode;
import lombok.NoArgsConstructor;
import lombok.NonNull;
import org.sentrysoftware.metricshub.engine.connector.deserializer.custom.NonBlankDeserializer;
import org.sentrysoftware.metricshub.engine.connector.model.common.ExecuteForEachEntryOf;
import org.sentrysoftware.metricshub.engine.connector.model.monitor.task.source.compute.Compute;
import org.sentrysoftware.metricshub.engine.strategy.source.ISourceProcessor;
import org.sentrysoftware.metricshub.engine.strategy.source.SourceTable;

/**
* Represents a source that executes an jawk script to retrieve data.
*/
@Data
@NoArgsConstructor
@AllArgsConstructor
@EqualsAndHashCode(callSuper = true)
@Builder
public class JawkSource extends Source {

private static final long serialVersionUID = 1L;

/**
* The JAWK script to be executed for the computation task.
*/
@NonNull
@JsonSetter(nulls = FAIL)
private String script;

/**
* The input on which to execute the JAWK task.
*/
@NonNull
@JsonSetter(nulls = FAIL)
@JsonDeserialize(using = NonBlankDeserializer.class)
private String input;

/**
* The separators parameter for the JAWK task.
*/
private String separators;

/**
* Builder for creating instances of {@code JawkSource}.
*
* @param type The type of the source.
* @param computes List of computations to be applied to the source.
* @param forceSerialization Flag indicating whether to force serialization.
* @param key The key associated with the source.
* @param executeForEachEntryOf The execution context for each entry of the source.
* @param script The script to execute.
* @param input The input on which to execute the JAWK task.
* @param separators The separators parameter for the JAWK task.
*/
@Builder
@JsonCreator
public JawkSource(
@JsonProperty(value = "type") String type,
@JsonProperty(value = "computes") List<Compute> computes,
@JsonProperty(value = "forceSerialization") boolean forceSerialization,
@JsonProperty(value = "key") String key,
@JsonProperty(value = "executeForEachEntryOf") ExecuteForEachEntryOf executeForEachEntryOf,
@JsonProperty(value = "script") String script,
@JsonProperty(value = "input") final String input,
@JsonProperty(value = "separators") final String separators
) {
super(type, computes, forceSerialization, key, executeForEachEntryOf);
this.script = script;
this.input = input;
this.separators = separators;
}

@Override
public Source copy() {
return JawkSource
.builder()
.type(type)
.key(key)
.forceSerialization(forceSerialization)
.computes(getComputes() != null ? new ArrayList<>(getComputes()) : null)
.executeForEachEntryOf(executeForEachEntryOf != null ? executeForEachEntryOf.copy() : null)
.script(script)
.input(input)
.separators(separators)
.build();
}

@Override
public void update(UnaryOperator<String> updater) {
// For now, there is nothing to update
}

@Override
public SourceTable accept(ISourceProcessor sourceProcessor) {
return sourceProcessor.process(this);
}

@Override
public String toString() {
final StringJoiner stringJoiner = new StringJoiner(NEW_LINE);

stringJoiner.add(super.toString());

addNonNull(stringJoiner, "- script=", script);
addNonNull(stringJoiner, "- input=", input);
addNonNull(stringJoiner, "- separators=", separators);

return stringJoiner.toString();
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -57,6 +57,7 @@
@JsonSubTypes.Type(value = IpmiSource.class, name = "ipmi"),
@JsonSubTypes.Type(value = CommandLineSource.class, name = "osCommand"),
@JsonSubTypes.Type(value = CommandLineSource.class, name = "commandLine"),
@JsonSubTypes.Type(value = JawkSource.class, name = "jawk"),
@JsonSubTypes.Type(value = SnmpGetSource.class, name = "snmpGet"),
@JsonSubTypes.Type(value = SnmpTableSource.class, name = "snmpTable"),
@JsonSubTypes.Type(value = SqlSource.class, name = "sql"),
Expand Down
Loading

0 comments on commit de3e67c

Please sign in to comment.