Merge pull request #382 from sentrysoftware/feature/issue-380-impleme…

…nt-state-metrics-compression

Issue #380: Implement StateSet metrics compression

metricshub-agent/src/main/java/org/sentrysoftware/metricshub/agent/config/AgentConfig.java

@@ -128,6 +128,10 @@ public class AgentConfig {
 	@JsonSetter(nulls = SKIP)
 	private Map<String, ResourceGroupConfig> resourceGroups = new HashMap<>();
 
+	@Default
+	@JsonSetter(nulls = SKIP)
+	private String stateSetCompression = StateSetMetricCompression.SUPPRESS_ZEROS;
+
 	/**
 	 * Build a new empty instance
 	 *

metricshub-agent/src/main/java/org/sentrysoftware/metricshub/agent/config/ResourceConfig.java

@@ -105,6 +105,8 @@ public class ResourceConfig {
 	@JsonIgnore
 	private Connector connector;
 
+	private String stateSetCompression;
+
 	/**
 	 * Creates and returns a shallow copy of all the fields in this
 	 * ResourceConfig object except the attributes map which is deeply copied.
@@ -135,6 +137,7 @@ public ResourceConfig copy() {
 			.variables(variables)
 			.connectors(connectors)
 			.connector(connector)
+			.stateSetCompression(stateSetCompression)
 			.build();
 	}
 }

metricshub-agent/src/main/java/org/sentrysoftware/metricshub/agent/config/ResourceGroupConfig.java

@@ -74,4 +74,6 @@ public class ResourceGroupConfig {
 	@Default
 	@JsonSetter(nulls = SKIP)
 	private Map<String, ResourceConfig> resources = new HashMap<>();
+
+	private String stateSetCompression;
 }

metricshub-agent/src/main/java/org/sentrysoftware/metricshub/agent/config/StateSetMetricCompression.java

@@ -0,0 +1,43 @@
+package org.sentrysoftware.metricshub.agent.config;
+
+/*-
+ * ╱╲╱╲╱╲╱╲╱╲╱╲╱╲╱╲╱╲╱╲╱╲╱╲╱╲╱╲╱╲╱╲╱╲╱╲╱╲╱╲
+ * MetricsHub Agent
+ * ჻჻჻჻჻჻
+ * Copyright 2023 - 2024 Sentry Software
+ * ჻჻჻჻჻჻
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ * ╲╱╲╱╲╱╲╱╲╱╲╱╲╱╲╱╲╱╲╱╲╱╲╱╲╱╲╱╲╱╲╱╲╱╲╱╲╱╲╱
+ */
+
+import lombok.AccessLevel;
+import lombok.NoArgsConstructor;
+
+/**
+ * StateSetMetricCompression represents the possible compression methods for the state set metrics.
+ */
+@NoArgsConstructor(access = AccessLevel.PRIVATE)
+public class StateSetMetricCompression {
+
+	/**
+	 * Compression mechanism that records the zero value only on the initial state change,
+	 * retaining only the non-zero metric value thereafter.
+	 */
+	public static final String SUPPRESS_ZEROS = "suppressZeros";
+
+	/**
+	 * No compression, all zero and non-zero values are recorded.
+	 */
+	public static final String NONE = "none";
+}

metricshub-agent/src/main/java/org/sentrysoftware/metricshub/agent/helper/ConfigHelper.java

@@ -520,6 +520,11 @@ private static void normalizeResourceConfigUsingAgentConfig(
 			resourceConfig.setJobTimeout(agentConfig.getJobTimeout());
 		}
 
+		// Set the state set compression
+		if (resourceConfig.getStateSetCompression() == null) {
+			resourceConfig.setStateSetCompression(agentConfig.getStateSetCompression());
+		}
+
 		// Set agent attributes in the agent configuration attributes map
 		final Map<String, String> attributes = new HashMap<>();
 		mergeAttributes(agentConfig.getAttributes(), attributes);
@@ -598,6 +603,11 @@ private static void normalizeResourceConfig(
 			resourceConfig.setJobTimeout(resourceGroupConfig.getJobTimeout());
 		}
 
+		// Set the state set compression
+		if (resourceConfig.getStateSetCompression() == null) {
+			resourceConfig.setStateSetCompression(resourceGroupConfig.getStateSetCompression());
+		}
+
 		// Set agent attributes in the resource group attributes map
 		final Map<String, String> attributes = new HashMap<>();
 		mergeAttributes(resourceGroupConfig.getAttributes(), attributes);
@@ -672,6 +682,11 @@ private static void normalizeResourceGroupConfig(
 			resourceGroupConfig.setJobTimeout(agentConfig.getJobTimeout());
 		}
 
+		// Set the state set compression
+		if (resourceGroupConfig.getStateSetCompression() == null) {
+			resourceGroupConfig.setStateSetCompression(agentConfig.getStateSetCompression());
+		}
+
 		// Set agent attributes in the resource group attributes map
 		final Map<String, String> attributes = new HashMap<>();
 		mergeAttributes(agentConfig.getAttributes(), attributes);

metricshub-agent/src/main/java/org/sentrysoftware/metricshub/agent/service/signal/AbstractNotCompressedStateMetricObserver.java

@@ -0,0 +1,62 @@
+package org.sentrysoftware.metricshub.agent.service.signal;
+
+/*-
+ * ╱╲╱╲╱╲╱╲╱╲╱╲╱╲╱╲╱╲╱╲╱╲╱╲╱╲╱╲╱╲╱╲╱╲╱╲╱╲╱╲
+ * MetricsHub Agent
+ * ჻჻჻჻჻჻
+ * Copyright 2023 - 2024 Sentry Software
+ * ჻჻჻჻჻჻
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ * ╲╱╲╱╲╱╲╱╲╱╲╱╲╱╲╱╲╱╲╱╲╱╲╱╲╱╲╱╲╱╲╱╲╱╲╱╲╱╲╱
+ */
+
+import io.opentelemetry.api.common.Attributes;
+import io.opentelemetry.api.metrics.Meter;
+import io.opentelemetry.api.metrics.ObservableDoubleMeasurement;
+import org.sentrysoftware.metricshub.engine.telemetry.metric.StateSetMetric;
+
+/**
+ * Abstract class for observing state set metrics without applying any compression.
+ * Extends {@link AbstractStateMetricObserver}.
+ */
+public abstract class AbstractNotCompressedStateMetricObserver extends AbstractStateMetricObserver {
+
+	/**
+	 * Constructs a new {@code AbstractNotCompressedStateSetMetricObserver} with the specified parameters.
+	 *
+	 * @param meter       The OpenTelemetry meter to use for creating the metric.
+	 * @param attributes  The attributes to associate with the metric.
+	 * @param metricName  The name of the metric.
+	 * @param unit        The unit of the metric.
+	 * @param description The description of the metric.
+	 * @param state       The state used to observe the metric. E.g. "ok".
+	 * @param metric      The StateSetMetric to observe.
+	 */
+	protected AbstractNotCompressedStateMetricObserver(
+		final Meter meter,
+		final Attributes attributes,
+		final String metricName,
+		final String unit,
+		final String description,
+		final String state,
+		final StateSetMetric metric
+	) {
+		super(meter, attributes, metricName, unit, description, state, metric);
+	}
+
+	@Override
+	protected void recordMetricValue(final ObservableDoubleMeasurement recorder, final String value) {
+		recorder.record(value.equalsIgnoreCase(state) ? 1 : 0, attributes);
+	}
+}

metricshub-agent/src/main/java/org/sentrysoftware/metricshub/agent/service/signal/AbstractStateMetricObserver.java

@@ -38,6 +38,17 @@
 @ToString(callSuper = true)
 public abstract class AbstractStateMetricObserver extends AbstractMetricObserver {
 
+	/**
+	 * Constructs a new {@code AbstractStateMetricObserver} with the specified parameters.
+	 *
+	 * @param meter       The OpenTelemetry meter to use for creating the metric.
+	 * @param attributes  The attributes to associate with the metric.
+	 * @param metricName  The name of the metric.
+	 * @param unit        The unit of the metric.
+	 * @param description The description of the metric.
+	 * @param state       The state used to observe the metric. E.g. "ok"
+	 * @param metric      The StateSetMetric to observe.
+	 */
 	protected AbstractStateMetricObserver(
 		final Meter meter,
 		final Attributes attributes,
@@ -56,20 +67,28 @@ protected AbstractStateMetricObserver(
 	protected StateSetMetric metric;
 
 	/**
-	 * Observe the given state metric value
+	 * Observe the given state metric value.
 	 *
 	 * @param recorder An interface for observing measurements with double values.
 	 */
 	protected void observeStateMetric(final ObservableDoubleMeasurement recorder) {
-		getMetricValue().ifPresent(value -> recorder.record(value.equalsIgnoreCase(state) ? 1 : 0, attributes));
+		getMetricValue().ifPresent(value -> recordMetricValue(recorder, value));
 	}
 
+	/**
+	 * Record the metric value using the given recorder.
+	 *
+	 * @param recorder The recorder to record the metric value.
+	 * @param value    The value as String to be recorded as a double value.
+	 */
+	protected abstract void recordMetricValue(ObservableDoubleMeasurement recorder, String value);
+
 	/**
 	 * Get the metric value
 	 *
 	 * @return Optional of a {@link String} value
 	 */
-	public Optional<String> getMetricValue() {
+	protected Optional<String> getMetricValue() {
 		if (metric != null && metric.isUpdated()) {
 			return Optional.ofNullable(metric.getValue());
 		}

metricshub-agent/src/main/java/org/sentrysoftware/metricshub/agent/service/signal/AbstractSuppressZerosStateMetricObserver.java

@@ -0,0 +1,69 @@
+package org.sentrysoftware.metricshub.agent.service.signal;
+
+/*-
+ * ╱╲╱╲╱╲╱╲╱╲╱╲╱╲╱╲╱╲╱╲╱╲╱╲╱╲╱╲╱╲╱╲╱╲╱╲╱╲╱╲
+ * MetricsHub Agent
+ * ჻჻჻჻჻჻
+ * Copyright 2023 - 2024 Sentry Software
+ * ჻჻჻჻჻჻
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ * ╲╱╲╱╲╱╲╱╲╱╲╱╲╱╲╱╲╱╲╱╲╱╲╱╲╱╲╱╲╱╲╱╲╱╲╱╲╱╲╱
+ */
+
+import io.opentelemetry.api.common.Attributes;
+import io.opentelemetry.api.metrics.Meter;
+import io.opentelemetry.api.metrics.ObservableDoubleMeasurement;
+import org.sentrysoftware.metricshub.engine.telemetry.metric.StateSetMetric;
+
+/**
+ * Abstract class for observing state set metrics by applying a suppression of zero values.
+ * Extends {@link AbstractStateMetricObserver}.
+ */
+public abstract class AbstractSuppressZerosStateMetricObserver extends AbstractStateMetricObserver {
+
+	/**
+	 * Constructs a new {@code AbstractSuppressZerosStateMetricObserver} with the specified parameters.
+	 *
+	 * @param meter       The OpenTelemetry meter to use for creating the metric.
+	 * @param attributes  The attributes to associate with the metric.
+	 * @param metricName  The name of the metric.
+	 * @param unit        The unit of the metric.
+	 * @param description The description of the metric.
+	 * @param state       The state used to observe the metric. E.g. "ok".
+	 * @param metric      The StateSetMetric to observe.
+	 */
+	protected AbstractSuppressZerosStateMetricObserver(
+		final Meter meter,
+		final Attributes attributes,
+		final String metricName,
+		final String unit,
+		final String description,
+		final String state,
+		final StateSetMetric metric
+	) {
+		super(meter, attributes, metricName, unit, description, state, metric);
+	}
+
+	@Override
+	protected void recordMetricValue(final ObservableDoubleMeasurement recorder, final String value) {
+		if (value.equalsIgnoreCase(state)) {
+			recorder.record(1, attributes);
+		} else {
+			final String previousValue = metric.getPreviousValue();
+			if (previousValue != null && !previousValue.equalsIgnoreCase(value) && previousValue.equalsIgnoreCase(state)) {
+				recorder.record(0, attributes);
+			}
+		}
+	}
+}

metricshub-agent/src/main/java/org/sentrysoftware/metricshub/agent/service/signal/CounterStateMetricObserver.java

@@ -30,7 +30,7 @@
  * Observer for a counter state metric, extending {@link AbstractStateMetricObserver}.
  * This observer initializes the metric and provides a callback for observing state changes.
  */
-public class CounterStateMetricObserver extends AbstractStateMetricObserver {
+public class CounterStateMetricObserver extends AbstractNotCompressedStateMetricObserver {
 
 	/**
 	 * Constructs a {@code CounterStateMetricObserver} with the specified parameters.

metricshub-agent/src/main/java/org/sentrysoftware/metricshub/agent/service/signal/CounterSuppressZerosStateMetricObserver.java

@@ -0,0 +1,63 @@
+package org.sentrysoftware.metricshub.agent.service.signal;
+
+/*-
+ * ╱╲╱╲╱╲╱╲╱╲╱╲╱╲╱╲╱╲╱╲╱╲╱╲╱╲╱╲╱╲╱╲╱╲╱╲╱╲╱╲
+ * MetricsHub Agent
+ * ჻჻჻჻჻჻
+ * Copyright 2023 - 2024 Sentry Software
+ * ჻჻჻჻჻჻
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ * ╲╱╲╱╲╱╲╱╲╱╲╱╲╱╲╱╲╱╲╱╲╱╲╱╲╱╲╱╲╱╲╱╲╱╲╱╲╱╲╱
+ */
+
+import io.opentelemetry.api.common.Attributes;
+import io.opentelemetry.api.metrics.Meter;
+import lombok.Builder;
+import org.sentrysoftware.metricshub.engine.telemetry.metric.StateSetMetric;
+
+/**
+ * Observer for a counter state metric with suppression of zero values, extending {@link AbstractSuppressZerosStateMetricObserver}.
+ * This observer initializes the metric and provides a callback for observing state changes.
+ */
+public class CounterSuppressZerosStateMetricObserver extends AbstractSuppressZerosStateMetricObserver {
+
+	/**
+	 * Constructs a {@code CounterSuppressZerosStateMetricObserver} with the specified parameters.
+	 *
+	 * @param meter       The OpenTelemetry meter to use for creating the metric.
+	 * @param attributes  The attributes to associate with the metric.
+	 * @param metricName  The name of the metric.
+	 * @param unit        The unit of the metric.
+	 * @param description The description of the metric.
+	 * @param state       The state used to observe the metric. E.g. "ok".
+	 * @param metric      The StateSetMetric to observe.
+	 */
+	@Builder(setterPrefix = "with")
+	public CounterSuppressZerosStateMetricObserver(
+		final Meter meter,
+		final Attributes attributes,
+		final String metricName,
+		final String unit,
+		final String description,
+		final String state,
+		final StateSetMetric metric
+	) {
+		super(meter, attributes, metricName, unit, description, state, metric);
+	}
+
+	@Override
+	public void init() {
+		newDoubleCounterBuilder().buildWithCallback(super::observeStateMetric);
+	}
+}

metricshub-agent/src/main/java/org/sentrysoftware/metricshub/agent/service/signal/GaugeStateMetricObserver.java

@@ -35,7 +35,7 @@
 @Data
 @EqualsAndHashCode(callSuper = true)
 @ToString(callSuper = true)
-public class GaugeStateMetricObserver extends AbstractStateMetricObserver {
+public class GaugeStateMetricObserver extends AbstractNotCompressedStateMetricObserver {
 
 	/**
 	 * Constructs a new {@code GaugeStateMetricObserver} with the specified parameters.