Move the compression config to the top-level, change default to gzip

SumoLogic · May 16, 2024 · 013695f · 013695f
1 parent 99b58e2
commit 013695f
Show file tree

Hide file tree

Showing 31 changed files with 73 additions and 66 deletions.
diff --git a/.changelog/3705.added.txt b/.changelog/3705.added.txt
@@ -1 +1 @@
-feat(compression): Make the compression type configurable, default: zstd
+feat(compression): Make the compression type configurable, default: gzip
diff --git a/deploy/helm/sumologic/README.md b/deploy/helm/sumologic/README.md
@@ -432,7 +432,7 @@ The following table lists the configurable parameters of the Sumo Logic chart an
 | `otellogswindows.metrics.enabled`                                                     | Enable OpenTelemetry Logs Collector for Windows Nodes metrics                                                                                                                                                                                                                                              | `true`                                                                                                                                                                               |
 | `otellogswindows.serviceLabels`                                                       | Add custom labels to OpenTelemetry Logs Collector for Windows Nodes Service                                                                                                                                                                                                                                | `{}`                                                                                                                                                                                 |
 | `otellogswindows.additionalDaemonSets`                                                | OpenTelemetry Logs Collector for Windows Nodes Daemonset per node customization options. See [Best Practices](https://help.sumologic.com/docs/send-data/kubernetes/best-practices/#setting-different-resources-on-different-nodes-for-logs-collector).                                                     | `{}`                                                                                                                                                                                 |
-| `metadata.compression`                                                                | Compression for logs, metrics and events                                                                                                                                                                                                                                                                   | `zstd`                                                                                                                                                                               |
+| `compression`                                                                         | Compression for logs, metrics and events                                                                                                                                                                                                                                                                   | `["gzip", "deflate", "snappy", "zstd"]`                                                                                                                                              |
 | `metadata.image.repository`                                                           | Image repository for otelcol docker container.                                                                                                                                                                                                                                                             | ``                                                                                                                                                                                   |
 | `metadata.image.tag`                                                                  | Image tag for otelcol docker container.                                                                                                                                                                                                                                                                    | ``                                                                                                                                                                                   |
 | `metadata.image.pullPolicy`                                                           | Image pullPolicy for otelcol docker container.                                                                                                                                                                                                                                                             | `IfNotPresent`                                                                                                                                                                       |

diff --git a/deploy/helm/sumologic/conf/events/otelcol/config.yaml b/deploy/helm/sumologic/conf/events/otelcol/config.yaml
@@ -2,7 +2,7 @@ exporters:
   sumologic:
     client: {{ include "sumologic.sumo_client" . }}
     endpoint: {{ include "sumologic.events.exporter.endpoint" . }}
-    compression: {{ .Values.metadata.compression }}
+    compression: {{ .Values.compression }}
     log_format: {{ include "sumologic.events.exporter.format" . }}
     sending_queue:
       enabled: true
@@ -19,7 +19,7 @@ exporters:
   sumologic/sumologic-mock:
     client: {{ include "sumologic.sumo_client" . }}
     endpoint: {{ include "sumologic-mock.receiver-endpoint" . }}
-    compression: {{ .Values.metadata.compression }}
+    compression: {{ .Values.compression }}
     log_format: {{ include "sumologic.events.exporter.format" . }}
     sending_queue:
       enabled: true

diff --git a/deploy/helm/sumologic/conf/logs/collector/otelcol/config.yaml b/deploy/helm/sumologic/conf/logs/collector/otelcol/config.yaml
@@ -1,7 +1,7 @@
 exporters:
   otlphttp:
     endpoint: http://${LOGS_METADATA_SVC}.${NAMESPACE}.svc.{{ .Values.sumologic.clusterDNSDomain }}.:4318
-    compression: {{ .Values.metadata.compression }}
+    compression: {{ .Values.compression }}
     sending_queue:
       queue_size: 10
     # this improves load balancing at the cost of more network traffic

diff --git a/deploy/helm/sumologic/conf/logs/otelcol/config.yaml b/deploy/helm/sumologic/conf/logs/otelcol/config.yaml
@@ -8,7 +8,7 @@ exporters:
 {{ if .Values.sumologic.logs.container.enabled }}
   sumologic/containers:
     endpoint: ${SUMO_ENDPOINT_DEFAULT_LOGS_SOURCE}
-    compression: {{ .Values.metadata.compression }}
+    compression: {{ .Values.compression }}
     log_format: {{ include "logs.otelcol.container.exporter.format" . }}
     sending_queue:
       enabled: true
@@ -21,7 +21,7 @@ exporters:
 {{- if eq (include "sumologic-mock.forward-logs-metadata" .) "true" }}
   sumologic/sumologic-mock-containers:
     endpoint: {{ include "sumologic-mock.receiver-endpoint" . }}
-    compression: {{ .Values.metadata.compression }}
+    compression: {{ .Values.compression }}
     log_format: {{ include "logs.otelcol.container.exporter.format" . }}
     sending_queue:
       enabled: true
@@ -37,7 +37,7 @@ exporters:
 {{ if .Values.sumologic.logs.systemd.enabled }}
   sumologic/systemd:
     endpoint: ${SUMO_ENDPOINT_DEFAULT_LOGS_SOURCE}
-    compression: {{ .Values.metadata.compression }}
+    compression: {{ .Values.compression }}
     log_format: json
     sending_queue:
       enabled: true
@@ -50,7 +50,7 @@ exporters:
 {{- if eq (include "sumologic-mock.forward-logs-metadata" .) "true" }}
   sumologic/sumologic-mock-systemd:
     endpoint: {{ include "sumologic-mock.receiver-endpoint" . }}
-    compression: {{ .Values.metadata.compression }}
+    compression: {{ .Values.compression }}
     log_format: json
     sending_queue:
       enabled: true
@@ -67,7 +67,7 @@ exporters:
   sumologic:
     client: {{ include "sumologic.sumo_client" . }}
     endpoint: ${SUMO_ENDPOINT_DEFAULT_OTLP_LOGS_SOURCE}
-    compression: {{ .Values.metadata.compression }}
+    compression: {{ .Values.compression }}
     log_format: otlp
     sending_queue:
       enabled: true
@@ -81,7 +81,7 @@ exporters:
   sumologic/sumologic-mock:
     client: {{ include "sumologic.sumo_client" . }}
     endpoint: {{ include "sumologic-mock.receiver-endpoint" . }}
-    compression: {{ .Values.metadata.compression }}
+    compression: {{ .Values.compression }}
     log_format: otlp
     sending_queue:
       enabled: true

diff --git a/deploy/helm/sumologic/conf/metrics/collector/otelcol/config.yaml b/deploy/helm/sumologic/conf/metrics/collector/otelcol/config.yaml
@@ -7,6 +7,7 @@ exporters:
 {{- end }}
   otlphttp:
     endpoint: http://${METADATA_METRICS_SVC}.${NAMESPACE}.svc.{{ .Values.sumologic.clusterDNSDomain }}.:4318
+    compression: {{ .Values.compression }}
     sending_queue:
       queue_size: 10000
       num_consumers: 10

diff --git a/deploy/helm/sumologic/conf/metrics/otelcol/exporters.yaml b/deploy/helm/sumologic/conf/metrics/otelcol/exporters.yaml
@@ -8,7 +8,7 @@ sumologic/sumologic-mock-default:
   client: {{ include "sumologic.sumo_client" . }}
   metric_format: {{ include "metrics.otelcol.exporter.format" . }}
   endpoint: {{ include "sumologic-mock.receiver-endpoint" . }}
-  compression: {{ .Values.metadata.compression }}
+  compression: {{ .Values.compression }}
   ## Sumo doesn't yet natively support OTLP Histograms
   decompose_otlp_histograms: true
   ## Configuration for sending queue
@@ -30,7 +30,7 @@ sumologic/sumologic-mock-http:
   client: {{ include "sumologic.sumo_client" . }}
   metric_format: prometheus
   endpoint: {{ include "sumologic-mock.receiver-endpoint" . }}
-  compression: {{ .Values.metadata.compression }}
+  compression: {{ .Values.compression }}
   sending_queue:
     enabled: true
 {{- if .Values.metadata.persistence.enabled }}
@@ -49,7 +49,7 @@ sumologic/default:
   client: {{ include "sumologic.sumo_client" . }}
   metric_format: {{ include "metrics.otelcol.exporter.format" . }}
   endpoint: {{ include "metrics.otelcol.exporter.endpoint" . }}
-  compression: {{ .Values.metadata.compression }}
+  compression: {{ .Values.compression }}
   ## Sumo doesn't yet natively support OTLP Histograms
   decompose_otlp_histograms: true
   ## Configuration for sending queue
@@ -71,7 +71,7 @@ sumologic/apiserver:
   client: {{ include "sumologic.sumo_client" . }}
   metric_format: prometheus
   endpoint: ${SUMO_ENDPOINT_APISERVER_METRICS_SOURCE}
-  compression: {{ .Values.metadata.compression }}
+  compression: {{ .Values.compression }}
   sending_queue:
     enabled: true
 {{- if .Values.metadata.persistence.enabled }}
@@ -86,7 +86,7 @@ sumologic/control_plane:
   client: {{ include "sumologic.sumo_client" . }}
   metric_format: prometheus
   endpoint: ${SUMO_ENDPOINT_CONTROL_PLANE_METRICS_SOURCE}
-  compression: {{ .Values.metadata.compression }}
+  compression: {{ .Values.compression }}
   sending_queue:
     enabled: true
 {{- if .Values.metadata.persistence.enabled }}
@@ -101,7 +101,7 @@ sumologic/controller:
   client: {{ include "sumologic.sumo_client" . }}
   metric_format: prometheus
   endpoint: ${SUMO_ENDPOINT_CONTROLLER_METRICS_SOURCE}
-  compression: {{ .Values.metadata.compression }}
+  compression: {{ .Values.compression }}
   sending_queue:
     enabled: true
 {{- if .Values.metadata.persistence.enabled }}
@@ -116,7 +116,7 @@ sumologic/kubelet:
   client: {{ include "sumologic.sumo_client" . }}
   metric_format: prometheus
   endpoint: ${SUMO_ENDPOINT_KUBELET_METRICS_SOURCE}
-  compression: {{ .Values.metadata.compression }}
+  compression: {{ .Values.compression }}
   sending_queue:
     enabled: true
 {{- if .Values.metadata.persistence.enabled }}
@@ -131,7 +131,7 @@ sumologic/node:
   client: {{ include "sumologic.sumo_client" . }}
   metric_format: prometheus
   endpoint: ${SUMO_ENDPOINT_NODE_METRICS_SOURCE}
-  compression: {{ .Values.metadata.compression }}
+  compression: {{ .Values.compression }}
   sending_queue:
     enabled: true
 {{- if .Values.metadata.persistence.enabled }}
@@ -146,7 +146,7 @@ sumologic/scheduler:
   client: {{ include "sumologic.sumo_client" . }}
   metric_format: prometheus
   endpoint: ${SUMO_ENDPOINT_SCHEDULER_METRICS_SOURCE}
-  compression: {{ .Values.metadata.compression }}
+  compression: {{ .Values.compression }}
   sending_queue:
     enabled: true
 {{- if .Values.metadata.persistence.enabled }}
@@ -161,7 +161,7 @@ sumologic/state:
   client: {{ include "sumologic.sumo_client" . }}
   metric_format: prometheus
   endpoint: ${SUMO_ENDPOINT_STATE_METRICS_SOURCE}
-  compression: {{ .Values.metadata.compression }}
+  compression: {{ .Values.compression }}
   sending_queue:
     enabled: true
 {{- if .Values.metadata.persistence.enabled }}

diff --git a/deploy/helm/sumologic/values.yaml b/deploy/helm/sumologic/values.yaml
@@ -6,6 +6,11 @@ fullnameOverride: ""
 ## Use the same namespace as namespaceOverride in 'kube-prometheus-stack.namespaceOverride' if Prometheus setup is also enabled
 namespaceOverride: ""
 
+## Compression for logs, metrics and events
+## Compression types can be gzip, snappy, zstd or deflate
+## Default is gzip
+compression: gzip
+
 sumologic:
   ### Setup
 
@@ -1556,9 +1561,6 @@ metadata:
     ## Add custom labels to all otelcol statefulset PVC (logs and metrics)
     pvcLabels: {}
 
-  ## Compression for logs, metrics and events:
-  compression: zstd
-
   ## Configure metrics pipeline.
   ## This section affects only otelcol provider.
   metrics:

diff --git a/tests/helm/testdata/goldenfile/events_otc/basic.output.yaml b/tests/helm/testdata/goldenfile/events_otc/basic.output.yaml
@@ -15,7 +15,7 @@ data:
     exporters:
       sumologic:
         client: k8s_%CURRENT_CHART_VERSION%
-        compression: zstd
+        compression: gzip
         endpoint: ${SUMO_ENDPOINT_DEFAULT_OTLP_EVENTS_SOURCE}
         log_format: otlp
         sending_queue:

diff --git a/tests/helm/testdata/goldenfile/events_otc/options.output.yaml b/tests/helm/testdata/goldenfile/events_otc/options.output.yaml
@@ -15,7 +15,7 @@ data:
     exporters:
       sumologic:
         client: k8s_%CURRENT_CHART_VERSION%
-        compression: zstd
+        compression: gzip
         endpoint: ${SUMO_ENDPOINT_DEFAULT_OTLP_EVENTS_SOURCE}
         log_format: otlp
         sending_queue:

diff --git a/tests/helm/testdata/goldenfile/events_otc/sumologic-mock.output.yaml b/tests/helm/testdata/goldenfile/events_otc/sumologic-mock.output.yaml
@@ -17,15 +17,15 @@ data:
         verbosity: detailed
       sumologic:
         client: k8s_%CURRENT_CHART_VERSION%
-        compression: zstd
+        compression: gzip
         endpoint: ${SUMO_ENDPOINT_DEFAULT_OTLP_EVENTS_SOURCE}
         log_format: otlp
         sending_queue:
           enabled: true
           storage: file_storage
       sumologic/sumologic-mock:
         client: k8s_%CURRENT_CHART_VERSION%
-        compression: zstd
+        compression: gzip
         endpoint: http://RELEASE-NAME-sumologic-mock.sumologic.svc.cluster.local.:3000/receiver
         log_format: otlp
         sending_queue:

diff --git a/tests/helm/testdata/goldenfile/logs_otc/basic.output.yaml b/tests/helm/testdata/goldenfile/logs_otc/basic.output.yaml
@@ -14,7 +14,7 @@ data:
   config.yaml: |
     exporters:
       otlphttp:
-        compression: zstd
+        compression: gzip
         disable_keep_alives: true
         endpoint: http://${LOGS_METADATA_SVC}.${NAMESPACE}.svc.cluster.local.:4318
         sending_queue:

diff --git a/tests/helm/testdata/goldenfile/logs_otc/debug.output.yaml b/tests/helm/testdata/goldenfile/logs_otc/debug.output.yaml
@@ -16,7 +16,7 @@ data:
       debug:
         verbosity: detailed
       otlphttp:
-        compression: zstd
+        compression: gzip
         disable_keep_alives: true
         endpoint: http://${LOGS_METADATA_SVC}.${NAMESPACE}.svc.cluster.local.:4318
         sending_queue:

diff --git a/tests/helm/testdata/goldenfile/logs_otc/options.output.yaml b/tests/helm/testdata/goldenfile/logs_otc/options.output.yaml
@@ -14,7 +14,7 @@ data:
   config.yaml: |
     exporters:
       otlphttp:
-        compression: zstd
+        compression: gzip
         disable_keep_alives: true
         endpoint: http://${LOGS_METADATA_SVC}.${NAMESPACE}.svc.cluster.local.:4318
         sending_queue:

diff --git a/tests/helm/testdata/goldenfile/logs_otc_daemonset/multiple_multiline.output.yaml b/tests/helm/testdata/goldenfile/logs_otc_daemonset/multiple_multiline.output.yaml
@@ -14,7 +14,7 @@ data:
   config.yaml: |
     exporters:
       otlphttp:
-        compression: zstd
+        compression: gzip
         disable_keep_alives: true
         endpoint: http://${LOGS_METADATA_SVC}.${NAMESPACE}.svc.cluster.local.:4318
         sending_queue:

diff --git a/tests/helm/testdata/goldenfile/metadata_logs_otc/debug.output.yaml b/tests/helm/testdata/goldenfile/metadata_logs_otc/debug.output.yaml
@@ -17,7 +17,7 @@ data:
         verbosity: detailed
       sumologic:
         client: k8s_%CURRENT_CHART_VERSION%
-        compression: zstd
+        compression: gzip
         endpoint: ${SUMO_ENDPOINT_DEFAULT_OTLP_LOGS_SOURCE}
         log_format: otlp
         sending_queue:

diff --git a/tests/helm/testdata/goldenfile/metadata_logs_otc/debug_with_sumologic_mock.output.yaml b/tests/helm/testdata/goldenfile/metadata_logs_otc/debug_with_sumologic_mock.output.yaml
@@ -17,7 +17,7 @@ data:
         verbosity: detailed
       sumologic:
         client: k8s_%CURRENT_CHART_VERSION%
-        compression: zstd
+        compression: gzip
         endpoint: ${SUMO_ENDPOINT_DEFAULT_OTLP_LOGS_SOURCE}
         log_format: otlp
         sending_queue:
@@ -27,7 +27,7 @@ data:
           storage: file_storage
       sumologic/sumologic-mock:
         client: k8s_%CURRENT_CHART_VERSION%
-        compression: zstd
+        compression: gzip
         endpoint: http://RELEASE-NAME-sumologic-mock.sumologic.svc.cluster.local.:3000/receiver
         log_format: otlp
         sending_queue:

diff --git a/tests/helm/testdata/goldenfile/metadata_logs_otc/debug_with_sumologic_mock_http.output.yaml b/tests/helm/testdata/goldenfile/metadata_logs_otc/debug_with_sumologic_mock_http.output.yaml
@@ -16,7 +16,7 @@ data:
       debug:
         verbosity: detailed
       sumologic/containers:
-        compression: zstd
+        compression: gzip
         endpoint: ${SUMO_ENDPOINT_DEFAULT_LOGS_SOURCE}
         log_format: json
         sending_queue:
@@ -25,7 +25,7 @@ data:
           queue_size: 10000
           storage: file_storage
       sumologic/sumologic-mock-containers:
-        compression: zstd
+        compression: gzip
         endpoint: http://RELEASE-NAME-sumologic-mock.sumologic.svc.cluster.local.:3000/receiver
         log_format: json
         sending_queue:
@@ -34,7 +34,7 @@ data:
           queue_size: 10000
           storage: file_storage
       sumologic/sumologic-mock-systemd:
-        compression: zstd
+        compression: gzip
         endpoint: http://RELEASE-NAME-sumologic-mock.sumologic.svc.cluster.local.:3000/receiver
         log_format: json
         sending_queue:
@@ -43,7 +43,7 @@ data:
           queue_size: 10000
           storage: file_storage
       sumologic/systemd:
-        compression: zstd
+        compression: gzip
         endpoint: ${SUMO_ENDPOINT_DEFAULT_LOGS_SOURCE}
         log_format: json
         sending_queue:

diff --git a/tests/helm/testdata/goldenfile/metadata_logs_otc/otel.output.yaml b/tests/helm/testdata/goldenfile/metadata_logs_otc/otel.output.yaml
@@ -15,7 +15,7 @@ data:
     exporters:
       sumologic:
         client: k8s_%CURRENT_CHART_VERSION%
-        compression: zstd
+        compression: gzip
         endpoint: ${SUMO_ENDPOINT_DEFAULT_OTLP_LOGS_SOURCE}
         log_format: otlp
         sending_queue:

diff --git a/tests/helm/testdata/goldenfile/metadata_logs_otc/templates.output.yaml b/tests/helm/testdata/goldenfile/metadata_logs_otc/templates.output.yaml
@@ -15,7 +15,7 @@ data:
     exporters:
       sumologic:
         client: k8s_%CURRENT_CHART_VERSION%
-        compression: zstd
+        compression: gzip
         endpoint: ${SUMO_ENDPOINT_DEFAULT_OTLP_LOGS_SOURCE}
         log_format: otlp
         sending_queue:

diff --git a/tests/helm/testdata/goldenfile/metadata_metrics_otc/additional_endpoints.output.yaml b/tests/helm/testdata/goldenfile/metadata_metrics_otc/additional_endpoints.output.yaml
@@ -15,7 +15,7 @@ data:
     exporters:
       sumologic/default:
         client: k8s_%CURRENT_CHART_VERSION%
-        compression: zstd
+        compression: gzip
         decompose_otlp_histograms: true
         endpoint: ${SUMO_ENDPOINT_DEFAULT_OTLP_METRICS_SOURCE}
         max_request_body_size: 16777216

diff --git a/tests/helm/testdata/goldenfile/metadata_metrics_otc/basic.output.yaml b/tests/helm/testdata/goldenfile/metadata_metrics_otc/basic.output.yaml
@@ -15,7 +15,7 @@ data:
     exporters:
       sumologic/default:
         client: k8s_%CURRENT_CHART_VERSION%
-        compression: zstd
+        compression: gzip
         decompose_otlp_histograms: true
         endpoint: ${SUMO_ENDPOINT_DEFAULT_OTLP_METRICS_SOURCE}
         max_request_body_size: 16777216