feat(telemetry): add delta temporality support for OTLP metrics

api/v1alpha1/envoyproxy_metric_types.go

@@ -98,8 +98,17 @@ type ProxyOpenTelemetrySink struct {
 	// +kubebuilder:validation:Maximum=65535
 	// +kubebuilder:default=4317
 	Port int32 `json:"port,omitempty"`
-
-	// TODO: add support for customizing OpenTelemetry sink in https://www.envoyproxy.io/docs/envoy/latest/api-v3/extensions/stat_sinks/open_telemetry/v3/open_telemetry.proto#envoy-v3-api-msg-extensions-stat-sinks-open-telemetry-v3-sinkconfig
+	// ReportCountersAsDeltas configures the OpenTelemetry sink to report
+	// counters as delta temporality instead of cumulative.
+	//
+	// +optional
+	ReportCountersAsDeltas *bool `json:"reportCountersAsDeltas,omitempty"`
+	// ReportHistogramsAsDeltas configures the OpenTelemetry sink to report
+	// histograms as delta temporality instead of cumulative.
+	// Required for backends like Elastic that drop cumulative histograms.
+	//
+	// +optional
+	ReportHistogramsAsDeltas *bool `json:"reportHistogramsAsDeltas,omitempty"`
 }
 
 type ProxyPrometheusProvider struct {

charts/gateway-crds-helm/templates/generated/gateway.envoyproxy.io_envoyproxies.yaml

@@ -14773,6 +14773,17 @@ spec:
                                   maximum: 65535
                                   minimum: 0
                                   type: integer
+                                reportCountersAsDeltas:
+                                  description: |-
+                                    ReportCountersAsDeltas configures the OpenTelemetry sink to report
+                                    counters as delta temporality instead of cumulative.
+                                  type: boolean
+                                reportHistogramsAsDeltas:
+                                  description: |-
+                                    ReportHistogramsAsDeltas configures the OpenTelemetry sink to report
+                                    histograms as delta temporality instead of cumulative.
+                                    Required for backends like Elastic that drop cumulative histograms.
+                                  type: boolean
                               type: object
                               x-kubernetes-validations:
                               - message: host or backendRefs needs to be set

charts/gateway-helm/crds/generated/gateway.envoyproxy.io_envoyproxies.yaml

@@ -14772,6 +14772,17 @@ spec:
                                   maximum: 65535
                                   minimum: 0
                                   type: integer
+                                reportCountersAsDeltas:
+                                  description: |-
+                                    ReportCountersAsDeltas configures the OpenTelemetry sink to report
+                                    counters as delta temporality instead of cumulative.
+                                  type: boolean
+                                reportHistogramsAsDeltas:
+                                  description: |-
+                                    ReportHistogramsAsDeltas configures the OpenTelemetry sink to report
+                                    histograms as delta temporality instead of cumulative.
+                                    Required for backends like Elastic that drop cumulative histograms.
+                                  type: boolean
                               type: object
                               x-kubernetes-validations:
                               - message: host or backendRefs needs to be set

internal/xds/bootstrap/bootstrap.go

@@ -15,6 +15,7 @@ import (
 	"text/template"
 
 	"k8s.io/apimachinery/pkg/util/sets"
+	"k8s.io/utils/ptr"
 
 	egv1a1 "github.com/envoyproxy/gateway/api/v1alpha1"
 	netutils "github.com/envoyproxy/gateway/internal/utils/net"
@@ -119,6 +120,10 @@ type metricSink struct {
 	Address string
 	// Port is the port of the XDS Server that Envoy is managed by.
 	Port uint32
+	// ReportCountersAsDeltas configures counters to use delta temporality.
+	ReportCountersAsDeltas bool
+	// ReportHistogramsAsDeltas configures histograms to use delta temporality.
+	ReportHistogramsAsDeltas bool
 }
 
 type adminServerParameters struct {
@@ -218,8 +223,10 @@ func GetRenderedBootstrapConfig(opts *RenderBootstrapConfigOptions) (string, err
 			addresses.Insert(addr)
 
 			metricSinks = append(metricSinks, metricSink{
-				Address: host,
-				Port:    port,
+				Address:                  host,
+				Port:                     port,
+				ReportCountersAsDeltas:   ptr.Deref(sink.OpenTelemetry.ReportCountersAsDeltas, false),
+				ReportHistogramsAsDeltas: ptr.Deref(sink.OpenTelemetry.ReportHistogramsAsDeltas, false),
 			})
 		}
 

internal/xds/bootstrap/bootstrap.yaml.tpl

@@ -65,6 +65,12 @@ stats_sinks:
     grpc_service:
       envoy_grpc:
         cluster_name: otel_metric_sink_{{ $idx }}
+    {{- if $sink.ReportCountersAsDeltas }}
+    report_counters_as_deltas: true
+    {{- end }}
+    {{- if $sink.ReportHistogramsAsDeltas }}
+    report_histograms_as_deltas: true
+    {{- end }}
 {{- end }}
 {{- end }}
 static_resources:

internal/xds/bootstrap/bootstrap_test.go

@@ -140,6 +140,28 @@ func TestGetRenderedBootstrapConfig(t *testing.T) {
 				SdsConfig: sds,
 			},
 		},
+		{
+			name: "otel-metrics-delta-temporality",
+			opts: &RenderBootstrapConfigOptions{
+				ProxyMetrics: &egv1a1.ProxyMetrics{
+					Prometheus: &egv1a1.ProxyPrometheusProvider{
+						Disable: true,
+					},
+					Sinks: []egv1a1.ProxyMetricSink{
+						{
+							Type: egv1a1.MetricSinkTypeOpenTelemetry,
+							OpenTelemetry: &egv1a1.ProxyOpenTelemetrySink{
+								Host:                     ptr.To("otel-collector.monitoring.svc"),
+								Port:                     4317,
+								ReportCountersAsDeltas:   ptr.To(true),
+								ReportHistogramsAsDeltas: ptr.To(true),
+							},
+						},
+					},
+				},
+				SdsConfig: sds,
+			},
+		},
 		{
 			name: "custom-stats-matcher",
 			opts: &RenderBootstrapConfigOptions{

internal/xds/bootstrap/testdata/render/otel-metrics-delta-temporality.yaml

@@ -0,0 +1,130 @@
+admin:
+  access_log:
+  - name: envoy.access_loggers.file
+    typed_config:
+      "@type": type.googleapis.com/envoy.extensions.access_loggers.file.v3.FileAccessLog
+      path: /dev/null
+  address:
+    socket_address:
+      address: 127.0.0.1
+      port_value: 19000
+cluster_manager:
+  local_cluster_name: local_cluster
+node:
+  locality:
+    zone: $(ENVOY_SERVICE_ZONE)
+layered_runtime:
+  layers:
+  - name: global_config
+    static_layer:
+      envoy.restart_features.use_eds_cache_for_ads: true
+      re2.max_program_size.error_level: 4294967295
+      re2.max_program_size.warn_level: 1000
+dynamic_resources:
+  ads_config:
+    api_type: DELTA_GRPC
+    transport_api_version: V3
+    grpc_services:
+    - envoy_grpc:
+        cluster_name: xds_cluster
+    set_node_on_first_message_only: true
+  lds_config:
+    ads: {}
+    resource_api_version: V3
+  cds_config:
+    ads: {}
+    resource_api_version: V3
+stats_sinks:
+- name: "envoy.stat_sinks.open_telemetry"
+  typed_config:
+    "@type": type.googleapis.com/envoy.extensions.stat_sinks.open_telemetry.v3.SinkConfig
+    grpc_service:
+      envoy_grpc:
+        cluster_name: otel_metric_sink_0
+    report_counters_as_deltas: true
+    report_histograms_as_deltas: true
+static_resources:
+  clusters:
+  - name: otel_metric_sink_0
+    connect_timeout: 0.250s
+    type: STRICT_DNS
+    typed_extension_protocol_options:
+      envoy.extensions.upstreams.http.v3.HttpProtocolOptions:
+        "@type": "type.googleapis.com/envoy.extensions.upstreams.http.v3.HttpProtocolOptions"
+        explicit_http_config:
+          http2_protocol_options: {}
+    lb_policy: ROUND_ROBIN
+    load_assignment:
+      cluster_name: otel_metric_sink_0
+      endpoints:
+      - lb_endpoints:
+        - endpoint:
+            address:
+              socket_address:
+                address: otel-collector.monitoring.svc
+                port_value: 4317
+  - connect_timeout: 10s
+    eds_cluster_config:
+      eds_config:
+        ads: {}
+        resource_api_version: 'V3'
+      service_name: local_cluster
+    load_balancing_policy:
+      policies:
+      - typed_extension_config:
+          name: 'envoy.load_balancing_policies.least_request'
+          typed_config:
+            '@type': 'type.googleapis.com/envoy.extensions.load_balancing_policies.least_request.v3.LeastRequest'
+            locality_lb_config:
+              zone_aware_lb_config:
+                min_cluster_size: '1'
+    name: local_cluster
+    type: EDS
+  - connect_timeout: 10s
+    load_assignment:
+      cluster_name: xds_cluster
+      endpoints:
+      - load_balancing_weight: 1
+        lb_endpoints:
+        - load_balancing_weight: 1
+          endpoint:
+            address:
+              socket_address:
+                address: envoy-gateway
+                port_value: 18000
+    typed_extension_protocol_options:
+      envoy.extensions.upstreams.http.v3.HttpProtocolOptions:
+        "@type": "type.googleapis.com/envoy.extensions.upstreams.http.v3.HttpProtocolOptions"
+        explicit_http_config:
+          http2_protocol_options:
+            connection_keepalive:
+              interval: 30s
+              timeout: 5s
+    name: xds_cluster
+    type: STRICT_DNS
+    transport_socket:
+      name: envoy.transport_sockets.tls
+      typed_config:
+        "@type": type.googleapis.com/envoy.extensions.transport_sockets.tls.v3.UpstreamTlsContext
+        common_tls_context:
+          tls_params:
+            tls_maximum_protocol_version: TLSv1_3
+          tls_certificate_sds_secret_configs:
+          - name: xds_certificate
+            sds_config:
+              path_config_source:
+                path: /sds/xds-certificate.json
+              resource_api_version: V3
+          validation_context_sds_secret_config:
+            name: xds_trusted_ca
+            sds_config:
+              path_config_source:
+                path: /sds/xds-trusted-ca.json
+              resource_api_version: V3
+overload_manager:
+  refresh_interval: 0.25s
+  resource_monitors:
+  - name: "envoy.resource_monitors.global_downstream_max_connections"
+    typed_config:
+      "@type": type.googleapis.com/envoy.extensions.resource_monitors.downstream_connections.v3.DownstreamConnectionsConfig
+      max_active_downstream_connections: 50000

site/content/en/latest/api/extension_types.md

@@ -4051,6 +4051,8 @@ _Appears in:_
 | `backendSettings` | _[ClusterSettings](#clustersettings)_ |  false  |  | BackendSettings holds configuration for managing the connection<br />to the backend. |
 | `host` | _string_ |  false  |  | Host define the service hostname.<br />Deprecated: Use BackendRefs instead. |
 | `port` | _integer_ |  false  | 4317 | Port defines the port the service is exposed on.<br />Deprecated: Use BackendRefs instead. |
+| `reportCountersAsDeltas` | _boolean_ |  false  |  | ReportCountersAsDeltas configures the OpenTelemetry sink to report<br />counters as delta temporality instead of cumulative. |
+| `reportHistogramsAsDeltas` | _boolean_ |  false  |  | ReportHistogramsAsDeltas configures the OpenTelemetry sink to report<br />histograms as delta temporality instead of cumulative.<br />Required for backends like Elastic that drop cumulative histograms. |
 
 
 #### ProxyPrometheusProvider

test/helm/gateway-crds-helm/all.out.yaml

@@ -43342,6 +43342,17 @@ spec:
                                   maximum: 65535
                                   minimum: 0
                                   type: integer
+                                reportCountersAsDeltas:
+                                  description: |-
+                                    ReportCountersAsDeltas configures the OpenTelemetry sink to report
+                                    counters as delta temporality instead of cumulative.
+                                  type: boolean
+                                reportHistogramsAsDeltas:
+                                  description: |-
+                                    ReportHistogramsAsDeltas configures the OpenTelemetry sink to report
+                                    histograms as delta temporality instead of cumulative.
+                                    Required for backends like Elastic that drop cumulative histograms.
+                                  type: boolean
                               type: object
                               x-kubernetes-validations:
                               - message: host or backendRefs needs to be set

test/helm/gateway-crds-helm/envoy-gateway-crds.out.yaml

@@ -22522,6 +22522,17 @@ spec:
                                   maximum: 65535
                                   minimum: 0
                                   type: integer
+                                reportCountersAsDeltas:
+                                  description: |-
+                                    ReportCountersAsDeltas configures the OpenTelemetry sink to report
+                                    counters as delta temporality instead of cumulative.
+                                  type: boolean
+                                reportHistogramsAsDeltas:
+                                  description: |-
+                                    ReportHistogramsAsDeltas configures the OpenTelemetry sink to report
+                                    histograms as delta temporality instead of cumulative.
+                                    Required for backends like Elastic that drop cumulative histograms.
+                                  type: boolean
                               type: object
                               x-kubernetes-validations:
                               - message: host or backendRefs needs to be set