Skip to content

Commit 963f476

Browse files
codefromthecryptcodefromthecrypt
committed
feat(telemetry): add delta temporality support for OTLP metrics
Adds `reportCountersAsDeltas` and `reportHistogramsAsDeltas` fields to OpenTelemetry metric sink configuration. Required for backends like Elastic that drop cumulative histograms: https://www.elastic.co/docs/reference/opentelemetry/compatibility/limitations Signed-off-by: Adrian Cole <[email protected]>
1 parent b9123a8 commit 963f476

File tree

11 files changed

+234
-4
lines changed

11 files changed

+234
-4
lines changed

‎api/v1alpha1/envoyproxy_metric_types.go‎

Lines changed: 11 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -98,8 +98,17 @@ type ProxyOpenTelemetrySink struct {
9898
// +kubebuilder:validation:Maximum=65535
9999
// +kubebuilder:default=4317
100100
Port int32 `json:"port,omitempty"`
101-
102-
// TODO: add support for customizing OpenTelemetry sink in https://www.envoyproxy.io/docs/envoy/latest/api-v3/extensions/stat_sinks/open_telemetry/v3/open_telemetry.proto#envoy-v3-api-msg-extensions-stat-sinks-open-telemetry-v3-sinkconfig
101+
// ReportCountersAsDeltas configures the OpenTelemetry sink to report
102+
// counters as delta temporality instead of cumulative.
103+
//
104+
// +optional
105+
ReportCountersAsDeltas *bool `json:"reportCountersAsDeltas,omitempty"`
106+
// ReportHistogramsAsDeltas configures the OpenTelemetry sink to report
107+
// histograms as delta temporality instead of cumulative.
108+
// Required for backends like Elastic that drop cumulative histograms.
109+
//
110+
// +optional
111+
ReportHistogramsAsDeltas *bool `json:"reportHistogramsAsDeltas,omitempty"`
103112
}
104113

105114
type ProxyPrometheusProvider struct {

‎api/v1alpha1/zz_generated.deepcopy.go‎

Lines changed: 10 additions & 0 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎charts/gateway-crds-helm/templates/generated/gateway.envoyproxy.io_envoyproxies.yaml‎

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -14773,6 +14773,17 @@ spec:
1477314773
maximum: 65535
1477414774
minimum: 0
1477514775
type: integer
14776+
reportCountersAsDeltas:
14777+
description: |-
14778+
ReportCountersAsDeltas configures the OpenTelemetry sink to report
14779+
counters as delta temporality instead of cumulative.
14780+
type: boolean
14781+
reportHistogramsAsDeltas:
14782+
description: |-
14783+
ReportHistogramsAsDeltas configures the OpenTelemetry sink to report
14784+
histograms as delta temporality instead of cumulative.
14785+
Required for backends like Elastic that drop cumulative histograms.
14786+
type: boolean
1477614787
type: object
1477714788
x-kubernetes-validations:
1477814789
- message: host or backendRefs needs to be set

‎charts/gateway-helm/crds/generated/gateway.envoyproxy.io_envoyproxies.yaml‎

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -14772,6 +14772,17 @@ spec:
1477214772
maximum: 65535
1477314773
minimum: 0
1477414774
type: integer
14775+
reportCountersAsDeltas:
14776+
description: |-
14777+
ReportCountersAsDeltas configures the OpenTelemetry sink to report
14778+
counters as delta temporality instead of cumulative.
14779+
type: boolean
14780+
reportHistogramsAsDeltas:
14781+
description: |-
14782+
ReportHistogramsAsDeltas configures the OpenTelemetry sink to report
14783+
histograms as delta temporality instead of cumulative.
14784+
Required for backends like Elastic that drop cumulative histograms.
14785+
type: boolean
1477514786
type: object
1477614787
x-kubernetes-validations:
1477714788
- message: host or backendRefs needs to be set

‎internal/xds/bootstrap/bootstrap.go‎

Lines changed: 9 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -15,6 +15,7 @@ import (
1515
"text/template"
1616

1717
"k8s.io/apimachinery/pkg/util/sets"
18+
"k8s.io/utils/ptr"
1819

1920
egv1a1 "github.com/envoyproxy/gateway/api/v1alpha1"
2021
netutils "github.com/envoyproxy/gateway/internal/utils/net"
@@ -119,6 +120,10 @@ type metricSink struct {
119120
Address string
120121
// Port is the port of the XDS Server that Envoy is managed by.
121122
Port uint32
123+
// ReportCountersAsDeltas configures counters to use delta temporality.
124+
ReportCountersAsDeltas bool
125+
// ReportHistogramsAsDeltas configures histograms to use delta temporality.
126+
ReportHistogramsAsDeltas bool
122127
}
123128

124129
type adminServerParameters struct {
@@ -218,8 +223,10 @@ func GetRenderedBootstrapConfig(opts *RenderBootstrapConfigOptions) (string, err
218223
addresses.Insert(addr)
219224

220225
metricSinks = append(metricSinks, metricSink{
221-
Address: host,
222-
Port: port,
226+
Address: host,
227+
Port: port,
228+
ReportCountersAsDeltas: ptr.Deref(sink.OpenTelemetry.ReportCountersAsDeltas, false),
229+
ReportHistogramsAsDeltas: ptr.Deref(sink.OpenTelemetry.ReportHistogramsAsDeltas, false),
223230
})
224231
}
225232

‎internal/xds/bootstrap/bootstrap.yaml.tpl‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -65,6 +65,12 @@ stats_sinks:
6565
grpc_service:
6666
envoy_grpc:
6767
cluster_name: otel_metric_sink_{{ $idx }}
68+
{{- if $sink.ReportCountersAsDeltas }}
69+
report_counters_as_deltas: true
70+
{{- end }}
71+
{{- if $sink.ReportHistogramsAsDeltas }}
72+
report_histograms_as_deltas: true
73+
{{- end }}
6874
{{- end }}
6975
{{- end }}
7076
static_resources:

‎internal/xds/bootstrap/bootstrap_test.go‎

Lines changed: 22 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -140,6 +140,28 @@ func TestGetRenderedBootstrapConfig(t *testing.T) {
140140
SdsConfig: sds,
141141
},
142142
},
143+
{
144+
name: "otel-metrics-delta-temporality",
145+
opts: &RenderBootstrapConfigOptions{
146+
ProxyMetrics: &egv1a1.ProxyMetrics{
147+
Prometheus: &egv1a1.ProxyPrometheusProvider{
148+
Disable: true,
149+
},
150+
Sinks: []egv1a1.ProxyMetricSink{
151+
{
152+
Type: egv1a1.MetricSinkTypeOpenTelemetry,
153+
OpenTelemetry: &egv1a1.ProxyOpenTelemetrySink{
154+
Host: ptr.To("otel-collector.monitoring.svc"),
155+
Port: 4317,
156+
ReportCountersAsDeltas: ptr.To(true),
157+
ReportHistogramsAsDeltas: ptr.To(true),
158+
},
159+
},
160+
},
161+
},
162+
SdsConfig: sds,
163+
},
164+
},
143165
{
144166
name: "custom-stats-matcher",
145167
opts: &RenderBootstrapConfigOptions{

‎internal/xds/bootstrap/testdata/render/otel-metrics-delta-temporality.yaml‎

Lines changed: 130 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,130 @@
1+
admin:
2+
access_log:
3+
- name: envoy.access_loggers.file
4+
typed_config:
5+
"@type": type.googleapis.com/envoy.extensions.access_loggers.file.v3.FileAccessLog
6+
path: /dev/null
7+
address:
8+
socket_address:
9+
address: 127.0.0.1
10+
port_value: 19000
11+
cluster_manager:
12+
local_cluster_name: local_cluster
13+
node:
14+
locality:
15+
zone: $(ENVOY_SERVICE_ZONE)
16+
layered_runtime:
17+
layers:
18+
- name: global_config
19+
static_layer:
20+
envoy.restart_features.use_eds_cache_for_ads: true
21+
re2.max_program_size.error_level: 4294967295
22+
re2.max_program_size.warn_level: 1000
23+
dynamic_resources:
24+
ads_config:
25+
api_type: DELTA_GRPC
26+
transport_api_version: V3
27+
grpc_services:
28+
- envoy_grpc:
29+
cluster_name: xds_cluster
30+
set_node_on_first_message_only: true
31+
lds_config:
32+
ads: {}
33+
resource_api_version: V3
34+
cds_config:
35+
ads: {}
36+
resource_api_version: V3
37+
stats_sinks:
38+
- name: "envoy.stat_sinks.open_telemetry"
39+
typed_config:
40+
"@type": type.googleapis.com/envoy.extensions.stat_sinks.open_telemetry.v3.SinkConfig
41+
grpc_service:
42+
envoy_grpc:
43+
cluster_name: otel_metric_sink_0
44+
report_counters_as_deltas: true
45+
report_histograms_as_deltas: true
46+
static_resources:
47+
clusters:
48+
- name: otel_metric_sink_0
49+
connect_timeout: 0.250s
50+
type: STRICT_DNS
51+
typed_extension_protocol_options:
52+
envoy.extensions.upstreams.http.v3.HttpProtocolOptions:
53+
"@type": "type.googleapis.com/envoy.extensions.upstreams.http.v3.HttpProtocolOptions"
54+
explicit_http_config:
55+
http2_protocol_options: {}
56+
lb_policy: ROUND_ROBIN
57+
load_assignment:
58+
cluster_name: otel_metric_sink_0
59+
endpoints:
60+
- lb_endpoints:
61+
- endpoint:
62+
address:
63+
socket_address:
64+
address: otel-collector.monitoring.svc
65+
port_value: 4317
66+
- connect_timeout: 10s
67+
eds_cluster_config:
68+
eds_config:
69+
ads: {}
70+
resource_api_version: 'V3'
71+
service_name: local_cluster
72+
load_balancing_policy:
73+
policies:
74+
- typed_extension_config:
75+
name: 'envoy.load_balancing_policies.least_request'
76+
typed_config:
77+
'@type': 'type.googleapis.com/envoy.extensions.load_balancing_policies.least_request.v3.LeastRequest'
78+
locality_lb_config:
79+
zone_aware_lb_config:
80+
min_cluster_size: '1'
81+
name: local_cluster
82+
type: EDS
83+
- connect_timeout: 10s
84+
load_assignment:
85+
cluster_name: xds_cluster
86+
endpoints:
87+
- load_balancing_weight: 1
88+
lb_endpoints:
89+
- load_balancing_weight: 1
90+
endpoint:
91+
address:
92+
socket_address:
93+
address: envoy-gateway
94+
port_value: 18000
95+
typed_extension_protocol_options:
96+
envoy.extensions.upstreams.http.v3.HttpProtocolOptions:
97+
"@type": "type.googleapis.com/envoy.extensions.upstreams.http.v3.HttpProtocolOptions"
98+
explicit_http_config:
99+
http2_protocol_options:
100+
connection_keepalive:
101+
interval: 30s
102+
timeout: 5s
103+
name: xds_cluster
104+
type: STRICT_DNS
105+
transport_socket:
106+
name: envoy.transport_sockets.tls
107+
typed_config:
108+
"@type": type.googleapis.com/envoy.extensions.transport_sockets.tls.v3.UpstreamTlsContext
109+
common_tls_context:
110+
tls_params:
111+
tls_maximum_protocol_version: TLSv1_3
112+
tls_certificate_sds_secret_configs:
113+
- name: xds_certificate
114+
sds_config:
115+
path_config_source:
116+
path: /sds/xds-certificate.json
117+
resource_api_version: V3
118+
validation_context_sds_secret_config:
119+
name: xds_trusted_ca
120+
sds_config:
121+
path_config_source:
122+
path: /sds/xds-trusted-ca.json
123+
resource_api_version: V3
124+
overload_manager:
125+
refresh_interval: 0.25s
126+
resource_monitors:
127+
- name: "envoy.resource_monitors.global_downstream_max_connections"
128+
typed_config:
129+
"@type": type.googleapis.com/envoy.extensions.resource_monitors.downstream_connections.v3.DownstreamConnectionsConfig
130+
max_active_downstream_connections: 50000

‎site/content/en/latest/api/extension_types.md‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -4051,6 +4051,8 @@ _Appears in:_
40514051
| `backendSettings` | _[ClusterSettings](#clustersettings)_ | false | | BackendSettings holds configuration for managing the connection<br />to the backend. |
40524052
| `host` | _string_ | false | | Host define the service hostname.<br />Deprecated: Use BackendRefs instead. |
40534053
| `port` | _integer_ | false | 4317 | Port defines the port the service is exposed on.<br />Deprecated: Use BackendRefs instead. |
4054+
| `reportCountersAsDeltas` | _boolean_ | false | | ReportCountersAsDeltas configures the OpenTelemetry sink to report<br />counters as delta temporality instead of cumulative. |
4055+
| `reportHistogramsAsDeltas` | _boolean_ | false | | ReportHistogramsAsDeltas configures the OpenTelemetry sink to report<br />histograms as delta temporality instead of cumulative.<br />Required for backends like Elastic that drop cumulative histograms. |
40544056

40554057

40564058
#### ProxyPrometheusProvider

‎test/helm/gateway-crds-helm/all.out.yaml‎

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -43342,6 +43342,17 @@ spec:
4334243342
maximum: 65535
4334343343
minimum: 0
4334443344
type: integer
43345+
reportCountersAsDeltas:
43346+
description: |-
43347+
ReportCountersAsDeltas configures the OpenTelemetry sink to report
43348+
counters as delta temporality instead of cumulative.
43349+
type: boolean
43350+
reportHistogramsAsDeltas:
43351+
description: |-
43352+
ReportHistogramsAsDeltas configures the OpenTelemetry sink to report
43353+
histograms as delta temporality instead of cumulative.
43354+
Required for backends like Elastic that drop cumulative histograms.
43355+
type: boolean
4334543356
type: object
4334643357
x-kubernetes-validations:
4334743358
- message: host or backendRefs needs to be set

0 commit comments

Comments
 (0)