Skip to content

Commit

Permalink
Updating OTEL
Browse files Browse the repository at this point in the history
  • Loading branch information
@ahus1
ahus1 committed Jul 11, 2023
1 parent 3717dc3 commit 318e608
Show file tree
Hide file tree
Showing 4 changed files with 103 additions and 142 deletions.
6 changes: 3 additions & 3 deletions doc/kubernetes/modules/ROOT/pages/util/otel.adoc
View file
Original file line number Diff line number Diff line change
Expand Up @@ -94,11 +94,11 @@ Use the following query to filter for metrics reported by OpenTelemetry:

There are some additional metrics recorded via OpenTelemetry which are not available from the regular Keycloak metrics endpoint:

`http_server_duration_bucket`:: For each URL, HTTP method and return code, it records buckets by duration.
Use this information identify latency percentiles for URLs, and find URLs which return error codes.
`http_server_duration_seconds_bucket`:: For each URL, HTTP method and return code, it records buckets by duration.
Use this information to identify latency percentiles for URLs, and find URLs which return error codes.
+
====
http_server_duration_bucket{container="keycloak", endpoint="otel-prometheus", http_flavor="1.1", http_method="GET", http_route="/admin/realms/++{realm}++/client-scopes", http_scheme="https", http_status_code="200", instance="172.17.0.8:9464", job="keycloak/keycloak-otel", le="10.0", namespace="keycloak", net_host_name="keycloak.192.168.39.120.nip.io", otel_scope_name="io.opentelemetry.netty-4.1", otel_scope_version="1.24.0-alpha", pod="keycloak-0"}
http_server_duration_seconds_bucket{otel_scope_name="io.opentelemetry.netty-4.1",otel_scope_version="1.27.0-alpha",http_request_method="GET",http_response_status_code="200",http_route="/health/live",network_protocol_name="http",network_protocol_version="1.1",server_address="10.130.4.106",server_port="8443",url_scheme="https",le="0.01"} 2.0
====

`worker_pool_queue_delay_bucket`:: Delay for executions in the worker pool, bucketed by the delay so tail latencies are available.
Expand Down
2 changes: 2 additions & 0 deletions provision/minikube/keycloak/templates/keycloak.yaml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -137,6 +137,8 @@ spec:
# https://github.com/open-telemetry/opentelemetry-java-instrumentation/issues/8927
- name: QUARKUS_HTTP_HTTP2
value: 'false'
- name: OTEL_SEMCONV_STABILITY_OPT_IN
value: 'http'
# https://github.com/open-telemetry/opentelemetry-java-instrumentation
# https://github.com/open-telemetry/opentelemetry-java/blob/main/sdk-extensions/autoconfigure/README.md
- name: OTEL_RESOURCE_ATTRIBUTES
Expand Down
119 changes: 49 additions & 70 deletions provision/minikube/monitoring/dashboards/authentication-code-otel.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -24,7 +24,6 @@
"editable": true,
"fiscalYearStartMonth": 0,
"graphTooltip": 0,
"id": 5,
"links": [],
"liveNow": false,
"panels": [
Expand Down Expand Up @@ -55,8 +54,6 @@
"mode": "palette-classic"
},
"custom": {
"axisCenteredZero": false,
"axisColorMode": "text",
"axisLabel": "",
"axisPlacement": "auto",
"barAlignment": 0,
Expand Down Expand Up @@ -135,7 +132,7 @@
},
"editorMode": "code",
"exemplar": false,
"expr": "sum(rate(http_server_duration_bucket{http_route=~\"(/realms/{realm}/protocol/{protocol}/token|/realms/{realm}/protocol/{protocol}/auth|/realms/{realm}/protocol/{protocol}/logout|realms/{realm}/login-actions/authenticate)\", le=\"250.0\", namespace=\"$namespace\", pod=\"$pod_name\"}[2m])) without (le) / rate(http_server_duration_count{http_route=~\"(/realms/{realm}/protocol/{protocol}/token|/realms/{realm}/protocol/{protocol}/auth|/realms/{realm}/protocol/{protocol}/logout|realms/{realm}/login-actions/authenticate)\", namespace=\"$namespace\", pod=\"$pod_name\"}[2m])",
"expr": "sum(rate(http_server_duration_seconds_bucket{http_route=~\"(/realms/{realm}/protocol/{protocol}/token|/realms/{realm}/protocol/{protocol}/auth|/realms/{realm}/protocol/{protocol}/logout|realms/{realm}/login-actions/authenticate)\", le=\"0.25\", namespace=\"$namespace\", pod=\"$pod_name\"}[2m])) without (le) / rate(http_server_duration_count{http_route=~\"(/realms/{realm}/protocol/{protocol}/token|/realms/{realm}/protocol/{protocol}/auth|/realms/{realm}/protocol/{protocol}/logout|realms/{realm}/login-actions/authenticate)\", namespace=\"$namespace\", pod=\"$pod_name\"}[2m])",
"format": "time_series",
"hide": false,
"instant": false,
Expand Down Expand Up @@ -172,32 +169,32 @@
"type": "timeseries"
},
{
"cards": {},
"color": {
"cardColor": "#b4ff00",
"colorScale": "sqrt",
"colorScheme": "interpolateOranges",
"exponent": 0.5,
"mode": "spectrum"
},
"dataFormat": "timeseries",
"datasource": {
"type": "prometheus",
"uid": "PBFA97CFB590B2093"
},
"fieldConfig": {
"defaults": {
"custom": {
"hideFrom": {
"legend": false,
"tooltip": false,
"viz": false
},
"scaleDistribution": {
"type": "linear"
}
}
},
"overrides": []
},
"gridPos": {
"h": 9,
"w": 6,
"x": 6,
"y": 1
},
"heatmap": {},
"hideZeroBuckets": false,
"highlightCards": true,
"id": 2,
"legend": {
"show": false
},
"options": {
"calculate": false,
"cellGap": 1,
Expand Down Expand Up @@ -232,6 +229,7 @@
}
},
"pluginVersion": "9.4.7",
"reverseYBuckets": false,
"targets": [
{
"datasource": {
Expand All @@ -240,7 +238,7 @@
},
"editorMode": "code",
"exemplar": true,
"expr": "sum by (le) (idelta(http_server_duration_bucket{http_route=~\"(/realms/{realm}/protocol/{protocol}/token|/realms/{realm}/protocol/{protocol}/auth|/realms/{realm}/protocol/{protocol}/logout|realms/{realm}/login-actions/authenticate)\", namespace=\"$namespace\", pod=\"$pod_name\"} [2m]))",
"expr": "sum by (le) (idelta(http_server_duration_seconds_bucket{http_route=~\"(/realms/{realm}/protocol/{protocol}/token|/realms/{realm}/protocol/{protocol}/auth|/realms/{realm}/protocol/{protocol}/logout|realms/{realm}/login-actions/authenticate)\", namespace=\"$namespace\", pod=\"$pod_name\"} [2m]))",
"format": "heatmap",
"interval": "",
"legendFormat": "{{le}}",
Expand All @@ -249,7 +247,20 @@
}
],
"title": "$pod_name - Response time histogram",
"type": "heatmap"
"tooltip": {
"show": true,
"showHistogram": false
},
"type": "heatmap",
"xAxis": {
"show": true
},
"yAxis": {
"format": "short",
"logBase": 1,
"show": true
},
"yBucketBound": "auto"
},
{
"datasource": {
Expand All @@ -262,8 +273,6 @@
"mode": "palette-classic"
},
"custom": {
"axisCenteredZero": false,
"axisColorMode": "text",
"axisLabel": "",
"axisPlacement": "auto",
"barAlignment": 0,
Expand Down Expand Up @@ -305,7 +314,7 @@
}
]
},
"unit": "ms"
"unit": "s"
},
"overrides": []
},
Expand Down Expand Up @@ -335,7 +344,7 @@
"uid": "PBFA97CFB590B2093"
},
"editorMode": "code",
"expr": "histogram_quantile(0.99, sum(rate(http_server_duration_bucket{http_route=~\"(/realms/{realm}/protocol/{protocol}/token|/realms/{realm}/protocol/{protocol}/auth|/realms/{realm}/protocol/{protocol}/logout|realms/{realm}/login-actions/authenticate)\", namespace=\"$namespace\", pod=\"$pod_name\"}[2m])) by (le,http_route))",
"expr": "histogram_quantile(0.99, sum(rate(http_server_duration_seconds_bucket{http_route=~\"(/realms/{realm}/protocol/{protocol}/token|/realms/{realm}/protocol/{protocol}/auth|/realms/{realm}/protocol/{protocol}/logout|realms/{realm}/login-actions/authenticate)\", namespace=\"$namespace\", pod=\"$pod_name\"}[2m])) by (le,http_route))",
"legendFormat": "__auto",
"range": true,
"refId": "A"
Expand All @@ -347,16 +356,16 @@
],
"refresh": false,
"revision": 1,
"schemaVersion": 38,
"schemaVersion": 36,
"style": "dark",
"tags": [],
"templating": {
"list": [
{
"current": {
"selected": false,
"text": "runner-keycloak",
"value": "runner-keycloak"
"selected": true,
"text": "keycloak",
"value": "keycloak"
},
"datasource": {
"type": "prometheus",
Expand Down Expand Up @@ -388,51 +397,21 @@
"$__all"
]
},
"definition": "http_server_duration_seconds_count{namespace=\"$namespace\"}",
"hide": 0,
"includeAll": true,
"multi": true,
"name": "pod_name",
"options": [
{
"selected": true,
"text": "All",
"value": "$__all"
},
{
"selected": false,
"text": "keycloak-0",
"value": "keycloak-0"
},
{
"selected": false,
"text": "keycloak-1",
"value": "keycloak-1"
},
{
"selected": false,
"text": "keycloak-2",
"value": "keycloak-2"
},
{
"selected": false,
"text": "keycloak-3",
"value": "keycloak-3"
},
{
"selected": false,
"text": "keycloak-4",
"value": "keycloak-4"
},
{
"selected": false,
"text": "keycloak-5",
"value": "keycloak-5"
}
],
"query": "keycloak-0,keycloak-1,keycloak-2,keycloak-3,keycloak-4,keycloak-5",
"queryValue": "",
"options": [],
"query": {
"query": "http_server_duration_seconds_count{namespace=\"$namespace\"}",
"refId": "StandardVariableQuery"
},
"refresh": 2,
"regex": "/pod=\"([^\"]*)\"/",
"skipUrlSync": false,
"type": "custom"
"sort": 1,
"type": "query"
}
]
},
Expand All @@ -444,6 +423,6 @@
"timezone": "",
"title": "Authentication Code SLO",
"uid": "R3kK_894z",
"version": 1,
"version": 2,
"weekStart": ""
}
Loading

0 comments on commit 318e608

Please sign in to comment.