From 4447657698c31ee97081d873fda7f12023819ba6 Mon Sep 17 00:00:00 2001 From: Yahima Duarte Date: Tue, 17 Dec 2024 18:51:51 +0100 Subject: [PATCH 1/3] add dashboard for azure front door (#1375) * add dashboard for azure front door * fix threhold for percentage * fix lint --- csp-mixin/.lint | 3 + csp-mixin/azureconfig.libsonnet | 3 +- csp-mixin/config.libsonnet | 2 + csp-mixin/dashboards.libsonnet | 31 +- csp-mixin/panels/azurefrontdoor.libsonnet | 300 ++++++++++++++++++ csp-mixin/panels/main.libsonnet | 1 + csp-mixin/rows.libsonnet | 81 +++++ csp-mixin/signals/azurefrontdoor.libsonnet | 110 +++++++ .../signals/azurefrontdoorOverview.libsonnet | 186 +++++++++++ 9 files changed, 713 insertions(+), 4 deletions(-) create mode 100644 csp-mixin/panels/azurefrontdoor.libsonnet create mode 100644 csp-mixin/signals/azurefrontdoor.libsonnet create mode 100644 csp-mixin/signals/azurefrontdoorOverview.libsonnet diff --git a/csp-mixin/.lint b/csp-mixin/.lint index 3652e0e64..b0a8d022d 100644 --- a/csp-mixin/.lint +++ b/csp-mixin/.lint @@ -23,6 +23,7 @@ exclusions: - dashboard: "GCP Compute Engine" reason: "Using instance_name as instance label" - dashboard: "Azure Virtual Machines" + - dashboard: "Azure Front Door" reason: "Aggregation is at the group level, and resourceName is used for instance label" target-job-rule: reason: "Using filtering selector with job" @@ -47,6 +48,7 @@ exclusions: - dashboard: "GCP Compute Engine" reason: "Using instance_name as instance label" - dashboard: "Azure Virtual Machines" + - dashboard: "Azure Front Door" reason: "Aggregation is at the group level, and resourceName is used for instance label" panel-datasource-rule: reason: "Many panels use --Mixed-- DS" @@ -73,6 +75,7 @@ exclusions: - dashboard: Azure Service Bus - dashboard: GCP Compute Engine - dashboard: Azure Virtual Machines + - dashboard: Azure Front Door target-rate-interval-rule: entries: - dashboard: GCP Compute Engine diff --git a/csp-mixin/azureconfig.libsonnet b/csp-mixin/azureconfig.libsonnet index 510fbb11b..fa397c722 100644 --- a/csp-mixin/azureconfig.libsonnet +++ b/csp-mixin/azureconfig.libsonnet @@ -7,10 +7,11 @@ enableAvailability: true, bucketLabel: 'resourceName', }, - azurevm+: { + commomVars+: { groupLabel: 'resourceGroup', subscriptionLabel: 'subscriptionName', instanceLabel: 'resourceName', + dimensionEndpoint: 'dimensionEndpoint', }, // UID Prefix for each dashboard uid: 'azure', diff --git a/csp-mixin/config.libsonnet b/csp-mixin/config.libsonnet index ac462d5cb..0e250bd4d 100644 --- a/csp-mixin/config.libsonnet +++ b/csp-mixin/config.libsonnet @@ -19,6 +19,8 @@ gcpce: (import './signals/gcpce.libsonnet')(this), gcpceOverview: (import './signals/gcpceOverview.libsonnet')(this), gcpvpc: (import './signals/gcpvpc.libsonnet')(this), + azurefrontdoorOverview: (import './signals/azurefrontdoorOverview.libsonnet')(this), + azurefrontdoor: (import './signals/azurefrontdoor.libsonnet')(this), }, blobStorage: { enableAvailability: false, diff --git a/csp-mixin/dashboards.libsonnet b/csp-mixin/dashboards.libsonnet index f63b608c5..8c0f36e19 100644 --- a/csp-mixin/dashboards.libsonnet +++ b/csp-mixin/dashboards.libsonnet @@ -168,11 +168,11 @@ local commonlib = import 'common-lib/common/main.libsonnet'; + g.dashboard.withRefresh(csplib.config.dashboardRefresh) + g.dashboard.timepicker.withTimeOptions(csplib.config.dashboardPeriod) + g.dashboard.withVariables([ - if std.asciiLower(v.label) == std.asciiLower(csplib.config.azurevm.groupLabel) + if std.asciiLower(v.label) == std.asciiLower(csplib.config.commomVars.groupLabel) then v { label: 'Group' } - else if std.asciiLower(v.label) == std.asciiLower(csplib.config.azurevm.subscriptionLabel) + else if std.asciiLower(v.label) == std.asciiLower(csplib.config.commomVars.subscriptionLabel) then v { label: 'Subscription' } - else if std.asciiLower(v.label) == std.asciiLower(csplib.config.azurevm.instanceLabel) + else if std.asciiLower(v.label) == std.asciiLower(csplib.config.commomVars.instanceLabel) then v { label: 'Instance' } else v for v in variables @@ -184,6 +184,31 @@ local commonlib = import 'common-lib/common/main.libsonnet'; ) ), + [csplib.config.uid + '-frontdoor.json']: + local variables = csplib.signals.azurefrontdoor.getVariablesMultiChoice(); + g.dashboard.new(csplib.config.dashboardNamePrefix + 'Front Door') + + g.dashboard.withUid(csplib.config.uid + '-frontdoor') + + g.dashboard.withTags(csplib.config.dashboardTags) + + g.dashboard.withTimezone(csplib.config.dashboardTimezone) + + g.dashboard.withRefresh(csplib.config.dashboardRefresh) + + g.dashboard.timepicker.withTimeOptions(csplib.config.dashboardPeriod) + + g.dashboard.withVariables([ + if std.asciiLower(v.label) == std.asciiLower(csplib.config.commomVars.groupLabel) + then v { label: 'Group' } + else if std.asciiLower(v.label) == std.asciiLower(csplib.config.commomVars.subscriptionLabel) + then v { label: 'Subscription' } + else if std.asciiLower(v.label) == std.asciiLower(csplib.config.commomVars.dimensionEndpoint) + then v { label: 'Endpoint' } + else v + for v in variables + ]) + + g.dashboard.withPanels( + g.util.grid.wrapPanels( + csplib.grafana.rows.afd_overview + + csplib.grafana.rows.afd_endpoints, + ) + ), + [csplib.config.uid + '-queuestorage.json']: local variables = csplib.signals.azurequeuestore.getVariablesMultiChoice(); g.dashboard.new(csplib.config.dashboardNamePrefix + 'Queue storage') diff --git a/csp-mixin/panels/azurefrontdoor.libsonnet b/csp-mixin/panels/azurefrontdoor.libsonnet new file mode 100644 index 000000000..7e5b2e436 --- /dev/null +++ b/csp-mixin/panels/azurefrontdoor.libsonnet @@ -0,0 +1,300 @@ +local g = import '../g.libsonnet'; +local commonlib = import 'common-lib/common/main.libsonnet'; +{ + new(this): { + // Azure Front Door + + afd_endpoint_count: + this.signals.azurefrontdoorOverview.endpointCount.asStat() + + commonlib.panels.generic.stat.base.stylize(), + + afd_top5_errors: + this.signals.azurefrontdoorOverview.top5Errors.asTable(format='table') + + commonlib.panels.generic.table.base.stylize() + + g.panel.table.standardOptions.withOverrides( + [ + { + matcher: { + id: 'byName', + options: 'Value', + }, + properties: [ + { + id: 'custom.width', + value: 80, + }, + { + id: 'color', + }, + { + id: 'custom.cellOptions', + value: { + type: 'gauge', + valueDisplayMode: 'color', + }, + }, + { + id: 'unit', + value: 'percent', + }, + { + id: 'thresholds', + value: { + mode: 'percentage', + steps: [ + { + color: 'text', + value: 0, + }, + { + color: 'red', + value: 1, + }, + ], + }, + }, + { + id: 'min', + value: 0, + }, + { + id: 'max', + value: 100, + }, + ], + }, + ] + ) + + g.panel.table.queryOptions.withTransformations([ + { + id: 'organize', + options: { + excludeByName: { + Time: true, + subscriptionName: true, + }, + includeByName: {}, + indexByName: { + Time: 0, + Value: 7, + dimensionClientCountry: 6, + dimensionClientRegion: 5, + job: 1, + resourceGroup: 2, + resourceName: 3, + subscriptionName: 4, + }, + renameByName: { + dimensionClientCountry: 'Country', + dimensionClientRegion: 'Region', + job: 'Job', + resourceGroup: 'Group', + resourceName: 'Resource', + }, + }, + }, + ]), + + afd_total_requests: + this.signals.azurefrontdoorOverview.totalRequests.asTimeSeries() + + commonlib.panels.generic.timeSeries.base.stylize() + + g.panel.timeSeries.options.legend.withShowLegend(false), + + afd_requests_by_country: + this.signals.azurefrontdoorOverview.requestsByCountry.asTimeSeries() + + commonlib.panels.generic.timeSeries.base.stylize(), + + afd_requests_by_status: + this.signals.azurefrontdoorOverview.requestsByStatus.asTimeSeries() + + commonlib.panels.generic.timeSeries.base.stylize(), + + afd_requests_by_errors: + commonlib.panels.generic.timeSeries.base.new('Request by Errors percentage', targets=[]) + + this.signals.azurefrontdoorOverview.error4xx.asPanelMixin() + + this.signals.azurefrontdoorOverview.error5xx.asPanelMixin() + + commonlib.panels.generic.timeSeries.base.stylize() + + g.panel.table.standardOptions.withOverrides( + [ + { + matcher: { + id: 'byName', + options: '4xx', + }, + properties: [ + { + id: 'color', + value: { + fixedColor: 'orange', + mode: 'fixed', + }, + }, + { + id: 'unit', + value: 'percent', + }, + ], + }, + { + matcher: { + id: 'byName', + options: '5xx', + }, + properties: [ + { + id: 'color', + value: { + fixedColor: 'red', + mode: 'fixed', + }, + }, + { + id: 'unit', + value: 'percent', + }, + ], + }, + ] + ), + + afd_requests_responses_size: + commonlib.panels.generic.timeSeries.base.new('Request/Responses size', targets=[]) + + commonlib.panels.generic.timeSeries.base.stylize() + + g.panel.timeSeries.panelOptions.withDescription('The number of bytes sent as requests/responses from clients to AFDX and from HTTP/S proxy to clients.') + + this.signals.azurefrontdoorOverview.requestsSize.asPanelMixin() + + this.signals.azurefrontdoorOverview.responsesSize.asPanelMixin() + + g.panel.timeSeries.standardOptions.withUnit('decbytes') + + g.panel.timeSeries.fieldConfig.defaults.custom.withAxisLabel('requests(-) | responses(+)') + + g.panel.table.standardOptions.withOverrides( + [ + { + matcher: { + id: 'byName', + options: 'Requests', + }, + properties: [ + { + id: 'custom.transform', + value: 'negative-Y', + }, + { + id: 'unit', + value: 'decbytes', + }, + ], + }, + ] + ), + + afd_total_latency: + this.signals.azurefrontdoorOverview.totalLatency.asTimeSeries() + + commonlib.panels.generic.timeSeries.base.stylize() + + g.panel.timeSeries.options.legend.withShowLegend(false), + + afd_origin_health: + this.signals.azurefrontdoorOverview.originHealthPercentage.asTimeSeries() + + commonlib.panels.generic.timeSeries.base.stylize() + + g.panel.timeSeries.options.legend.withShowLegend(false) + + g.panel.table.standardOptions.withOverrides( + [ + { + matcher: { + id: 'byName', + options: 'Value', + }, + properties: [ + { + id: 'unit', + value: 'percent', + }, + { + id: 'thresholds', + value: { + mode: 'percentage', + steps: [ + { + color: 'text', + value: 0, + }, + { + value: 1, + color: 'red', + }, + { + value: 85, + color: 'yellow', + }, + { + color: 'green', + value: 90, + }, + ], + }, + }, + { + id: 'color', + value: { + mode: 'thresholds', + }, + }, + ], + }, + ] + ), + + afd_origin_latency: + this.signals.azurefrontdoorOverview.originLatency.asTimeSeries() + + commonlib.panels.generic.timeSeries.base.stylize() + + g.panel.timeSeries.options.legend.withShowLegend(false), + + afd_requests_by_endpoint: + this.signals.azurefrontdoor.requestsByEndpoint.asTimeSeries() + + commonlib.panels.generic.timeSeries.base.stylize(), + + afd_requests_size_by_endpoint: + this.signals.azurefrontdoor.requestsSizeByEndpoints.asTimeSeries() + + commonlib.panels.generic.timeSeries.base.stylize(), + + afd_responses_size_by_endpoint: + this.signals.azurefrontdoor.responsesSizeByEndpoints.asTimeSeries() + + commonlib.panels.generic.timeSeries.base.stylize(), + + afd_total_latency_by_endpoint: + this.signals.azurefrontdoor.totalLatencyByEndpoints.asTimeSeries() + + commonlib.panels.generic.timeSeries.base.stylize(), + + afd_errors_by_endpoint: + this.signals.azurefrontdoor.errorsByEndpoints.asTimeSeries() + + commonlib.panels.generic.timeSeries.base.stylize() + + g.panel.table.standardOptions.withOverrides( + [ + { + matcher: { + id: 'byName', + options: 'Value', + }, + properties: [ + { + id: 'color', + value: { + mode: 'fixed', + fixedColor: 'red', + }, + }, + { + id: 'unit', + value: 'percent', + }, + ], + }, + ] + ), + + afd_origin_requests_by_endpoint: + this.signals.azurefrontdoor.originRequestsByEndpoints.asTimeSeries() + + commonlib.panels.generic.timeSeries.base.stylize(), + + afd_origin_latency_by_endpoint: + this.signals.azurefrontdoor.originLatencyByEndpoints.asTimeSeries() + + commonlib.panels.generic.timeSeries.base.stylize(), + }, +} diff --git a/csp-mixin/panels/main.libsonnet b/csp-mixin/panels/main.libsonnet index af8a92dcd..5f530977a 100644 --- a/csp-mixin/panels/main.libsonnet +++ b/csp-mixin/panels/main.libsonnet @@ -10,5 +10,6 @@ gcploadbalancer: (import './gcploadbalancer.libsonnet').new(this), gcpce: (import './gcpce.libsonnet').new(this), gcpvpc: (import './gcpvpc.libsonnet').new(this), + azurefrontdoor: (import './azurefrontdoor.libsonnet').new(this), }, } diff --git a/csp-mixin/rows.libsonnet b/csp-mixin/rows.libsonnet index ce2b71925..79c07a5b2 100644 --- a/csp-mixin/rows.libsonnet +++ b/csp-mixin/rows.libsonnet @@ -566,5 +566,86 @@ local g = import './g.libsonnet'; + g.panel.timeSeries.gridPos.withW(12) + g.panel.timeSeries.gridPos.withH(8), ], + + // Azure Front Door + afd_overview: [ + g.panel.row.new('Overview'), + + this.grafana.panels.azurefrontdoor.afd_endpoint_count + + g.panel.timeSeries.gridPos.withW(24) + + g.panel.timeSeries.gridPos.withH(5), + + this.grafana.panels.azurefrontdoor.afd_top5_errors + + g.panel.timeSeries.gridPos.withW(24) + + g.panel.timeSeries.gridPos.withH(8), + + this.grafana.panels.azurefrontdoor.afd_total_requests + + g.panel.timeSeries.gridPos.withW(12) + + g.panel.timeSeries.gridPos.withH(8), + + this.grafana.panels.azurefrontdoor.afd_requests_by_country + + g.panel.timeSeries.gridPos.withW(12) + + g.panel.timeSeries.gridPos.withH(8), + + this.grafana.panels.azurefrontdoor.afd_requests_by_status + + g.panel.timeSeries.gridPos.withW(12) + + g.panel.timeSeries.gridPos.withH(8), + + this.grafana.panels.azurefrontdoor.afd_requests_by_errors + + g.panel.timeSeries.gridPos.withW(12) + + g.panel.timeSeries.gridPos.withH(8), + + this.grafana.panels.azurefrontdoor.afd_requests_responses_size + + g.panel.timeSeries.gridPos.withW(12) + + g.panel.timeSeries.gridPos.withH(8), + + this.grafana.panels.azurefrontdoor.afd_total_latency + + g.panel.timeSeries.gridPos.withW(12) + + g.panel.timeSeries.gridPos.withH(8), + + this.grafana.panels.azurefrontdoor.afd_origin_health + + g.panel.timeSeries.gridPos.withW(12) + + g.panel.timeSeries.gridPos.withH(8), + + this.grafana.panels.azurefrontdoor.afd_origin_latency + + g.panel.timeSeries.gridPos.withW(12) + + g.panel.timeSeries.gridPos.withH(8), + ], + + afd_endpoints: [ + g.panel.row.new('Endpoints') + + g.panel.row.withCollapsed(true) + + g.panel.row.withPanels( + [ + this.grafana.panels.azurefrontdoor.afd_requests_by_endpoint + + g.panel.timeSeries.gridPos.withW(24) + + g.panel.timeSeries.gridPos.withH(8), + + this.grafana.panels.azurefrontdoor.afd_requests_size_by_endpoint + + g.panel.timeSeries.gridPos.withW(12) + + g.panel.timeSeries.gridPos.withH(8), + + this.grafana.panels.azurefrontdoor.afd_responses_size_by_endpoint + + g.panel.timeSeries.gridPos.withW(12) + + g.panel.timeSeries.gridPos.withH(8), + + this.grafana.panels.azurefrontdoor.afd_total_latency_by_endpoint + + g.panel.timeSeries.gridPos.withW(12) + + g.panel.timeSeries.gridPos.withH(8), + + this.grafana.panels.azurefrontdoor.afd_errors_by_endpoint + + g.panel.timeSeries.gridPos.withW(12) + + g.panel.timeSeries.gridPos.withH(8), + + this.grafana.panels.azurefrontdoor.afd_origin_requests_by_endpoint + + g.panel.timeSeries.gridPos.withW(12) + + g.panel.timeSeries.gridPos.withH(8), + + this.grafana.panels.azurefrontdoor.afd_origin_latency_by_endpoint + + g.panel.timeSeries.gridPos.withW(12) + + g.panel.timeSeries.gridPos.withH(8), + ] + ), + ], }, } diff --git a/csp-mixin/signals/azurefrontdoor.libsonnet b/csp-mixin/signals/azurefrontdoor.libsonnet new file mode 100644 index 000000000..d68ab4b5c --- /dev/null +++ b/csp-mixin/signals/azurefrontdoor.libsonnet @@ -0,0 +1,110 @@ +local commonlib = import 'common-lib/common/main.libsonnet'; +function(this) + { + local s = self, + filteringSelector: this.filteringSelector, + groupLabels: this.groupLabels, + instanceLabels: ['dimensionEndpoint'], + aggLevel: 'instance', + discoveryMetric: { + azuremonitor: 'azure_microsoft_cdn_profiles_requestcount_total_count', + }, + signals: { + requestsByEndpoint: { + name: 'Total requests', + description: 'Number of requests by endpoints.', + type: 'gauge', + unit: 'short', + aggFunction: 'sum', + sources: { + azuremonitor: { + expr: 'azure_microsoft_cdn_profiles_requestcount_total_count{%(queriesSelector)s}', + legendCustomTemplate: '{{dimensionEndpoint}}', + }, + }, + }, + + requestsSizeByEndpoints: { + name: 'Requests size', + description: 'The number of bytes sent as requests from clients to AFDX.', + type: 'gauge', + unit: 'decbytes', + aggFunction: 'sum', + sources: { + azuremonitor: { + expr: 'azure_microsoft_cdn_profiles_requestsize_total_bytes{%(queriesSelector)s}', + legendCustomTemplate: '{{dimensionEndpoint}}', + }, + }, + }, + + responsesSizeByEndpoints: { + name: 'Responses size', + description: 'The number of bytes sent as responses from HTTP/S proxy to clients.', + type: 'gauge', + unit: 'decbytes', + aggFunction: 'sum', + sources: { + azuremonitor: { + expr: 'azure_microsoft_cdn_profiles_responsesize_total_bytes{%(queriesSelector)s}', + legendCustomTemplate: '{{dimensionEndpoint}}', + }, + }, + }, + + totalLatencyByEndpoints: { + name: 'Total latency', + description: 'The time calculated from when the client request was received by the HTTP/S proxy until the client acknowledged the last response byte from the HTTP/S proxy\tTotalLatency', + type: 'gauge', + unit: 'ms', + aggFunction: 'avg', + sources: { + azuremonitor: { + expr: 'azure_microsoft_cdn_profiles_totallatency_average_milliseconds{%(queriesSelector)s}', + legendCustomTemplate: '{{dimensionEndpoint}}', + }, + }, + }, + + errorsByEndpoints: { + name: 'Percentage of errors', + description: '', + type: 'raw', + unit: 'percent', + sources: { + azuremonitor: { + expr: 'avg by (job, resourceGroup, subscriptionName, dimensionEndpoint) (azure_microsoft_cdn_profiles_percentage4xx_average_percent{%(queriesSelector)s}) + avg by (job, resourceGroup, subscriptionName, dimensionEndpoint) (azure_microsoft_cdn_profiles_percentage5xx_average_percent{%(queriesSelector)s})', + legendCustomTemplate: '{{dimensionEndpoint}}', + }, + }, + }, + + originRequestsByEndpoints: { + name: 'Origin Request count', + description: 'The number of requests sent from AFDX to origin.', + type: 'gauge', + unit: 'short', + aggFunction: 'sum', + sources: { + azuremonitor: { + expr: 'azure_microsoft_cdn_profiles_originrequestcount_total_count{%(queriesSelector)s}', + legendCustomTemplate: '{{dimensionEndpoint}}', + }, + }, + }, + + originLatencyByEndpoints: { + name: 'Origin Latency average', + description: 'The time calculated from when the request was sent by AFDX edge to the backend until AFDX received the last response byte from the backend.', + type: 'gauge', + unit: 'ms', + aggFunction: 'avg', + sources: { + azuremonitor: { + expr: 'azure_microsoft_cdn_profiles_originlatency_average_milliseconds{%(queriesSelector)s}', + legendCustomTemplate: '{{dimensionEndpoint}}', + }, + }, + }, + }, + } diff --git a/csp-mixin/signals/azurefrontdoorOverview.libsonnet b/csp-mixin/signals/azurefrontdoorOverview.libsonnet new file mode 100644 index 000000000..f5ebe9fbd --- /dev/null +++ b/csp-mixin/signals/azurefrontdoorOverview.libsonnet @@ -0,0 +1,186 @@ +local commonlib = import 'common-lib/common/main.libsonnet'; +function(this) + { + local s = self, + filteringSelector: this.filteringSelector, + groupLabels: this.groupLabels, + instanceLabels: [], + aggLevel: 'instance', + discoveryMetric: { + azuremonitor: 'azure_microsoft_cdn_profiles_requestcount_total_count', + }, + signals: { + endpointCount: { + name: 'Endpoints Count', + description: 'Number of endpoints', + type: 'gauge', + aggFunction: 'sum', + unit: 'short', + sources: { + azuremonitor: { + expr: 'azure_microsoft_cdn_profiles_requestcount_total_count{%(queriesSelector)s}', + legendCustomTemplate: '', + aggKeepLabels: ['dimensionEndpoint'], + exprWrappers: [['count(', ')']], + }, + }, + }, + + top5Errors: { + name: 'Top 5 Endpoints by Errors', + description: 'Top 5 Endpoints with higher percentage of client requests for which the response status code is 4XX or 5XX', + type: 'raw', + unit: 'percent', + sources: { + azuremonitor: { + expr: 'avg by (job, resourceGroup, subscriptionName, resourceName, dimensionClientCountry, dimensionClientRegion) (azure_microsoft_cdn_profiles_percentage4xx_average_percent{%(queriesSelector)s}) + avg by (job, resourceGroup, subscriptionName, resourceName, dimensionClientCountry, dimensionClientRegion) (azure_microsoft_cdn_profiles_percentage5xx_average_percent{%(queriesSelector)s})', + legendCustomTemplate: '', + exprWrappers: [['topk(5,', ')']], + }, + }, + }, + + totalRequests: { + name: 'Total Requests', + description: 'The number of client requests served by the HTTP/S proxy', + type: 'raw', + unit: 'short', + sources: { + azuremonitor: { + expr: 'azure_microsoft_cdn_profiles_requestcount_total_count{%(queriesSelector)s}', + legendCustomTemplate: 'Total Requests', + aggKeepLabels: ['dimensionEndpoint'], + exprWrappers: [['sum(', ')']], + }, + }, + }, + + requestsByCountry: { + name: 'Requests by country', + description: 'The number of client requests served by the HTTP/S proxy grouped by country', + type: 'gauge', + aggFunction: 'sum', + unit: 'short', + sources: { + azuremonitor: { + expr: 'azure_microsoft_cdn_profiles_requestcount_total_count{%(queriesSelector)s}', + legendCustomTemplate: '{{dimensionClientCountry}}', + aggKeepLabels: ['dimensionClientCountry'], + }, + }, + }, + + requestsByStatus: { + name: 'Requests by status', + description: 'The number of client requests served by the HTTP/S proxy grouped by status group', + type: 'gauge', + aggFunction: 'sum', + unit: 'short', + sources: { + azuremonitor: { + expr: 'azure_microsoft_cdn_profiles_requestcount_total_count{%(queriesSelector)s}', + legendCustomTemplate: '{{dimensionHttpStatusGroup}}', + aggKeepLabels: ['dimensionHttpStatusGroup'], + }, + }, + }, + + error4xx: { + name: '4XX Errors percentage', + description: '', + type: 'raw', + unit: 'percent', + sources: { + azuremonitor: { + expr: 'azure_microsoft_cdn_profiles_percentage4xx_average_percent{%(queriesSelector)s}', + legendCustomTemplate: '4xx', + exprWrappers: [['avg(', ')']], + }, + }, + }, + + error5xx: { + name: '5XX Errors percentage', + description: '', + type: 'raw', + unit: 'percent', + sources: { + azuremonitor: { + expr: 'azure_microsoft_cdn_profiles_percentage5xx_average_percent{%(queriesSelector)s}', + legendCustomTemplate: '5xx', + exprWrappers: [['avg(', ')']], + }, + }, + }, + + requestsSize: { + name: 'Requests size', + description: 'The number of bytes sent as requests from clients to AFDX.', + type: 'raw', + unit: 'decbytes', + sources: { + azuremonitor: { + expr: 'azure_microsoft_cdn_profiles_requestsize_total_bytes{%(queriesSelector)s}', + legendCustomTemplate: 'Requests', + exprWrappers: [['sum(', ')']], + }, + }, + }, + + responsesSize: { + name: 'Responses size', + description: 'The number of bytes sent as responses from HTTP/S proxy to clients.', + type: 'raw', + unit: 'decbytes', + sources: { + azuremonitor: { + expr: 'azure_microsoft_cdn_profiles_responsesize_total_bytes{%(queriesSelector)s}', + legendCustomTemplate: 'Responses', + exprWrappers: [['sum(', ')']], + }, + }, + }, + + totalLatency: { + name: 'Total latency', + description: 'The time calculated from when the client request was received by the HTTP/S proxy until the client acknowledged the last response byte from the HTTP/S proxy\tTotalLatency', + type: 'raw', + unit: 'ms', + sources: { + azuremonitor: { + expr: 'azure_microsoft_cdn_profiles_totallatency_average_milliseconds{%(queriesSelector)s}', + legendCustomTemplate: '', + exprWrappers: [['avg(', ')']], + }, + }, + }, + + originHealthPercentage: { + name: 'Origin Health percentage', + description: 'The percentage of successful health probes from AFDX to backends.', + type: 'raw', + unit: 'percent', + sources: { + azuremonitor: { + expr: 'azure_microsoft_cdn_profiles_originhealthpercentage_average_percent{%(queriesSelector)s}', + legendCustomTemplate: '', + exprWrappers: [['avg(', ')']], + }, + }, + }, + + originLatency: { + name: 'Origin Latency average', + description: 'The time calculated from when the request was sent by AFDX edge to the backend until AFDX received the last response byte from the backend.', + type: 'raw', + unit: 'ms', + sources: { + azuremonitor: { + expr: 'azure_microsoft_cdn_profiles_originlatency_average_milliseconds{%(queriesSelector)s}', + legendCustomTemplate: '', + exprWrappers: [['avg(', ')']], + }, + }, + }, + }, + } From 1821eeb8c3524bad16b0427ba63c3c3f84c7897b Mon Sep 17 00:00:00 2001 From: v-zhuravlev Date: Wed, 18 Dec 2024 19:31:31 +0800 Subject: [PATCH 2/3] Add more compact legnds in opensearch (#1377) --- .../opensearch-cluster-overview.libsonnet | 21 ++++---- .../opensearch-node-overview.libsonnet | 48 ++++++++++--------- 2 files changed, 37 insertions(+), 32 deletions(-) diff --git a/opensearch-mixin/dashboards/opensearch-cluster-overview.libsonnet b/opensearch-mixin/dashboards/opensearch-cluster-overview.libsonnet index 4bcd7659d..c8d18c418 100644 --- a/opensearch-mixin/dashboards/opensearch-cluster-overview.libsonnet +++ b/opensearch-mixin/dashboards/opensearch-cluster-overview.libsonnet @@ -2,6 +2,7 @@ local g = import '../g.libsonnet'; local grafana = import 'grafonnet/grafana.libsonnet'; local prometheus = grafana.prometheus; local commonlib = import 'common-lib/common/main.libsonnet'; +local xtd = import 'github.com/jsonnet-libs/xtd/main.libsonnet'; local utils = commonlib.utils; local dashboardUidSuffix = '-cluster-overview'; @@ -15,6 +16,8 @@ local dashboardUidSuffix = '-cluster-overview'; varMetric='opensearch_cluster_status', ), + local legendGroupLabels = xtd.array.slice($._config.groupLabels, -1), + local panels = (import '../panels.libsonnet').new( $._config.groupLabels, $._config.instanceLabels, @@ -35,7 +38,7 @@ local dashboardUidSuffix = '-cluster-overview'; agg: std.join(',', $._config.groupLabels), }, datasource=promDatasource, - legendFormat=utils.labelsToPanelLegend($._config.groupLabels) + legendFormat=utils.labelsToPanelLegend(legendGroupLabels) ), ], type: 'stat', @@ -116,7 +119,7 @@ local dashboardUidSuffix = '-cluster-overview'; agg: std.join(',', $._config.groupLabels), }, datasource=promDatasource, - legendFormat=utils.labelsToPanelLegend($._config.groupLabels) + legendFormat=utils.labelsToPanelLegend(legendGroupLabels) ), ], type: 'stat', @@ -175,7 +178,7 @@ local dashboardUidSuffix = '-cluster-overview'; agg: std.join(',', $._config.groupLabels), }, datasource=promDatasource, - legendFormat=utils.labelsToPanelLegend($._config.groupLabels) + legendFormat=utils.labelsToPanelLegend(legendGroupLabels) ), ], type: 'stat', @@ -234,7 +237,7 @@ local dashboardUidSuffix = '-cluster-overview'; agg: std.join(',', $._config.groupLabels), }, datasource=promDatasource, - legendFormat=utils.labelsToPanelLegend($._config.groupLabels) + legendFormat=utils.labelsToPanelLegend(legendGroupLabels) ), ], type: 'stat', @@ -293,7 +296,7 @@ local dashboardUidSuffix = '-cluster-overview'; agg: std.join(',', $._config.groupLabels), }, datasource=promDatasource, - legendFormat=utils.labelsToPanelLegend($._config.groupLabels) + legendFormat=utils.labelsToPanelLegend(legendGroupLabels) ), ], @@ -587,7 +590,7 @@ local dashboardUidSuffix = '-cluster-overview'; agg: std.join(',', $._config.groupLabels), }, datasource=promDatasource, - legendFormat=utils.labelsToPanelLegend($._config.groupLabels), + legendFormat=utils.labelsToPanelLegend(legendGroupLabels), ), ], type: 'timeseries', @@ -666,7 +669,7 @@ local dashboardUidSuffix = '-cluster-overview'; agg: std.join(',', $._config.groupLabels), }, datasource=promDatasource, - legendFormat=utils.labelsToPanelLegend($._config.groupLabels), + legendFormat=utils.labelsToPanelLegend(legendGroupLabels), ), ], type: 'timeseries', @@ -745,7 +748,7 @@ local dashboardUidSuffix = '-cluster-overview'; agg: std.join(',', $._config.groupLabels), }, datasource=promDatasource, - legendFormat=utils.labelsToPanelLegend($._config.groupLabels), + legendFormat=utils.labelsToPanelLegend(legendGroupLabels), ), ], type: 'timeseries', @@ -824,7 +827,7 @@ local dashboardUidSuffix = '-cluster-overview'; agg: std.join(',', $._config.groupLabels), }, datasource=promDatasource, - legendFormat=utils.labelsToPanelLegend($._config.groupLabels), + legendFormat=utils.labelsToPanelLegend(legendGroupLabels), ), ], type: 'timeseries', diff --git a/opensearch-mixin/dashboards/opensearch-node-overview.libsonnet b/opensearch-mixin/dashboards/opensearch-node-overview.libsonnet index b823dd941..32b18e5ff 100644 --- a/opensearch-mixin/dashboards/opensearch-node-overview.libsonnet +++ b/opensearch-mixin/dashboards/opensearch-node-overview.libsonnet @@ -3,7 +3,7 @@ local grafana = (import 'grafonnet/grafana.libsonnet'); local commonlib = import 'common-lib/common/main.libsonnet'; local utils = commonlib.utils; local prometheus = grafana.prometheus; - +local xtd = import 'github.com/jsonnet-libs/xtd/main.libsonnet'; local dashboardUidSuffix = '-node-overview'; { @@ -17,6 +17,8 @@ local dashboardUidSuffix = '-node-overview'; enableLokiLogs=$._config.enableLokiLogs, ), + local legendInstanceLabels = xtd.array.slice($._config.instanceLabels, -1), + local panels = (import '../panels.libsonnet').new( $._config.groupLabels, $._config.instanceLabels, @@ -51,7 +53,7 @@ local dashboardUidSuffix = '-node-overview'; agg: std.join(',', $._config.groupLabels + $._config.instanceLabels), }, ) - + g.query.prometheus.withLegendFormat(utils.labelsToPanelLegend($._config.instanceLabels)), + + g.query.prometheus.withLegendFormat(utils.labelsToPanelLegend(legendInstanceLabels)), ], description="CPU usage percentage of the node's Operating System.", @@ -70,7 +72,7 @@ local dashboardUidSuffix = '-node-overview'; agg: std.join(',', $._config.groupLabels + $._config.instanceLabels), }, ) - + g.query.prometheus.withLegendFormat(utils.labelsToPanelLegend($._config.instanceLabels)), + + g.query.prometheus.withLegendFormat(utils.labelsToPanelLegend(legendInstanceLabels)), ], description='Memory usage percentage of the node for the Operating System and OpenSearch', ) @@ -88,7 +90,7 @@ local dashboardUidSuffix = '-node-overview'; agg: std.join(',', $._config.groupLabels + $._config.instanceLabels), }, ) - + g.query.prometheus.withLegendFormat('%s - read' % utils.labelsToPanelLegend($._config.instanceLabels)), + + g.query.prometheus.withLegendFormat('%s - read' % utils.labelsToPanelLegend(legendInstanceLabels)), g.query.prometheus.new( promDatasource.uid, 'sum by(%(agg)s) (rate(opensearch_fs_io_total_write_bytes{%(queriesSelector)s}[$__rate_interval]))' @@ -97,7 +99,7 @@ local dashboardUidSuffix = '-node-overview'; agg: std.join(',', $._config.groupLabels + $._config.instanceLabels), }, ) - + g.query.prometheus.withLegendFormat('%s - write' % utils.labelsToPanelLegend($._config.instanceLabels)), + + g.query.prometheus.withLegendFormat('%s - write' % utils.labelsToPanelLegend(legendInstanceLabels)), ], description='Node file system read and write data.', ) @@ -115,7 +117,7 @@ local dashboardUidSuffix = '-node-overview'; agg: std.join(',', $._config.groupLabels + $._config.instanceLabels), }, ) - + g.query.prometheus.withLegendFormat(utils.labelsToPanelLegend($._config.instanceLabels)), + + g.query.prometheus.withLegendFormat(utils.labelsToPanelLegend(legendInstanceLabels)), ], description='Number of open connections for the selected node.', ) @@ -134,7 +136,7 @@ local dashboardUidSuffix = '-node-overview'; agg: std.join(',', $._config.groupLabels + $._config.instanceLabels), }, ) - + g.query.prometheus.withLegendFormat(utils.labelsToPanelLegend($._config.instanceLabels)), + + g.query.prometheus.withLegendFormat(utils.labelsToPanelLegend(legendInstanceLabels)), ], description='Disk usage percentage of the selected node.', ), @@ -151,7 +153,7 @@ local dashboardUidSuffix = '-node-overview'; agg: std.join(',', $._config.groupLabels + $._config.instanceLabels), }, ) - + g.query.prometheus.withLegendFormat(utils.labelsToPanelLegend($._config.instanceLabels)), + + g.query.prometheus.withLegendFormat(utils.labelsToPanelLegend(legendInstanceLabels)), ], description='Percentage of swap space used by OpenSearch and the Operating System on the selected node.', ) @@ -169,7 +171,7 @@ local dashboardUidSuffix = '-node-overview'; agg: std.join(',', $._config.groupLabels + $._config.instanceLabels), }, ) - + g.query.prometheus.withLegendFormat('%s - sent' % utils.labelsToPanelLegend($._config.instanceLabels)), + + g.query.prometheus.withLegendFormat('%s - sent' % utils.labelsToPanelLegend(legendInstanceLabels)), g.query.prometheus.new( promDatasource.uid, 'sum by (%(agg)s) (rate(opensearch_transport_rx_bytes_count{%(queriesSelector)s}[$__rate_interval])) * 8' @@ -179,7 +181,7 @@ local dashboardUidSuffix = '-node-overview'; agg: std.join(',', $._config.groupLabels + $._config.instanceLabels), } - ) + g.query.prometheus.withLegendFormat('%s - received' % utils.labelsToPanelLegend($._config.instanceLabels)), + ) + g.query.prometheus.withLegendFormat('%s - received' % utils.labelsToPanelLegend(legendInstanceLabels)), ], description='Node network traffic sent and received.', ) @@ -196,7 +198,7 @@ local dashboardUidSuffix = '-node-overview'; agg: std.join(',', $._config.groupLabels + $._config.instanceLabels), }, datasource=promDatasource, - legendFormat='%s - {{ name }}' % utils.labelsToPanelLegend($._config.instanceLabels), + legendFormat='%s - {{ name }}' % utils.labelsToPanelLegend(legendInstanceLabels), interval='1m', ), ], @@ -284,7 +286,7 @@ local dashboardUidSuffix = '-node-overview'; agg: std.join(',', $._config.groupLabels + $._config.instanceLabels), }, datasource=promDatasource, - legendFormat='%s - used' % utils.labelsToPanelLegend($._config.instanceLabels), + legendFormat='%s - used' % utils.labelsToPanelLegend(legendInstanceLabels), ), prometheus.target( 'sum by (%(agg)s) (opensearch_jvm_mem_heap_committed_bytes{%(queriesSelector)s})' @@ -293,7 +295,7 @@ local dashboardUidSuffix = '-node-overview'; agg: std.join(',', $._config.groupLabels + $._config.instanceLabels), }, datasource=promDatasource, - legendFormat='%s - commited' % utils.labelsToPanelLegend($._config.instanceLabels), + legendFormat='%s - commited' % utils.labelsToPanelLegend(legendInstanceLabels), ), ], type: 'timeseries', @@ -372,7 +374,7 @@ local dashboardUidSuffix = '-node-overview'; agg: std.join(',', $._config.groupLabels + $._config.instanceLabels), }, datasource=promDatasource, - legendFormat='%s - used' % utils.labelsToPanelLegend($._config.instanceLabels), + legendFormat='%s - used' % utils.labelsToPanelLegend(legendInstanceLabels), ), prometheus.target( 'sum by (%(agg)s) (opensearch_jvm_mem_nonheap_committed_bytes{%(queriesSelector)s})' @@ -381,7 +383,7 @@ local dashboardUidSuffix = '-node-overview'; agg: std.join(',', $._config.groupLabels + $._config.instanceLabels), }, datasource=promDatasource, - legendFormat='%s - commited' % utils.labelsToPanelLegend($._config.instanceLabels), + legendFormat='%s - commited' % utils.labelsToPanelLegend(legendInstanceLabels), ), ], type: 'timeseries', @@ -460,7 +462,7 @@ local dashboardUidSuffix = '-node-overview'; agg: std.join(',', $._config.groupLabels + $._config.instanceLabels), }, datasource=promDatasource, - legendFormat=utils.labelsToPanelLegend($._config.instanceLabels), + legendFormat=utils.labelsToPanelLegend(legendInstanceLabels), ), ], type: 'timeseries', @@ -543,7 +545,7 @@ local dashboardUidSuffix = '-node-overview'; agg: std.join(',', $._config.groupLabels + $._config.instanceLabels), }, datasource=promDatasource, - legendFormat='%s - {{bufferpool}}' % utils.labelsToPanelLegend($._config.instanceLabels), + legendFormat='%s - {{bufferpool}}' % utils.labelsToPanelLegend(legendInstanceLabels), ), ], type: 'timeseries', @@ -626,7 +628,7 @@ local dashboardUidSuffix = '-node-overview'; agg: std.join(',', $._config.groupLabels + $._config.instanceLabels), }, datasource=promDatasource, - legendFormat=utils.labelsToPanelLegend($._config.instanceLabels), + legendFormat=utils.labelsToPanelLegend(legendInstanceLabels), ), ], type: 'timeseries', @@ -710,7 +712,7 @@ local dashboardUidSuffix = '-node-overview'; agg: std.join(',', $._config.groupLabels + $._config.instanceLabels), }, datasource=promDatasource, - legendFormat=utils.labelsToPanelLegend($._config.instanceLabels), + legendFormat=utils.labelsToPanelLegend(legendInstanceLabels), interval='1m', ), ], @@ -794,7 +796,7 @@ local dashboardUidSuffix = '-node-overview'; agg: std.join(',', $._config.groupLabels + $._config.instanceLabels), }, datasource=promDatasource, - legendFormat=utils.labelsToPanelLegend($._config.instanceLabels), + legendFormat=utils.labelsToPanelLegend(legendInstanceLabels), interval='1m', ), ], @@ -874,7 +876,7 @@ local dashboardUidSuffix = '-node-overview'; agg: std.join(',', $._config.groupLabels + $._config.instanceLabels), }, datasource=promDatasource, - legendFormat='%s - {{bufferpool}}' % utils.labelsToPanelLegend($._config.instanceLabels), + legendFormat='%s - {{bufferpool}}' % utils.labelsToPanelLegend(legendInstanceLabels), ), ], type: 'timeseries', @@ -965,7 +967,7 @@ local dashboardUidSuffix = '-node-overview'; agg: std.join(',', $._config.groupLabels + $._config.instanceLabels), }, datasource=promDatasource, - legendFormat=utils.labelsToPanelLegend($._config.instanceLabels), + legendFormat=utils.labelsToPanelLegend(legendInstanceLabels), ), ], type: 'timeseries', @@ -1047,7 +1049,7 @@ local dashboardUidSuffix = '-node-overview'; agg: std.join(',', $._config.groupLabels + $._config.instanceLabels), }, datasource=promDatasource, - legendFormat=utils.labelsToPanelLegend($._config.instanceLabels), + legendFormat=utils.labelsToPanelLegend(legendInstanceLabels), ), ], type: 'timeseries', From c831e622e8ad90113f7963a697b8499d1e0d7ac7 Mon Sep 17 00:00:00 2001 From: v-zhuravlev Date: Wed, 18 Dec 2024 22:04:16 +0800 Subject: [PATCH 3/3] Logs lib: make allValue configurable and add adhoc option (#1378) * Add configurable customAllValue to logs-lib * Revert customAllValue default to .* * Add optional adhoc support for logs-lib --- logs-lib/logs/main.libsonnet | 6 ++++++ logs-lib/logs/variables.libsonnet | 28 +++++++++++++++++++++++----- 2 files changed, 29 insertions(+), 5 deletions(-) diff --git a/logs-lib/logs/main.libsonnet b/logs-lib/logs/main.libsonnet index 6b80ee4f0..386319620 100644 --- a/logs-lib/logs/main.libsonnet +++ b/logs-lib/logs/main.libsonnet @@ -16,6 +16,9 @@ local variables = import './variables.libsonnet'; showLogsVolume=true, logsVolumeGroupBy='level', extraFilters='', + customAllValue='.*', + adHocEnabled=false, + adHocLabels=[], ): { local this = self, @@ -25,6 +28,9 @@ local variables = import './variables.libsonnet'; datasourceRegex, filterSelector, labels, + customAllValue, + adHocEnabled, + adHocLabels, ), targets: targets( diff --git a/logs-lib/logs/variables.libsonnet b/logs-lib/logs/variables.libsonnet index 3e1dd48a7..012d121c3 100644 --- a/logs-lib/logs/variables.libsonnet +++ b/logs-lib/logs/variables.libsonnet @@ -1,12 +1,16 @@ -local utils = import '../utils.libsonnet'; -local g = import './g.libsonnet'; +local g = import '../g.libsonnet'; local var = g.dashboard.variable; +local utils = import '../utils.libsonnet'; + function( datasourceName, datasourceLabel, datasourceRegex, filterSelector, labels, + customAllValue, + adHocEnabled, + adHocLabels, ) { // strip trailing or starting comma if present that are not accepted in LoqQL @@ -24,7 +28,7 @@ function( ) + var.query.selectionOptions.withIncludeAll( value=true, - customAllValue='.+' + customAllValue=customAllValue, ) + var.query.selectionOptions.withMulti() + var.query.refresh.onTime() @@ -45,14 +49,28 @@ function( + var.datasource.withRegex(datasourceRegex) + var.query.generalOptions.withLabel(datasourceLabel), - regex_search: + regexSearch: var.textbox.new('regex_search', default='') + var.query.generalOptions.withLabel('Regex search'), + adHoc: + var.adhoc.new('adhoc', 'loki', '${' + self.datasource.name + '}') + + var.adhoc.generalOptions.withLabel('Adhoc filters') + + var.adhoc.generalOptions.withDescription('Add additional filters') + + (if std.length(adHocLabels) > 0 then { + defaultKeys: [ + { + text: l, + value: l, + } + for l in adHocLabels + ], + } else {}), toArray: [self.datasource] + variablesFromLabels(labels, _filteringSelector) - + [self.regex_search], + + [self.regexSearch] + + (if adHocEnabled then [self.adHoc] else []), queriesSelector: std.join(