From 71ae7d323251115f1075e6ada33c373d2bbb2a1e Mon Sep 17 00:00:00 2001 From: Ivan Shumkov Date: Wed, 24 Apr 2024 00:30:27 +0700 Subject: [PATCH] feat(dashmate): expose metrics and admin interface --- Dockerfile | 3 +- .../configs/defaults/getBaseConfigFactory.js | 20 ++- .../configs/getConfigFileMigrationsFactory.js | 9 ++ .../docker-compose.rate_limiter.metrics.yml | 20 +++ .../dashmate/docker-compose.rate_limiter.yml | 4 +- packages/dashmate/docker-compose.yml | 13 +- .../dashmate/src/config/configJsonSchema.js | 129 ++++++++++-------- .../src/config/generateEnvsFactory.js | 5 + .../setup/setupLocalPresetTaskFactory.js | 3 + .../platform/dapi/envoy/envoy.yaml.dot | 52 ++++++- .../{ => rate_limiter}/rate_limiter.yaml.dot | 0 .../rate_limiter/statsd_exporter.yaml.dot | 93 +++++++++++++ 12 files changed, 281 insertions(+), 70 deletions(-) create mode 100644 packages/dashmate/docker-compose.rate_limiter.metrics.yml rename packages/dashmate/templates/platform/dapi/envoy/{ => rate_limiter}/rate_limiter.yaml.dot (100%) create mode 100644 packages/dashmate/templates/platform/dapi/envoy/rate_limiter/statsd_exporter.yaml.dot diff --git a/Dockerfile b/Dockerfile index 7cafb4713d..a7c3030302 100644 --- a/Dockerfile +++ b/Dockerfile @@ -194,7 +194,8 @@ RUN --mount=type=cache,sharing=shared,id=cargo_registry_index,target=${CARGO_HOM if [[ -z "${SCCACHE_MEMCACHED}" ]] ; then unset SCCACHE_MEMCACHED ; fi ; \ cargo build \ --profile "$CARGO_BUILD_PROFILE" \ - --package drive-abci && \ + --package drive-abci \ + --locked && \ cp /platform/target/*/drive-abci /artifacts/ && \ if [[ "${RUSTC_WRAPPER}" == "sccache" ]] ; then sccache --show-stats; fi diff --git a/packages/dashmate/configs/defaults/getBaseConfigFactory.js b/packages/dashmate/configs/defaults/getBaseConfigFactory.js index 29968c0bb3..df144f15ba 100644 --- a/packages/dashmate/configs/defaults/getBaseConfigFactory.js +++ b/packages/dashmate/configs/defaults/getBaseConfigFactory.js @@ -149,6 +149,16 @@ export default function getBaseConfigFactory(homeDir) { maxPendingRequests: 50, }, }, + metrics: { + enabled: false, + host: '127.0.0.1', + port: 9090, + }, + admin: { + enabled: false, + host: '127.0.0.1', + port: 9901, + }, http: { host: '0.0.0.0', port: 443, @@ -157,6 +167,14 @@ export default function getBaseConfigFactory(homeDir) { docker: { image: 'envoyproxy/ratelimit:3fcc3609', }, + metrics: { + enabled: false, + docker: { + image: 'prom/statsd-exporter:v0.26.1', + }, + host: '127.0.0.1', + port: 9102, + }, unit: 'minute', requestsPerUnit: 150, blacklist: [], @@ -212,7 +230,7 @@ export default function getBaseConfigFactory(homeDir) { enabled: false, host: '127.0.0.1', port: 6669, - retention_secs: 60 * 3, + retention: 60 * 3, }, validatorSet: { llmqType: 4, diff --git a/packages/dashmate/configs/getConfigFileMigrationsFactory.js b/packages/dashmate/configs/getConfigFileMigrationsFactory.js index 981a955fb3..aadc4ce50b 100644 --- a/packages/dashmate/configs/getConfigFileMigrationsFactory.js +++ b/packages/dashmate/configs/getConfigFileMigrationsFactory.js @@ -538,6 +538,8 @@ export default function getConfigFileMigrationsFactory(homeDir, defaultConfigs) delete options.platform.dapi.envoy.http.responseTimeout; } + options.platform.dapi.envoy.metrics = base.get('platform.dapi.envoy.metrics'); + options.platform.dapi.envoy.admin = base.get('platform.dapi.envoy.admin'); options.platform.dapi.envoy.upstreams = base.get('platform.dapi.envoy.upstreams'); options.platform.dapi.envoy.rateLimiter.docker = base.get('platform.dapi.envoy.rateLimiter.docker'); @@ -545,6 +547,7 @@ export default function getConfigFileMigrationsFactory(homeDir, defaultConfigs) options.platform.dapi.envoy.rateLimiter.requestsPerUnit = base.get('platform.dapi.envoy.rateLimiter.requestsPerUnit'); options.platform.dapi.envoy.rateLimiter.blacklist = base.get('platform.dapi.envoy.rateLimiter.blacklist'); options.platform.dapi.envoy.rateLimiter.whitelist = base.get('platform.dapi.envoy.rateLimiter.whitelist'); + options.platform.dapi.envoy.rateLimiter.metrics = base.get('platform.dapi.envoy.rateLimiter.metrics'); if (options.platform.dapi.envoy.rateLimiter.fillInterval) { delete options.platform.dapi.envoy.rateLimiter.fillInterval; @@ -559,6 +562,12 @@ export default function getConfigFileMigrationsFactory(homeDir, defaultConfigs) } options.platform.dapi.envoy.docker.image = base.get('platform.dapi.envoy.docker.image'); + + if (options.platform.drive.abci.tokioConsole.retention_secs) { + options.platform.drive.abci.tokioConsole.retention = options.platform.drive.abci + .tokioConsole.retention_secs; + delete options.platform.drive.abci.tokioConsole.retention_secs; + } }); return configFile; diff --git a/packages/dashmate/docker-compose.rate_limiter.metrics.yml b/packages/dashmate/docker-compose.rate_limiter.metrics.yml new file mode 100644 index 0000000000..d436e2a3e9 --- /dev/null +++ b/packages/dashmate/docker-compose.rate_limiter.metrics.yml @@ -0,0 +1,20 @@ +version: '3.7' + +services: + dapi_envoy_rate_limiter_metrics: + image: ${PLATFORM_DAPI_ENVOY_RATE_LIMITER_METRICS_DOCKER_IMAGE:?err} + labels: + org.dashmate.service.title: "DAPI Envoy rate limiter metrics exporter" + restart: unless-stopped + entrypoint: /bin/statsd_exporter + command: + - "--statsd.mapping-config=/etc/statsd-exporter/config.yaml" + networks: + - envoy_rate_limiter + volumes: + - ${DASHMATE_HOME_DIR:?err}/${CONFIG_NAME:?err}/platform/dapi/envoy/rate_limiter/statsd_exporter.yaml:/etc/statsd-exporter/config.yaml:ro + expose: + - 9125 + - 9125/udp + ports: + - ${PLATFORM_DAPI_ENVOY_RATE_LIMITER_METRICS_HOST:?err}:${PLATFORM_DAPI_ENVOY_RATE_LIMITER_METRICS_PORT:?err}:9102 diff --git a/packages/dashmate/docker-compose.rate_limiter.yml b/packages/dashmate/docker-compose.rate_limiter.yml index 6fc99abdaf..7fa981b85e 100644 --- a/packages/dashmate/docker-compose.rate_limiter.yml +++ b/packages/dashmate/docker-compose.rate_limiter.yml @@ -29,7 +29,9 @@ services: - RUNTIME_ROOT=/data - RUNTIME_SUBDIRECTORY=ratelimit - RUNTIME_WATCH_ROOT=false - - DISABLE_STATS=true # TODO: We probably want to collect metrics? + - DISABLE_STATS=${PLATFORM_DAPI_ENVOY_RATE_LIMITER_METRICS_DISABLED:?err} + - STATSD_HOST=dapi_envoy_rate_limiter_metrics + - STATSD_PORT=9125 - CONFIG_TYPE=FILE - GRPC_MAX_CONNECTION_AGE=1h - GRPC_MAX_CONNECTION_AGE_GRACE=10m diff --git a/packages/dashmate/docker-compose.yml b/packages/dashmate/docker-compose.yml index 2693e9f729..e14be2c694 100644 --- a/packages/dashmate/docker-compose.yml +++ b/packages/dashmate/docker-compose.yml @@ -28,8 +28,8 @@ services: networks: - main ports: - - ${CORE_P2P_HOST:?err}:${CORE_P2P_PORT:?err}:${CORE_P2P_PORT:?err} # P2P - - ${CORE_RPC_HOST:?err}:${CORE_RPC_PORT:?err}:${CORE_RPC_PORT:?err} #RPC + - ${CORE_P2P_HOST:?err}:${CORE_P2P_PORT:?err}:${CORE_P2P_PORT:?err} + - ${CORE_RPC_HOST:?err}:${CORE_RPC_PORT:?err}:${CORE_RPC_PORT:?err} volumes: - core_data:/home/dash - ${DASHMATE_HOME_DIR:?err}/${CONFIG_NAME:?err}/core/dash.conf:/home/dash/.dashcore/dash.conf:ro @@ -79,7 +79,7 @@ services: - PROMETHEUS_BIND_ADDRESS=http://0.0.0.0:29090 - TOKIO_CONSOLE_ENABLED=${PLATFORM_DRIVE_ABCI_TOKIO_CONSOLE_ENABLED:?err} - TOKIO_CONSOLE_ADDRESS=0.0.0.0:${PLATFORM_DRIVE_ABCI_TOKIO_CONSOLE_PORT:?err} - - TOKIO_CONSOLE_RETENTION_SECS=${PLATFORM_DRIVE_ABCI_TOKIO_CONSOLE_RETENTION_SECS:?err} + - TOKIO_CONSOLE_RETENTION_SECS=${PLATFORM_DRIVE_ABCI_TOKIO_CONSOLE_RETENTION:?err} stop_grace_period: 30s networks: - main @@ -195,7 +195,12 @@ services: org.dashmate.service.title: "DAPI Envoy" restart: unless-stopped ports: - - ${PLATFORM_DAPI_ENVOY_HTTP_HOST:?err}:${PLATFORM_DAPI_ENVOY_HTTP_PORT:?err}:10000 # JSON RPC and gRPC Web & Native + # Actual entry point to the platform. + # Supports HTTP1 and HTTP2 + # Serves JSON RPC, gRPC, and gRPC-Web + - ${PLATFORM_DAPI_ENVOY_HTTP_HOST:?err}:${PLATFORM_DAPI_ENVOY_HTTP_PORT:?err}:10000 + - ${PLATFORM_DAPI_ENVOY_METRICS_HOST:?err}:${PLATFORM_DAPI_ENVOY_METRICS_PORT:?err}:9090 + - ${PLATFORM_DAPI_ENVOY_ADMIN_HOST:?err}:${PLATFORM_DAPI_ENVOY_ADMIN_PORT:?err}:9901 depends_on: - dapi_api - dapi_tx_filter_stream diff --git a/packages/dashmate/src/config/configJsonSchema.js b/packages/dashmate/src/config/configJsonSchema.js index c6f4d603ec..12a7f7ae69 100644 --- a/packages/dashmate/src/config/configJsonSchema.js +++ b/packages/dashmate/src/config/configJsonSchema.js @@ -50,6 +50,11 @@ export default { required: ['image', 'build'], additionalProperties: false, }, + host: { + type: 'string', + minLength: 1, + format: 'ipv4', + }, port: { type: 'integer', minimum: 0, @@ -84,39 +89,21 @@ export default { type: 'string', pattern: '^[0-9]+(\\.[0-9]+)?s$', }, - envoyUpstream: { + enabledHostPort: { type: 'object', properties: { - maxConnections: { - type: 'integer', - minimum: 1, - description: 'The maximum number of connections that Envoy will establish to all hosts in a cluster', + enabled: { + type: 'boolean', }, - maxPendingRequests: { - type: 'integer', - minimum: 1, - description: 'The maximum number of requests that will be queued if `maxRequests` is reached.', + host: { + $ref: '#/definitions/host', }, - maxRequests: { - type: 'integer', - minimum: 1, - description: 'The maximum number of parallel requests', + port: { + $ref: '#/definitions/port', }, }, - required: ['maxConnections', 'maxPendingRequests', 'maxRequests'], additionalProperties: false, - }, - ipList: { - type: 'string', - oneOf: [ - { - format: 'ipv4', - }, - { - format: 'ipv6', - }, - ], - minLength: 1, + required: ['enabled', 'host', 'port'], }, }, properties: { @@ -207,9 +194,7 @@ export default { type: 'object', properties: { host: { - type: 'string', - minLength: 1, - format: 'ipv4', + $ref: '#/definitions/host', }, port: { $ref: '#/definitions/port', @@ -404,18 +389,44 @@ export default { type: 'object', properties: { driveGrpc: { - $ref: '#/definitions/envoyUpstream', + $id: 'envoyUpstream', + type: 'object', + properties: { + maxConnections: { + type: 'integer', + minimum: 1, + description: 'The maximum number of connections that Envoy will establish to all hosts in a cluster', + }, + maxPendingRequests: { + type: 'integer', + minimum: 1, + description: 'The maximum number of requests that will be queued if `maxRequests` is reached.', + }, + maxRequests: { + type: 'integer', + minimum: 1, + description: 'The maximum number of parallel requests', + }, + }, + required: ['maxConnections', 'maxPendingRequests', 'maxRequests'], + additionalProperties: false, }, dapiApi: { - $ref: '#/definitions/envoyUpstream', + $ref: 'envoyUpstream', }, dapiCoreStreams: { - $ref: '#/definitions/envoyUpstream', + $ref: 'envoyUpstream', }, }, additionalProperties: false, required: ['driveGrpc', 'dapiApi', 'dapiCoreStreams'], }, + metrics: { + $ref: '#/definitions/enabledHostPort', + }, + admin: { + $ref: '#/definitions/enabledHostPort', + }, http: { type: 'object', properties: { @@ -448,22 +459,41 @@ export default { blacklist: { type: 'array', items: { - $ref: '#/definitions/ipList', + $ref: '#/definitions/host', }, description: 'List of IP addresses that are blacklisted from making requests', }, whitelist: { type: 'array', items: { - $ref: '#/definitions/ipList', + $ref: '#/definitions/host', }, description: 'List of IP addresses that are whitelisted to make requests without limits', }, + metrics: { + type: 'object', + properties: { + docker: { + $ref: '#/definitions/docker', + }, + enabled: { + type: 'boolean', + }, + host: { + $ref: '#/definitions/host', + }, + port: { + $ref: '#/definitions/port', + }, + }, + additionalProperties: false, + required: ['docker', 'enabled', 'host', 'port'], + }, enabled: { type: 'boolean', }, }, - required: ['docker', 'enabled', 'unit', 'requestsPerUnit', 'blacklist', 'whitelist'], + required: ['docker', 'enabled', 'unit', 'requestsPerUnit', 'blacklist', 'whitelist', 'metrics'], additionalProperties: false, }, ssl: { @@ -501,7 +531,7 @@ export default { additionalProperties: false, }, }, - required: ['docker', 'http', 'rateLimiter', 'ssl', 'maxHeapSizeInBytes', 'maxConnections', 'upstreams'], + required: ['docker', 'http', 'rateLimiter', 'ssl', 'maxHeapSizeInBytes', 'maxConnections', 'upstreams', 'metrics', 'admin'], additionalProperties: false, }, api: { @@ -588,19 +618,18 @@ export default { type: 'boolean', }, host: { - type: 'string', - minLength: 1, - format: 'ipv4', + $ref: '#/definitions/host', }, port: { $ref: '#/definitions/port', }, - retention_secs: { + retention: { type: 'integer', minimum: 0, + description: 'How many seconds keep data if console is not connected', }, }, - required: ['enabled', 'host', 'port', 'retention_secs'], + required: ['enabled', 'host', 'port', 'retention'], additionalProperties: false, }, validatorSet: { @@ -855,23 +884,7 @@ export default { additionalProperties: false, }, metrics: { - description: 'Prometheus metrics', - type: 'object', - properties: { - enabled: { - type: 'boolean', - }, - host: { - type: 'string', - minLength: 1, - format: 'ipv4', - }, - port: { - $ref: '#/definitions/port', - }, - }, - required: ['enabled', 'host', 'port'], - additionalProperties: false, + $ref: '#/definitions/enabledHostPort', }, node: { type: 'object', diff --git a/packages/dashmate/src/config/generateEnvsFactory.js b/packages/dashmate/src/config/generateEnvsFactory.js index efce5bd046..be95295835 100644 --- a/packages/dashmate/src/config/generateEnvsFactory.js +++ b/packages/dashmate/src/config/generateEnvsFactory.js @@ -58,6 +58,10 @@ export default function generateEnvsFactory(configFile, homeDir, getConfigProfil if (config.get('platform.dapi.envoy.rateLimiter.enabled')) { dockerComposeFiles.push('docker-compose.rate_limiter.yml'); + + if (config.get('platform.dapi.envoy.rateLimiter.metrics.enabled')) { + dockerComposeFiles.push('docker-compose.rate_limiter.metrics.yml'); + } } // we need this for compatibility with old configs @@ -88,6 +92,7 @@ export default function generateEnvsFactory(configFile, homeDir, getConfigProfil ), DASHMATE_HELPER_DOCKER_IMAGE, PLATFORM_DRIVE_TENDERDASH_LOG_DIRECTORY_PATH: tenderdashLogDirectoryPath, + PLATFORM_DAPI_ENVOY_RATE_LIMITER_METRICS_DISABLED: config.get('platform.dapi.envoy.rateLimiter.metrics.enabled'), ...convertObjectToEnvs(config.getOptions()), }; } diff --git a/packages/dashmate/src/listr/tasks/setup/setupLocalPresetTaskFactory.js b/packages/dashmate/src/listr/tasks/setup/setupLocalPresetTaskFactory.js index 70a8cd7f96..42be80e527 100644 --- a/packages/dashmate/src/listr/tasks/setup/setupLocalPresetTaskFactory.js +++ b/packages/dashmate/src/listr/tasks/setup/setupLocalPresetTaskFactory.js @@ -223,7 +223,10 @@ export default function setupLocalPresetTaskFactory( config.set('platform.drive.tenderdash.node.key', key); config.set('platform.drive.abci.tokioConsole.port', config.get('platform.drive.abci.tokioConsole.port') + (i * 100)); + config.set('platform.dapi.envoy.admin.port', config.get('platform.dapi.envoy.admin.port') + (i * 100)); config.set('platform.dapi.envoy.http.port', config.get('platform.dapi.envoy.http.port') + (i * 100)); + config.set('platform.dapi.envoy.metrics.port', config.get('platform.dapi.envoy.metrics.port') + (i * 100)); + config.set('platform.dapi.envoy.rateLimiter.metrics.port', config.get('platform.dapi.envoy.rateLimiter.metrics.port') + (i * 100)); config.set('platform.drive.tenderdash.p2p.port', config.get('platform.drive.tenderdash.p2p.port') + (i * 100)); config.set('platform.drive.tenderdash.rpc.port', config.get('platform.drive.tenderdash.rpc.port') + (i * 100)); config.set('platform.drive.tenderdash.pprof.port', config.get('platform.drive.tenderdash.pprof.port') + (i * 100)); diff --git a/packages/dashmate/templates/platform/dapi/envoy/envoy.yaml.dot b/packages/dashmate/templates/platform/dapi/envoy/envoy.yaml.dot index c62dc248f3..2d808d3bc6 100644 --- a/packages/dashmate/templates/platform/dapi/envoy/envoy.yaml.dot +++ b/packages/dashmate/templates/platform/dapi/envoy/envoy.yaml.dot @@ -54,8 +54,6 @@ authority: "%REQ(:AUTHORITY)%" http_filters: {{? it.platform.dapi.envoy.rateLimiter.enabled}} - # Rate limiter - # Local HTTP Rate limiter can limit only connections or requests per connection. - name: envoy.filters.http.ratelimit typed_config: "@type": type.googleapis.com/envoy.extensions.filters.http.ratelimit.v3.RateLimit @@ -181,10 +179,10 @@ static_resources: listeners: - - name: all_http + - name: dapi_and_drive_queries address: socket_address: - address: 0.0.0.0 + address: 0.0.0.0 # For docker container only. Must be a local/private interface. port_value: 10000 {{? it.platform.dapi.envoy.ssl.provider === 'self-signed'}} listener_filters: @@ -225,6 +223,34 @@ static_resources: private_key: filename: "/etc/ssl/private.key" {{?}} + {{? it.platform.dapi.envoy.metrics.enabled }} + - name: prometheus_metrics + address: + socket_address: + address: "0.0.0.0" + port_value: 9090 + filter_chains: + - filters: + - name: "envoy.http_connection_manager" + typed_config: + "@type": type.googleapis.com/envoy.extensions.filters.network.http_connection_manager.v3.HttpConnectionManager + stat_prefix: ingress_metrics + route_config: + name: local_route + virtual_hosts: + - name: local_service + domains: [ "*" ] + routes: + - match: + prefix: "/metrics" + route: + cluster: envoy_admin + prefix_rewrite: "/stats/prometheus" + http_filters: + - name: envoy.filters.http.router + typed_config: + "@type": type.googleapis.com/envoy.extensions.filters.http.router.v3.Router + {{?}} clusters: - name: dapi_api type: STRICT_DNS @@ -349,12 +375,28 @@ static_resources: address: dapi_envoy_rate_limiter port_value: 8081 {{?}} + {{? it.platform.dapi.envoy.metrics.enabled }} + - name: envoy_admin + connect_timeout: 0.25s + type: STATIC + load_assignment: + cluster_name: envoy_admin + endpoints: + - lb_endpoints: + - endpoint: + address: + socket_address: + address: "127.0.0.1" + port_value: 9901 + {{?}} +{{? it.platform.dapi.envoy.metrics.enabled }} admin: address: socket_address: address: 0.0.0.0 # For docker container only. Must be a local/private interface. - port_value: 8081 + port_value: 9901 +{{?}} # Dynamically adjust limits based on memory usage and number of active connections # TODO: We can use data provided by drive, tenderdash, or dapi to configure adaptive limits based on load diff --git a/packages/dashmate/templates/platform/dapi/envoy/rate_limiter.yaml.dot b/packages/dashmate/templates/platform/dapi/envoy/rate_limiter/rate_limiter.yaml.dot similarity index 100% rename from packages/dashmate/templates/platform/dapi/envoy/rate_limiter.yaml.dot rename to packages/dashmate/templates/platform/dapi/envoy/rate_limiter/rate_limiter.yaml.dot diff --git a/packages/dashmate/templates/platform/dapi/envoy/rate_limiter/statsd_exporter.yaml.dot b/packages/dashmate/templates/platform/dapi/envoy/rate_limiter/statsd_exporter.yaml.dot new file mode 100644 index 0000000000..31f16dd6a5 --- /dev/null +++ b/packages/dashmate/templates/platform/dapi/envoy/rate_limiter/statsd_exporter.yaml.dot @@ -0,0 +1,93 @@ +mappings: # Requires statsd exporter >= v0.6.0 since it uses the "drop" action. + - match: "ratelimit.service.rate_limit.*.*.near_limit" + name: "ratelimit_service_rate_limit_near_limit" + timer_type: "histogram" + labels: + domain: "$1" + key1: "$2" + - match: "ratelimit.service.rate_limit.*.*.over_limit" + name: "ratelimit_service_rate_limit_over_limit" + timer_type: "histogram" + labels: + domain: "$1" + key1: "$2" + - match: "ratelimit.service.rate_limit.*.*.total_hits" + name: "ratelimit_service_rate_limit_total_hits" + timer_type: "histogram" + labels: + domain: "$1" + key1: "$2" + - match: "ratelimit.service.rate_limit.*.*.within_limit" + name: "ratelimit_service_rate_limit_within_limit" + timer_type: "histogram" + labels: + domain: "$1" + key1: "$2" + + - match: "ratelimit.service.rate_limit.*.*.*.near_limit" + name: "ratelimit_service_rate_limit_near_limit" + timer_type: "histogram" + labels: + domain: "$1" + key1: "$2" + key2: "$3" + - match: "ratelimit.service.rate_limit.*.*.*.over_limit" + name: "ratelimit_service_rate_limit_over_limit" + timer_type: "histogram" + labels: + domain: "$1" + key1: "$2" + key2: "$3" + - match: "ratelimit.service.rate_limit.*.*.*.total_hits" + name: "ratelimit_service_rate_limit_total_hits" + timer_type: "histogram" + labels: + domain: "$1" + key1: "$2" + key2: "$3" + - match: "ratelimit.service.rate_limit.*.*.*.within_limit" + name: "ratelimit_service_rate_limit_within_limit" + timer_type: "histogram" + labels: + domain: "$1" + key1: "$2" + key2: "$3" + + - match: "ratelimit.service.call.should_rate_limit.*" + name: "ratelimit_service_should_rate_limit_error" + match_metric_type: counter + labels: + err_type: "$1" + + - match: "ratelimit_server.*.total_requests" + name: "ratelimit_service_total_requests" + match_metric_type: counter + labels: + grpc_method: "$1" + + - match: "ratelimit_server.*.response_time" + name: "ratelimit_service_response_time_seconds" + timer_type: histogram + labels: + grpc_method: "$1" + + - match: "ratelimit.service.config_load_success" + name: "ratelimit_service_config_load_success" + match_metric_type: counter + - match: "ratelimit.service.config_load_error" + name: "ratelimit_service_config_load_error" + match_metric_type: counter + + - match: "ratelimit.service.rate_limit.*.*.*.shadow_mode" + name: "ratelimit_service_rate_limit_shadow_mode" + timer_type: "histogram" + labels: + domain: "$1" + key1: "$2" + key2: "$3" + + # Enable below in production once you have the metrics you need + # - match: "." + # match_type: "regex" + # action: "drop" + # name: "dropped"