feat(dashmate): expose metrics and admin interface

dashpay · Apr 23, 2024 · 71ae7d3 · 71ae7d3
1 parent 4a83f75
commit 71ae7d3
Show file tree

Hide file tree

Showing 12 changed files with 281 additions and 70 deletions.
diff --git a/Dockerfile b/Dockerfile
@@ -194,7 +194,8 @@ RUN --mount=type=cache,sharing=shared,id=cargo_registry_index,target=${CARGO_HOM
     if [[ -z "${SCCACHE_MEMCACHED}" ]] ; then unset SCCACHE_MEMCACHED ; fi ; \
     cargo build \
         --profile "$CARGO_BUILD_PROFILE" \
-        --package drive-abci && \
+        --package drive-abci \
+        --locked && \
     cp /platform/target/*/drive-abci /artifacts/ && \
     if [[ "${RUSTC_WRAPPER}" == "sccache" ]] ; then sccache --show-stats; fi
 

diff --git a/packages/dashmate/configs/defaults/getBaseConfigFactory.js b/packages/dashmate/configs/defaults/getBaseConfigFactory.js
@@ -149,6 +149,16 @@ export default function getBaseConfigFactory(homeDir) {
                 maxPendingRequests: 50,
               },
             },
+            metrics: {
+              enabled: false,
+              host: '127.0.0.1',
+              port: 9090,
+            },
+            admin: {
+              enabled: false,
+              host: '127.0.0.1',
+              port: 9901,
+            },
             http: {
               host: '0.0.0.0',
               port: 443,
@@ -157,6 +167,14 @@ export default function getBaseConfigFactory(homeDir) {
               docker: {
                 image: 'envoyproxy/ratelimit:3fcc3609',
               },
+              metrics: {
+                enabled: false,
+                docker: {
+                  image: 'prom/statsd-exporter:v0.26.1',
+                },
+                host: '127.0.0.1',
+                port: 9102,
+              },
               unit: 'minute',
               requestsPerUnit: 150,
               blacklist: [],
@@ -212,7 +230,7 @@ export default function getBaseConfigFactory(homeDir) {
               enabled: false,
               host: '127.0.0.1',
               port: 6669,
-              retention_secs: 60 * 3,
+              retention: 60 * 3,
             },
             validatorSet: {
               llmqType: 4,

diff --git a/packages/dashmate/configs/getConfigFileMigrationsFactory.js b/packages/dashmate/configs/getConfigFileMigrationsFactory.js
@@ -538,13 +538,16 @@ export default function getConfigFileMigrationsFactory(homeDir, defaultConfigs)
               delete options.platform.dapi.envoy.http.responseTimeout;
             }
 
+            options.platform.dapi.envoy.metrics = base.get('platform.dapi.envoy.metrics');
+            options.platform.dapi.envoy.admin = base.get('platform.dapi.envoy.admin');
             options.platform.dapi.envoy.upstreams = base.get('platform.dapi.envoy.upstreams');
 
             options.platform.dapi.envoy.rateLimiter.docker = base.get('platform.dapi.envoy.rateLimiter.docker');
             options.platform.dapi.envoy.rateLimiter.unit = base.get('platform.dapi.envoy.rateLimiter.unit');
             options.platform.dapi.envoy.rateLimiter.requestsPerUnit = base.get('platform.dapi.envoy.rateLimiter.requestsPerUnit');
             options.platform.dapi.envoy.rateLimiter.blacklist = base.get('platform.dapi.envoy.rateLimiter.blacklist');
             options.platform.dapi.envoy.rateLimiter.whitelist = base.get('platform.dapi.envoy.rateLimiter.whitelist');
+            options.platform.dapi.envoy.rateLimiter.metrics = base.get('platform.dapi.envoy.rateLimiter.metrics');
 
             if (options.platform.dapi.envoy.rateLimiter.fillInterval) {
               delete options.platform.dapi.envoy.rateLimiter.fillInterval;
@@ -559,6 +562,12 @@ export default function getConfigFileMigrationsFactory(homeDir, defaultConfigs)
             }
 
             options.platform.dapi.envoy.docker.image = base.get('platform.dapi.envoy.docker.image');
+
+            if (options.platform.drive.abci.tokioConsole.retention_secs) {
+              options.platform.drive.abci.tokioConsole.retention = options.platform.drive.abci
+                .tokioConsole.retention_secs;
+              delete options.platform.drive.abci.tokioConsole.retention_secs;
+            }
           });
 
         return configFile;

diff --git a/packages/dashmate/docker-compose.rate_limiter.metrics.yml b/packages/dashmate/docker-compose.rate_limiter.metrics.yml
@@ -0,0 +1,20 @@
+version: '3.7'
+
+services:
+  dapi_envoy_rate_limiter_metrics:
+    image: ${PLATFORM_DAPI_ENVOY_RATE_LIMITER_METRICS_DOCKER_IMAGE:?err}
+    labels:
+      org.dashmate.service.title: "DAPI Envoy rate limiter metrics exporter"
+    restart: unless-stopped
+    entrypoint: /bin/statsd_exporter
+    command:
+      - "--statsd.mapping-config=/etc/statsd-exporter/config.yaml"
+    networks:
+      - envoy_rate_limiter
+    volumes:
+      - ${DASHMATE_HOME_DIR:?err}/${CONFIG_NAME:?err}/platform/dapi/envoy/rate_limiter/statsd_exporter.yaml:/etc/statsd-exporter/config.yaml:ro
+    expose:
+      - 9125
+      - 9125/udp
+    ports:
+      - ${PLATFORM_DAPI_ENVOY_RATE_LIMITER_METRICS_HOST:?err}:${PLATFORM_DAPI_ENVOY_RATE_LIMITER_METRICS_PORT:?err}:9102
diff --git a/packages/dashmate/docker-compose.rate_limiter.yml b/packages/dashmate/docker-compose.rate_limiter.yml
@@ -29,7 +29,9 @@ services:
       - RUNTIME_ROOT=/data
       - RUNTIME_SUBDIRECTORY=ratelimit
       - RUNTIME_WATCH_ROOT=false
-      - DISABLE_STATS=true # TODO: We probably want to collect metrics?
+      - DISABLE_STATS=${PLATFORM_DAPI_ENVOY_RATE_LIMITER_METRICS_DISABLED:?err}
+      - STATSD_HOST=dapi_envoy_rate_limiter_metrics
+      - STATSD_PORT=9125
       - CONFIG_TYPE=FILE
       - GRPC_MAX_CONNECTION_AGE=1h
       - GRPC_MAX_CONNECTION_AGE_GRACE=10m

diff --git a/packages/dashmate/docker-compose.yml b/packages/dashmate/docker-compose.yml
@@ -28,8 +28,8 @@ services:
     networks:
       - main
     ports:
-      - ${CORE_P2P_HOST:?err}:${CORE_P2P_PORT:?err}:${CORE_P2P_PORT:?err} # P2P
-      - ${CORE_RPC_HOST:?err}:${CORE_RPC_PORT:?err}:${CORE_RPC_PORT:?err} #RPC
+      - ${CORE_P2P_HOST:?err}:${CORE_P2P_PORT:?err}:${CORE_P2P_PORT:?err}
+      - ${CORE_RPC_HOST:?err}:${CORE_RPC_PORT:?err}:${CORE_RPC_PORT:?err}
     volumes:
       - core_data:/home/dash
       - ${DASHMATE_HOME_DIR:?err}/${CONFIG_NAME:?err}/core/dash.conf:/home/dash/.dashcore/dash.conf:ro
@@ -79,7 +79,7 @@ services:
       - PROMETHEUS_BIND_ADDRESS=http://0.0.0.0:29090
       - TOKIO_CONSOLE_ENABLED=${PLATFORM_DRIVE_ABCI_TOKIO_CONSOLE_ENABLED:?err}
       - TOKIO_CONSOLE_ADDRESS=0.0.0.0:${PLATFORM_DRIVE_ABCI_TOKIO_CONSOLE_PORT:?err}
-      - TOKIO_CONSOLE_RETENTION_SECS=${PLATFORM_DRIVE_ABCI_TOKIO_CONSOLE_RETENTION_SECS:?err}
+      - TOKIO_CONSOLE_RETENTION_SECS=${PLATFORM_DRIVE_ABCI_TOKIO_CONSOLE_RETENTION:?err}
     stop_grace_period: 30s
     networks:
       - main
@@ -195,7 +195,12 @@ services:
       org.dashmate.service.title: "DAPI Envoy"
     restart: unless-stopped
     ports:
-      - ${PLATFORM_DAPI_ENVOY_HTTP_HOST:?err}:${PLATFORM_DAPI_ENVOY_HTTP_PORT:?err}:10000 # JSON RPC and gRPC Web & Native
+      # Actual entry point to the platform.
+      # Supports HTTP1 and HTTP2
+      # Serves JSON RPC, gRPC, and gRPC-Web
+      - ${PLATFORM_DAPI_ENVOY_HTTP_HOST:?err}:${PLATFORM_DAPI_ENVOY_HTTP_PORT:?err}:10000
+      - ${PLATFORM_DAPI_ENVOY_METRICS_HOST:?err}:${PLATFORM_DAPI_ENVOY_METRICS_PORT:?err}:9090
+      - ${PLATFORM_DAPI_ENVOY_ADMIN_HOST:?err}:${PLATFORM_DAPI_ENVOY_ADMIN_PORT:?err}:9901
     depends_on:
       - dapi_api
       - dapi_tx_filter_stream

diff --git a/packages/dashmate/src/config/configJsonSchema.js b/packages/dashmate/src/config/configJsonSchema.js
@@ -50,6 +50,11 @@ export default {
       required: ['image', 'build'],
       additionalProperties: false,
     },
+    host: {
+      type: 'string',
+      minLength: 1,
+      format: 'ipv4',
+    },
     port: {
       type: 'integer',
       minimum: 0,
@@ -84,39 +89,21 @@ export default {
       type: 'string',
       pattern: '^[0-9]+(\\.[0-9]+)?s$',
     },
-    envoyUpstream: {
+    enabledHostPort: {
       type: 'object',
       properties: {
-        maxConnections: {
-          type: 'integer',
-          minimum: 1,
-          description: 'The maximum number of connections that Envoy will establish to all hosts in a cluster',
+        enabled: {
+          type: 'boolean',
         },
-        maxPendingRequests: {
-          type: 'integer',
-          minimum: 1,
-          description: 'The maximum number of requests that will be queued if `maxRequests` is reached.',
+        host: {
+          $ref: '#/definitions/host',
         },
-        maxRequests: {
-          type: 'integer',
-          minimum: 1,
-          description: 'The maximum number of parallel requests',
+        port: {
+          $ref: '#/definitions/port',
         },
       },
-      required: ['maxConnections', 'maxPendingRequests', 'maxRequests'],
       additionalProperties: false,
-    },
-    ipList: {
-      type: 'string',
-      oneOf: [
-        {
-          format: 'ipv4',
-        },
-        {
-          format: 'ipv6',
-        },
-      ],
-      minLength: 1,
+      required: ['enabled', 'host', 'port'],
     },
   },
   properties: {
@@ -207,9 +194,7 @@ export default {
           type: 'object',
           properties: {
             host: {
-              type: 'string',
-              minLength: 1,
-              format: 'ipv4',
+              $ref: '#/definitions/host',
             },
             port: {
               $ref: '#/definitions/port',
@@ -404,18 +389,44 @@ export default {
                   type: 'object',
                   properties: {
                     driveGrpc: {
-                      $ref: '#/definitions/envoyUpstream',
+                      $id: 'envoyUpstream',
+                      type: 'object',
+                      properties: {
+                        maxConnections: {
+                          type: 'integer',
+                          minimum: 1,
+                          description: 'The maximum number of connections that Envoy will establish to all hosts in a cluster',
+                        },
+                        maxPendingRequests: {
+                          type: 'integer',
+                          minimum: 1,
+                          description: 'The maximum number of requests that will be queued if `maxRequests` is reached.',
+                        },
+                        maxRequests: {
+                          type: 'integer',
+                          minimum: 1,
+                          description: 'The maximum number of parallel requests',
+                        },
+                      },
+                      required: ['maxConnections', 'maxPendingRequests', 'maxRequests'],
+                      additionalProperties: false,
                     },
                     dapiApi: {
-                      $ref: '#/definitions/envoyUpstream',
+                      $ref: 'envoyUpstream',
                     },
                     dapiCoreStreams: {
-                      $ref: '#/definitions/envoyUpstream',
+                      $ref: 'envoyUpstream',
                     },
                   },
                   additionalProperties: false,
                   required: ['driveGrpc', 'dapiApi', 'dapiCoreStreams'],
                 },
+                metrics: {
+                  $ref: '#/definitions/enabledHostPort',
+                },
+                admin: {
+                  $ref: '#/definitions/enabledHostPort',
+                },
                 http: {
                   type: 'object',
                   properties: {
@@ -448,22 +459,41 @@ export default {
                     blacklist: {
                       type: 'array',
                       items: {
-                        $ref: '#/definitions/ipList',
+                        $ref: '#/definitions/host',
                       },
                       description: 'List of IP addresses that are blacklisted from making requests',
                     },
                     whitelist: {
                       type: 'array',
                       items: {
-                        $ref: '#/definitions/ipList',
+                        $ref: '#/definitions/host',
                       },
                       description: 'List of IP addresses that are whitelisted to make requests without limits',
                     },
+                    metrics: {
+                      type: 'object',
+                      properties: {
+                        docker: {
+                          $ref: '#/definitions/docker',
+                        },
+                        enabled: {
+                          type: 'boolean',
+                        },
+                        host: {
+                          $ref: '#/definitions/host',
+                        },
+                        port: {
+                          $ref: '#/definitions/port',
+                        },
+                      },
+                      additionalProperties: false,
+                      required: ['docker', 'enabled', 'host', 'port'],
+                    },
                     enabled: {
                       type: 'boolean',
                     },
                   },
-                  required: ['docker', 'enabled', 'unit', 'requestsPerUnit', 'blacklist', 'whitelist'],
+                  required: ['docker', 'enabled', 'unit', 'requestsPerUnit', 'blacklist', 'whitelist', 'metrics'],
                   additionalProperties: false,
                 },
                 ssl: {
@@ -501,7 +531,7 @@ export default {
                   additionalProperties: false,
                 },
               },
-              required: ['docker', 'http', 'rateLimiter', 'ssl', 'maxHeapSizeInBytes', 'maxConnections', 'upstreams'],
+              required: ['docker', 'http', 'rateLimiter', 'ssl', 'maxHeapSizeInBytes', 'maxConnections', 'upstreams', 'metrics', 'admin'],
               additionalProperties: false,
             },
             api: {
@@ -588,19 +618,18 @@ export default {
                       type: 'boolean',
                     },
                     host: {
-                      type: 'string',
-                      minLength: 1,
-                      format: 'ipv4',
+                      $ref: '#/definitions/host',
                     },
                     port: {
                       $ref: '#/definitions/port',
                     },
-                    retention_secs: {
+                    retention: {
                       type: 'integer',
                       minimum: 0,
+                      description: 'How many seconds keep data if console is not connected',
                     },
                   },
-                  required: ['enabled', 'host', 'port', 'retention_secs'],
+                  required: ['enabled', 'host', 'port', 'retention'],
                   additionalProperties: false,
                 },
                 validatorSet: {
@@ -855,23 +884,7 @@ export default {
                   additionalProperties: false,
                 },
                 metrics: {
-                  description: 'Prometheus metrics',
-                  type: 'object',
-                  properties: {
-                    enabled: {
-                      type: 'boolean',
-                    },
-                    host: {
-                      type: 'string',
-                      minLength: 1,
-                      format: 'ipv4',
-                    },
-                    port: {
-                      $ref: '#/definitions/port',
-                    },
-                  },
-                  required: ['enabled', 'host', 'port'],
-                  additionalProperties: false,
+                  $ref: '#/definitions/enabledHostPort',
                 },
                 node: {
                   type: 'object',

diff --git a/packages/dashmate/src/config/generateEnvsFactory.js b/packages/dashmate/src/config/generateEnvsFactory.js
@@ -58,6 +58,10 @@ export default function generateEnvsFactory(configFile, homeDir, getConfigProfil
 
     if (config.get('platform.dapi.envoy.rateLimiter.enabled')) {
       dockerComposeFiles.push('docker-compose.rate_limiter.yml');
+
+      if (config.get('platform.dapi.envoy.rateLimiter.metrics.enabled')) {
+        dockerComposeFiles.push('docker-compose.rate_limiter.metrics.yml');
+      }
     }
 
     // we need this for compatibility with old configs
@@ -88,6 +92,7 @@ export default function generateEnvsFactory(configFile, homeDir, getConfigProfil
       ),
       DASHMATE_HELPER_DOCKER_IMAGE,
       PLATFORM_DRIVE_TENDERDASH_LOG_DIRECTORY_PATH: tenderdashLogDirectoryPath,
+      PLATFORM_DAPI_ENVOY_RATE_LIMITER_METRICS_DISABLED: config.get('platform.dapi.envoy.rateLimiter.metrics.enabled'),
       ...convertObjectToEnvs(config.getOptions()),
     };
   }

diff --git a/packages/dashmate/src/listr/tasks/setup/setupLocalPresetTaskFactory.js b/packages/dashmate/src/listr/tasks/setup/setupLocalPresetTaskFactory.js
@@ -223,7 +223,10 @@ export default function setupLocalPresetTaskFactory(
                   config.set('platform.drive.tenderdash.node.key', key);
 
                   config.set('platform.drive.abci.tokioConsole.port', config.get('platform.drive.abci.tokioConsole.port') + (i * 100));
+                  config.set('platform.dapi.envoy.admin.port', config.get('platform.dapi.envoy.admin.port') + (i * 100));
                   config.set('platform.dapi.envoy.http.port', config.get('platform.dapi.envoy.http.port') + (i * 100));
+                  config.set('platform.dapi.envoy.metrics.port', config.get('platform.dapi.envoy.metrics.port') + (i * 100));
+                  config.set('platform.dapi.envoy.rateLimiter.metrics.port', config.get('platform.dapi.envoy.rateLimiter.metrics.port') + (i * 100));
                   config.set('platform.drive.tenderdash.p2p.port', config.get('platform.drive.tenderdash.p2p.port') + (i * 100));
                   config.set('platform.drive.tenderdash.rpc.port', config.get('platform.drive.tenderdash.rpc.port') + (i * 100));
                   config.set('platform.drive.tenderdash.pprof.port', config.get('platform.drive.tenderdash.pprof.port') + (i * 100));