feat(infra): export prometheus metrics to s3

infra/modules/node-operator/main.tf

@@ -44,6 +44,9 @@ variable "config" {
       cpu_cores = number
       memory    = number
       disk      = number
+
+      s3_export          = optional(bool)
+      prom2parquet_image = optional(string)
     }))
 
     grafana = optional(object({
@@ -68,6 +71,7 @@ variable "config" {
 
 locals {
   create_ec2_instance_connect_endpoint = coalesce(var.config.create_ec2_instance_connect_endpoint, true)
+  prometheus_s3_export                 = try(var.config.prometheus.s3_export, false)
 }
 
 data "aws_region" "current" {}
@@ -134,22 +138,25 @@ module "db" {
 
     public_ip = false
 
-    ports = [
-      { port = local.db_primary_rpc_server_port, protocol = "udp", internal = true },
-      { port = local.db_secondary_rpc_server_port, protocol = "udp", internal = true },
-      { port = local.db_metrics_server_port, protocol = "tcp", internal = true },
-    ]
-
-    environment = {
-      PRIMARY_RPC_SERVER_PORT   = tostring(local.db_primary_rpc_server_port)
-      SECONDARY_RPC_SERVER_PORT = tostring(local.db_secondary_rpc_server_port)
-      METRICS_SERVER_PORT       = tostring(local.db_metrics_server_port)
-      ROCKSDB_DIR               = "/data"
-    }
+    containers = [{
+      image = var.config.db.image
+      ports = [
+        { port = local.db_primary_rpc_server_port, protocol = "udp", internal = true },
+        { port = local.db_secondary_rpc_server_port, protocol = "udp", internal = true },
+        { port = local.db_metrics_server_port, protocol = "tcp", internal = true },
+      ]
+
+      environment = {
+        PRIMARY_RPC_SERVER_PORT   = tostring(local.db_primary_rpc_server_port)
+        SECONDARY_RPC_SERVER_PORT = tostring(local.db_secondary_rpc_server_port)
+        METRICS_SERVER_PORT       = tostring(local.db_metrics_server_port)
+        ROCKSDB_DIR               = "/data"
+      }
 
-    secrets = {
-      SECRET_KEY = module.secret["ed25519_secret_key"]
-    }
+      secrets = {
+        SECRET_KEY = module.secret["ed25519_secret_key"]
+      }
+    }]
   })
 }
 
@@ -170,37 +177,41 @@ module "node" {
 
     public_ip = true
 
-    ports = [
-      { port = local.node_primary_rpc_server_port, protocol = "udp", internal = false },
-      { port = local.node_secondary_rpc_server_port, protocol = "udp", internal = false },
-      { port = local.node_metrics_server_port, protocol = "tcp", internal = true },
-    ]
-
-    environment = {
-      PRIMARY_RPC_SERVER_PORT            = tostring(local.node_primary_rpc_server_port)
-      SECONDARY_RPC_SERVER_PORT          = tostring(local.node_secondary_rpc_server_port)
-      METRICS_SERVER_PORT                = tostring(local.node_metrics_server_port)
-      DATABASE_RPC_SERVER_ADDRESS        = module.db.private_ip
-      DATABASE_PEER_ID                   = local.peer_id
-      DATABASE_PRIMARY_RPC_SERVER_PORT   = tostring(local.db_primary_rpc_server_port)
-      DATABASE_SECONDARY_RPC_SERVER_PORT = tostring(local.db_secondary_rpc_server_port)
-      SMART_CONTRACT_ADDRESS             = local.smart_contract_address
-    }
+    containers = [{
+      image = var.config.nodes[count.index].image
+      ports = [
+        { port = local.node_primary_rpc_server_port, protocol = "udp", internal = false },
+        { port = local.node_secondary_rpc_server_port, protocol = "udp", internal = false },
+        { port = local.node_metrics_server_port, protocol = "tcp", internal = true },
+      ]
+
+      environment = {
+        PRIMARY_RPC_SERVER_PORT            = tostring(local.node_primary_rpc_server_port)
+        SECONDARY_RPC_SERVER_PORT          = tostring(local.node_secondary_rpc_server_port)
+        METRICS_SERVER_PORT                = tostring(local.node_metrics_server_port)
+        DATABASE_RPC_SERVER_ADDRESS        = module.db.private_ip
+        DATABASE_PEER_ID                   = local.peer_id
+        DATABASE_PRIMARY_RPC_SERVER_PORT   = tostring(local.db_primary_rpc_server_port)
+        DATABASE_SECONDARY_RPC_SERVER_PORT = tostring(local.db_secondary_rpc_server_port)
+        SMART_CONTRACT_ADDRESS             = local.smart_contract_address
+      }
 
-    secrets = merge({
-      SECRET_KEY                    = module.secret["ed25519_secret_key"]
-      SMART_CONTRACT_ENCRYPTION_KEY = module.secret["smart_contract_encryption_key"]
-      RPC_PROVIDER_URL              = module.secret["rpc_provider_url"]
-      },
-      # configure pk only for the primary node
-      count.index != 0 ? {} : {
-        SMART_CONTRACT_SIGNER_PRIVATE_KEY = module.secret["ecdsa_private_key"]
-    })
+      secrets = merge({
+        SECRET_KEY                    = module.secret["ed25519_secret_key"]
+        SMART_CONTRACT_ENCRYPTION_KEY = module.secret["smart_contract_encryption_key"]
+        RPC_PROVIDER_URL              = module.secret["rpc_provider_url"]
+        },
+        # configure pk only for the primary node
+        count.index != 0 ? {} : {
+          SMART_CONTRACT_SIGNER_PRIVATE_KEY = module.secret["ecdsa_private_key"]
+      })
+    }]
   })
 }
 
 locals {
   prometheus_port        = 3000
+  prom2parquet_port      = 4000
   prometheus_domain_name = try("prometheus.${local.region_prefix}.${var.config.route53_zone.name}", null)
 }
 
@@ -250,27 +261,44 @@ module "prometheus" {
 
     public_ip = false
 
-    ports = [
-      { port = local.prometheus_port, protocol = "tcp", internal = true },
-    ]
-
-    environment = {}
-    secrets = {
-      CONFIG     = module.prometheus_config[0]
-      WEB_CONFIG = module.prometheus_web_config[0]
-    }
-
-    entry_point = ["/bin/sh", "-c"]
-    command = [<<-CMD
-      printenv CONFIG > /tmp/prometheus.yml && \
-      printenv WEB_CONFIG > /tmp/web.yml && \
-      exec /bin/prometheus \
-      --config.file=/tmp/prometheus.yml \
-      --web.config.file=/tmp/web.yml \
-      --web.listen-address=:${local.prometheus_port} \
-      --storage.tsdb.path=/data
-    CMD
-    ]
+    containers = concat(
+      [{
+        name  = "prometheus"
+        image = var.config.prometheus.image
+        ports = [
+          { port = local.prometheus_port, protocol = "tcp", internal = true },
+        ]
+
+        environment = {}
+        secrets = {
+          CONFIG     = module.prometheus_config[0]
+          WEB_CONFIG = module.prometheus_web_config[0]
+        }
+
+        entry_point = ["/bin/sh", "-c"]
+        command = [<<-CMD
+          printenv CONFIG > /tmp/prometheus.yml && \
+          printenv WEB_CONFIG > /tmp/web.yml && \
+          exec /bin/prometheus \
+          --config.file=/tmp/prometheus.yml \
+          --web.config.file=/tmp/web.yml \
+          --web.listen-address=:${local.prometheus_port} \
+          --storage.tsdb.path=/data
+        CMD
+        ]
+      }],
+      local.prometheus_s3_export ? [{
+        name      = "prom2parquet"
+        image     = var.config.prometheus.prom2parquet_image
+        essential = false
+        ports = [
+          { port = local.prom2parquet_port, protocol = "tcp", internal = true },
+        ]
+
+        environment = {}
+        secrets     = {}
+      }] : []
+    )
   })
 }
 
@@ -318,27 +346,30 @@ module "grafana" {
 
     public_ip = false
 
-    ports = [
-      { port = local.grafana_port, protocol = "tcp", internal = true },
-    ]
+    containers = [{
+      image = var.config.grafana.image
+      ports = [
+        { port = local.grafana_port, protocol = "tcp", internal = true },
+      ]
 
-    environment = {
-      GF_SERVER_HTTP_PORT    = tostring(local.grafana_port)
-      GF_PATHS_DATA          = "/data"
-      GF_SECURITY_ADMIN_USER = "admin"
-    }
+      environment = {
+        GF_SERVER_HTTP_PORT    = tostring(local.grafana_port)
+        GF_PATHS_DATA          = "/data"
+        GF_SECURITY_ADMIN_USER = "admin"
+      }
 
-    secrets = {
-      GF_SECURITY_ADMIN_PASSWORD   = module.secret["grafana_admin_password"]
-      PROMETHEUS_DATASOURCE_CONFIG = module.grafana_prometheus_datasource_config[0]
-    }
+      secrets = {
+        GF_SECURITY_ADMIN_PASSWORD   = module.secret["grafana_admin_password"]
+        PROMETHEUS_DATASOURCE_CONFIG = module.grafana_prometheus_datasource_config[0]
+      }
 
-    entry_point = ["/bin/sh", "-c"]
-    command = [<<-CMD
-      printenv PROMETHEUS_DATASOURCE_CONFIG > /etc/grafana/provisioning/datasources/prometheus.yaml && \
-      exec /run.sh
-    CMD
-    ]
+      entry_point = ["/bin/sh", "-c"]
+      command = [<<-CMD
+        printenv PROMETHEUS_DATASOURCE_CONFIG > /etc/grafana/provisioning/datasources/prometheus.yaml && \
+        exec /run.sh
+      CMD
+      ]
+    }]
   })
 }
 

infra/modules/service/main.tf

@@ -19,21 +19,26 @@
       availability_zone = string
     })
 
-    ports = list(object({
-      port     = number
-      protocol = string
-      internal = bool
+    containers = list(object({
+      name      = optional(string)
+      image     = string
+      essential = optional(bool)
+      ports = list(object({
+        port     = number
+        protocol = string
+        internal = bool
+      }))
+
+      environment = map(string)
+
+      secrets = map(object({
+        ssm_parameter_arn = string
+        version           = number
+      }))
+
+      entry_point = optional(list(string))
+      command     = optional(list(string))
     }))
-
-    environment = map(string)
-
-    secrets = map(object({
-      ssm_parameter_arn = string
-      version           = number
-    }))
-
-    entry_point = optional(list(string))
-    command     = optional(list(string))
   })
 }
 
@@ -68,6 +73,8 @@
     "x86-2cpu-4mem-normal"  = "c5a.large"
     "x86-8cpu-16mem-normal" = "c5a.2xlarge"
   }["${local.cpu_arch}-${var.config.cpu_cores}cpu-${var.config.memory}mem-${local.cpu_burst ? "burst" : "normal"}"]
+
+  secrets = concat(var.config.containers[*].secrets)
 }
 
 resource "aws_security_group" "this" {
@@ -77,7 +84,7 @@
 
 resource "aws_vpc_security_group_ingress_rule" "this" {
   for_each = {
-    for p in var.config.ports :
+    for p in concat(var.config.containers[*].ports) :
     "${p.port}:${p.protocol}:${p.internal ? "internal" : "external"}" => p
   }
 
@@ -143,7 +150,7 @@
 }
 
 resource "aws_iam_role_policy" "ssm" {
-  count = length(var.config.secrets) > 0 ? 1 : 0
+  count = length(local.secrets) > 0 ? 1 : 0
   name  = "ecs-exec-ssm-kms"
   role  = aws_iam_role.this.id
   policy = jsonencode({
@@ -152,7 +159,7 @@
       {
         Effect   = "Allow",
         Action   = ["ssm:GetParameter", "ssm:GetParameters", "ssm:GetParametersByPath"],
-        Resource = [for s in var.config.secrets : s.ssm_parameter_arn]
+        Resource = [for s in local.secrets : s.ssm_parameter_arn]
       },
       { Effect = "Allow", Action = ["kms:Decrypt"], Resource = "*" }
     ]
@@ -234,35 +241,35 @@
   network_mode             = "host"
   execution_role_arn       = aws_iam_role.this.arn
 
-  container_definitions = jsonencode([
+  container_definitions = jsonencode([for i in range(length(var.config.containers)) :
     {
-      name       = var.config.name
-      image      = var.config.image
+      name       = coalesce(var.config.containers[i].name, var.config.name)
+      image      = var.config.containers[i].image
       user       = "1001:1001"
-      entryPoint = var.config.entry_point
-      command    = var.config.command
+      entryPoint = var.config.containers[i].entry_point
+      command    = var.config.containers[i].command
       # Make sure that task doesn't require all the available memory of the instance.
       # Usually around 200-300 MBs are being used by the OS.
       # The task will be able to use more than the specified amount.
-      memoryReservation = var.config.memory * 1024 / 2
-      essential         = true
-      portMappings = [for p in var.config.ports : {
+      memoryReservation = i == 0 ? var.config.memory * 1024 / 2 : 0
+      essential         = coalesce(var.config.containers[i].essencial, true)
+      portMappings = [for p in var.config.containers[i].ports : {
         containerPort = p.port
         hostPort      = p.port
         protocol      = p.protocol
       }]
       # Specify secret versions as separate environment variables in order to force
       # task definition updates when secrets change.
       environment = concat(
-        [for k in sort(keys(var.config.environment)) : {
+        [for k in sort(keys(var.config.containers[i].environment)) : {
           name  = k
-          value = var.config.environment[k]
+          value = var.config.containers[i].environment[k]
         }],
-        [for k, v in var.config.secrets : {
+        [for k, v in var.config.containers[i].secrets : {
           name  = "${k}_VERSION"
           value = tostring(v.version)
       }])
-      secrets = [for k, v in var.config.secrets : {
+      secrets = [for k, v in var.config.containers[i].secrets : {
         name      = k
         valueFrom = v.ssm_parameter_arn
       }]

infra/testnet/main.tf

@@ -90,6 +90,9 @@ locals {
     cpu_cores = 2
     memory    = 4
     disk      = 20
+
+    s3_export = true
+    prom2parquet_image = "ghcr.io/walletconnect/prom2parquet:sha-51efb35"
   }
 
   grafana_config = {