diff --git a/infra/modules/node-operator/main.tf b/infra/modules/node-operator/main.tf index 1edaeefc..55adeaf3 100644 --- a/infra/modules/node-operator/main.tf +++ b/infra/modules/node-operator/main.tf @@ -44,6 +44,10 @@ variable "config" { cpu_cores = number memory = number disk = number + + prom2parquet_image = optional(string) + s3_bucket = optional(string) + s3_metrics_prefix = optional(string) })) grafana = optional(object({ @@ -68,6 +72,7 @@ variable "config" { locals { create_ec2_instance_connect_endpoint = coalesce(var.config.create_ec2_instance_connect_endpoint, true) + prometheus_s3_export = try(var.config.prometheus.prom2parquet_image, null) != null } data "aws_region" "current" {} @@ -134,22 +139,25 @@ module "db" { public_ip = false - ports = [ - { port = local.db_primary_rpc_server_port, protocol = "udp", internal = true }, - { port = local.db_secondary_rpc_server_port, protocol = "udp", internal = true }, - { port = local.db_metrics_server_port, protocol = "tcp", internal = true }, - ] - - environment = { - PRIMARY_RPC_SERVER_PORT = tostring(local.db_primary_rpc_server_port) - SECONDARY_RPC_SERVER_PORT = tostring(local.db_secondary_rpc_server_port) - METRICS_SERVER_PORT = tostring(local.db_metrics_server_port) - ROCKSDB_DIR = "/data" - } + containers = [{ + image = var.config.db.image + ports = [ + { port = local.db_primary_rpc_server_port, protocol = "udp", internal = true }, + { port = local.db_secondary_rpc_server_port, protocol = "udp", internal = true }, + { port = local.db_metrics_server_port, protocol = "tcp", internal = true }, + ] + + environment = { + PRIMARY_RPC_SERVER_PORT = tostring(local.db_primary_rpc_server_port) + SECONDARY_RPC_SERVER_PORT = tostring(local.db_secondary_rpc_server_port) + METRICS_SERVER_PORT = tostring(local.db_metrics_server_port) + ROCKSDB_DIR = "/data" + } - secrets = { - SECRET_KEY = module.secret["ed25519_secret_key"] - } + secrets = { + SECRET_KEY = module.secret["ed25519_secret_key"] + } + }] }) } @@ -170,37 +178,41 @@ module "node" { public_ip = true - ports = [ - { port = local.node_primary_rpc_server_port, protocol = "udp", internal = false }, - { port = local.node_secondary_rpc_server_port, protocol = "udp", internal = false }, - { port = local.node_metrics_server_port, protocol = "tcp", internal = true }, - ] - - environment = { - PRIMARY_RPC_SERVER_PORT = tostring(local.node_primary_rpc_server_port) - SECONDARY_RPC_SERVER_PORT = tostring(local.node_secondary_rpc_server_port) - METRICS_SERVER_PORT = tostring(local.node_metrics_server_port) - DATABASE_RPC_SERVER_ADDRESS = module.db.private_ip - DATABASE_PEER_ID = local.peer_id - DATABASE_PRIMARY_RPC_SERVER_PORT = tostring(local.db_primary_rpc_server_port) - DATABASE_SECONDARY_RPC_SERVER_PORT = tostring(local.db_secondary_rpc_server_port) - SMART_CONTRACT_ADDRESS = local.smart_contract_address - } + containers = [{ + image = var.config.nodes[count.index].image + ports = [ + { port = local.node_primary_rpc_server_port, protocol = "udp", internal = false }, + { port = local.node_secondary_rpc_server_port, protocol = "udp", internal = false }, + { port = local.node_metrics_server_port, protocol = "tcp", internal = true }, + ] + + environment = { + PRIMARY_RPC_SERVER_PORT = tostring(local.node_primary_rpc_server_port) + SECONDARY_RPC_SERVER_PORT = tostring(local.node_secondary_rpc_server_port) + METRICS_SERVER_PORT = tostring(local.node_metrics_server_port) + DATABASE_RPC_SERVER_ADDRESS = module.db.private_ip + DATABASE_PEER_ID = local.peer_id + DATABASE_PRIMARY_RPC_SERVER_PORT = tostring(local.db_primary_rpc_server_port) + DATABASE_SECONDARY_RPC_SERVER_PORT = tostring(local.db_secondary_rpc_server_port) + SMART_CONTRACT_ADDRESS = local.smart_contract_address + } - secrets = merge({ - SECRET_KEY = module.secret["ed25519_secret_key"] - SMART_CONTRACT_ENCRYPTION_KEY = module.secret["smart_contract_encryption_key"] - RPC_PROVIDER_URL = module.secret["rpc_provider_url"] - }, - # configure pk only for the primary node - count.index != 0 ? {} : { - SMART_CONTRACT_SIGNER_PRIVATE_KEY = module.secret["ecdsa_private_key"] - }) + secrets = merge({ + SECRET_KEY = module.secret["ed25519_secret_key"] + SMART_CONTRACT_ENCRYPTION_KEY = module.secret["smart_contract_encryption_key"] + RPC_PROVIDER_URL = module.secret["rpc_provider_url"] + }, + # configure pk only for the primary node + count.index != 0 ? {} : { + SMART_CONTRACT_SIGNER_PRIVATE_KEY = module.secret["ecdsa_private_key"] + }) + }] }) } locals { prometheus_port = 3000 + prom2parquet_port = 4000 prometheus_domain_name = try("prometheus.${local.region_prefix}.${var.config.route53_zone.name}", null) } @@ -219,6 +231,10 @@ module "prometheus_config" { targets = ["${module.node[0].private_ip}:${local.node_metrics_server_port}"] }] }] + remote_write = local.prometheus_s3_export ? [{ + url = "http://localhost:${local.prom2parquet_port}/receive" + remote_timeout = "30s" + }] : [] }) } @@ -250,27 +266,56 @@ module "prometheus" { public_ip = false - ports = [ - { port = local.prometheus_port, protocol = "tcp", internal = true }, - ] - - environment = {} - secrets = { - CONFIG = module.prometheus_config[0] - WEB_CONFIG = module.prometheus_web_config[0] - } - - entry_point = ["/bin/sh", "-c"] - command = [<<-CMD - printenv CONFIG > /tmp/prometheus.yml && \ - printenv WEB_CONFIG > /tmp/web.yml && \ - exec /bin/prometheus \ - --config.file=/tmp/prometheus.yml \ - --web.config.file=/tmp/web.yml \ - --web.listen-address=:${local.prometheus_port} \ - --storage.tsdb.path=/data - CMD - ] + containers = concat( + [{ + name = "prometheus" + image = var.config.prometheus.image + ports = [ + { port = local.prometheus_port, protocol = "tcp", internal = true }, + ] + + environment = {} + secrets = { + CONFIG = module.prometheus_config[0] + WEB_CONFIG = module.prometheus_web_config[0] + } + + entry_point = ["/bin/sh", "-c"] + command = [<<-CMD + printenv CONFIG > /tmp/prometheus.yml && \ + printenv WEB_CONFIG > /tmp/web.yml && \ + exec /bin/prometheus \ + --config.file=/tmp/prometheus.yml \ + --web.config.file=/tmp/web.yml \ + --web.listen-address=:${local.prometheus_port} \ + --storage.tsdb.path=/data + CMD + ] + }], + local.prometheus_s3_export ? [{ + name = "prom2parquet" + image = var.config.prometheus.prom2parquet_image + essential = false + ports = [ + { port = local.prom2parquet_port, protocol = "tcp", internal = true }, + ] + + environment = { + AWS_REGION = local.region + } + secrets = {} + + entry_point = ["/prom2parquet"] + command = [ + "--backend", "s3", + "--backend-root", "${var.config.prometheus.s3_bucket}", + "--prefix", "${var.config.prometheus.s3_metrics_prefix}/${local.region}", + "--server-port", tostring(local.prom2parquet_port), + ] + }] : [] + ) + + s3_buckets = var.config.prometheus.s3_bucket == null ? [] : [var.config.prometheus.s3_bucket] }) } @@ -318,27 +363,30 @@ module "grafana" { public_ip = false - ports = [ - { port = local.grafana_port, protocol = "tcp", internal = true }, - ] + containers = [{ + image = var.config.grafana.image + ports = [ + { port = local.grafana_port, protocol = "tcp", internal = true }, + ] - environment = { - GF_SERVER_HTTP_PORT = tostring(local.grafana_port) - GF_PATHS_DATA = "/data" - GF_SECURITY_ADMIN_USER = "admin" - } + environment = { + GF_SERVER_HTTP_PORT = tostring(local.grafana_port) + GF_PATHS_DATA = "/data" + GF_SECURITY_ADMIN_USER = "admin" + } - secrets = { - GF_SECURITY_ADMIN_PASSWORD = module.secret["grafana_admin_password"] - PROMETHEUS_DATASOURCE_CONFIG = module.grafana_prometheus_datasource_config[0] - } + secrets = { + GF_SECURITY_ADMIN_PASSWORD = module.secret["grafana_admin_password"] + PROMETHEUS_DATASOURCE_CONFIG = module.grafana_prometheus_datasource_config[0] + } - entry_point = ["/bin/sh", "-c"] - command = [<<-CMD - printenv PROMETHEUS_DATASOURCE_CONFIG > /etc/grafana/provisioning/datasources/prometheus.yaml && \ - exec /run.sh - CMD - ] + entry_point = ["/bin/sh", "-c"] + command = [<<-CMD + printenv PROMETHEUS_DATASOURCE_CONFIG > /etc/grafana/provisioning/datasources/prometheus.yaml && \ + exec /run.sh + CMD + ] + }] }) } diff --git a/infra/modules/service/main.tf b/infra/modules/service/main.tf index 8a95e729..6aa1ff68 100644 --- a/infra/modules/service/main.tf +++ b/infra/modules/service/main.tf @@ -19,21 +19,28 @@ variable "config" { availability_zone = string }) - ports = list(object({ - port = number - protocol = string - internal = bool + containers = list(object({ + name = optional(string) + image = string + essential = optional(bool) + ports = list(object({ + port = number + protocol = string + internal = bool + })) + + environment = map(string) + + secrets = map(object({ + ssm_parameter_arn = string + version = number + })) + + entry_point = optional(list(string)) + command = optional(list(string)) })) - environment = map(string) - - secrets = map(object({ - ssm_parameter_arn = string - version = number - })) - - entry_point = optional(list(string)) - command = optional(list(string)) + s3_buckets = optional(list(string)) }) } @@ -68,6 +75,9 @@ locals { "x86-2cpu-4mem-normal" = "c5a.large" "x86-8cpu-16mem-normal" = "c5a.2xlarge" }["${local.cpu_arch}-${var.config.cpu_cores}cpu-${var.config.memory}mem-${local.cpu_burst ? "burst" : "normal"}"] + + secrets = merge(var.config.containers[*].secrets...) + s3_buckets = coalesce(var.config.s3_buckets, []) } resource "aws_security_group" "this" { @@ -77,7 +87,7 @@ resource "aws_security_group" "this" { resource "aws_vpc_security_group_ingress_rule" "this" { for_each = { - for p in var.config.ports : + for p in flatten(var.config.containers[*].ports) : "${p.port}:${p.protocol}:${p.internal ? "internal" : "external"}" => p } @@ -143,7 +153,7 @@ resource "aws_iam_role_policy_attachment" "this" { } resource "aws_iam_role_policy" "ssm" { - count = length(var.config.secrets) > 0 ? 1 : 0 + count = length(local.secrets) > 0 ? 1 : 0 name = "ecs-exec-ssm-kms" role = aws_iam_role.this.id policy = jsonencode({ @@ -152,7 +162,7 @@ resource "aws_iam_role_policy" "ssm" { { Effect = "Allow", Action = ["ssm:GetParameter", "ssm:GetParameters", "ssm:GetParametersByPath"], - Resource = [for s in var.config.secrets : s.ssm_parameter_arn] + Resource = [for s in local.secrets : s.ssm_parameter_arn] }, { Effect = "Allow", Action = ["kms:Decrypt"], Resource = "*" } ] @@ -228,25 +238,59 @@ resource "aws_cloudwatch_log_group" "this" { name = "/ecs/${var.config.name}" } +resource "aws_iam_role" "task" { + count = length(local.s3_buckets) > 0 ? 1 : 0 + name = "${local.region}-${var.config.name}-task" + assume_role_policy = jsonencode({ + Version = "2012-10-17", + Statement = [ + { Effect = "Allow", Principal = { Service = "ecs-tasks.amazonaws.com" }, Action = "sts:AssumeRole" }, + ] + }) +} + +resource "aws_iam_role_policy" "s3" { + count = length(local.s3_buckets) > 0 ? 1 : 0 + name = "ecs-exec-s3" + role = aws_iam_role.task[0].id + policy = jsonencode({ + Version = "2012-10-17", + Statement = [ + { + Effect = "Allow", + Action = ["s3:ListBucket"], + Resource = [for bucket in local.s3_buckets : "arn:aws:s3:::${bucket}"] + }, + { + Effect = "Allow", + Action = ["s3:PutObject"], + Resource = [for bucket in local.s3_buckets : "arn:aws:s3:::${bucket}/*"] + }, + ] + }) +} + resource "aws_ecs_task_definition" "this" { family = var.config.name requires_compatibilities = ["EC2"] network_mode = "host" execution_role_arn = aws_iam_role.this.arn + task_role_arn = try(aws_iam_role.task[0].arn, null) - container_definitions = jsonencode([ + container_definitions = jsonencode([for i in range(length(var.config.containers)) : { - name = var.config.name - image = var.config.image + name = coalesce(var.config.containers[i].name, var.config.name) + image = var.config.containers[i].image user = "1001:1001" - entryPoint = var.config.entry_point - command = var.config.command - # Make sure that task doesn't require all the available memory of the instance. + entryPoint = var.config.containers[i].entry_point + command = var.config.containers[i].command + # Make sure that the primary task doesn't require all the available memory of the instance. # Usually around 200-300 MBs are being used by the OS. - # The task will be able to use more than the specified amount. - memoryReservation = var.config.memory * 1024 / 2 - essential = true - portMappings = [for p in var.config.ports : { + # For sidecards reserve the minimum possible amount (6 MB). + # The tasks will be able to use more than the specified amount. + memoryReservation = i == 0 ? var.config.memory * 1024 / 2 : 6 + essential = coalesce(var.config.containers[i].essential, true) + portMappings = [for p in var.config.containers[i].ports : { containerPort = p.port hostPort = p.port protocol = p.protocol @@ -254,15 +298,15 @@ resource "aws_ecs_task_definition" "this" { # Specify secret versions as separate environment variables in order to force # task definition updates when secrets change. environment = concat( - [for k in sort(keys(var.config.environment)) : { + [for k in sort(keys(var.config.containers[i].environment)) : { name = k - value = var.config.environment[k] + value = var.config.containers[i].environment[k] }], - [for k, v in var.config.secrets : { + [for k, v in var.config.containers[i].secrets : { name = "${k}_VERSION" value = tostring(v.version) }]) - secrets = [for k, v in var.config.secrets : { + secrets = [for k, v in var.config.containers[i].secrets : { name = k valueFrom = v.ssm_parameter_arn }] diff --git a/infra/testnet/main.tf b/infra/testnet/main.tf index 73ecb599..2ae7ec40 100644 --- a/infra/testnet/main.tf +++ b/infra/testnet/main.tf @@ -90,6 +90,10 @@ locals { cpu_cores = 2 memory = 4 disk = 20 + + prom2parquet_image = "ghcr.io/walletconnect/prom2parquet:sha-1fee3c3" + s3_bucket = "walletconnect.data-lake.staging" + s3_metrics_prefix = "wcn/testnet" } grafana_config = {