Skip to content
Deploy Production

Deploy Production #474

Sign in to view logs

Deploy Production

Deploy Production #474

Workflow file for this run

.github/workflows/prod.yml at 8f0505e
name: Deploy Production
permissions:
contents: read
deployments: write
id-token: write
security-events: write # Required for Trivy SARIF upload in test workflow
concurrency:
group: robosystems-deploy
cancel-in-progress: false
on:
workflow_dispatch:
inputs:
graph_container_refresh:
description: "Refresh graph containers via SSM after deploy"
required: false
type: boolean
default: true
graph_refresh_max_wait_minutes:
description: "Minutes to wait for in-flight destructive ops (materialization, SEC stage, ext mat) before cycling"
required: false
type: string
default: "30"
graph_refresh_force_ignore_busy:
description: "Skip busy-counter check and cycle graph containers immediately (emergency override)"
required: false
type: boolean
default: false
jobs:
runner:
runs-on: ubuntu-latest
outputs:
runners_available: ${{ steps.select.outputs.runners_available }}
runner_type: ${{ steps.select.outputs.runner_type }}
runner_config: ${{ steps.select.outputs.runner_config }}
steps:
- name: Checkout
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
with:
repository: ${{ github.repository }}
ref: ${{ github.ref }}
token: ${{ github.token }}
- name: Select runner
id: select
uses: ./.github/actions/select-runner
with:
runner_labels: ${{ vars.RUNNER_LABELS || 'github-hosted' }}
runner_scope: ${{ vars.RUNNER_SCOPE || 'both' }}
github_token: ${{ github.token }}
stack-config:
needs: [runner]
runs-on: ${{ fromJSON(needs.runner.outputs.runner_config) }}
permissions:
id-token: write
contents: read
outputs:
vpc_stack: ${{ steps.load.outputs.vpc_stack }}
s3_stack: ${{ steps.load.outputs.s3_stack }}
postgres_stack: ${{ steps.load.outputs.postgres_stack }}
valkey_stack: ${{ steps.load.outputs.valkey_stack }}
api_stack: ${{ steps.load.outputs.api_stack }}
graph_infra_stack: ${{ steps.load.outputs.graph_infra_stack }}
graph_volumes_stack: ${{ steps.load.outputs.graph_volumes_stack }}
prometheus_stack: ${{ steps.load.outputs.prometheus_stack }}
grafana_stack: ${{ steps.load.outputs.grafana_stack }}
bastion_stack: ${{ steps.load.outputs.bastion_stack }}
waf_stack: ${{ steps.load.outputs.waf_stack }}
audit_stack: ${{ steps.load.outputs.audit_stack }}
cloudtrail_stack: ${{ steps.load.outputs.cloudtrail_stack }}
security_stack: ${{ steps.load.outputs.security_stack }}
dagster_stack: ${{ steps.load.outputs.dagster_stack }}
worker_stack: ${{ steps.load.outputs.worker_stack }}
opensearch_stack: ${{ steps.load.outputs.opensearch_stack }}
domain_name_root: ${{ steps.load.outputs.domain_name_root }}
graph_ami_id: ${{ steps.ami.outputs.ami_id }}
steps:
- name: Checkout
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
with:
repository: ${{ github.repository }}
ref: ${{ github.ref }}
token: ${{ github.token }}
- name: Configure AWS credentials
uses: aws-actions/configure-aws-credentials@e6de054238d6b7531b4efff3b6587d9aade6a06c # v6
with:
role-to-assume: ${{ vars.AWS_ROLE_ARN }}
aws-region: ${{ vars.AWS_REGION || 'us-east-1' }}
- name: Get Graph AMI ID
id: ami
uses: ./.github/actions/get-graph-ami
with:
environment: prod
- name: Install PyYAML
run: pip install --user pyyaml
- name: Load Stack Configuration
id: load
shell: bash
run: |
chmod +x bin/tools/stack-config.sh
ENV="production"
# Load all stack names from config
echo "vpc_stack=$(./bin/tools/stack-config.sh $ENV vpc)" >> $GITHUB_OUTPUT
echo "s3_stack=$(./bin/tools/stack-config.sh $ENV s3)" >> $GITHUB_OUTPUT
echo "postgres_stack=$(./bin/tools/stack-config.sh $ENV postgres)" >> $GITHUB_OUTPUT
echo "valkey_stack=$(./bin/tools/stack-config.sh $ENV valkey)" >> $GITHUB_OUTPUT
echo "api_stack=$(./bin/tools/stack-config.sh $ENV api)" >> $GITHUB_OUTPUT
echo "graph_infra_stack=$(./bin/tools/stack-config.sh $ENV graph.infra)" >> $GITHUB_OUTPUT
echo "graph_volumes_stack=$(./bin/tools/stack-config.sh $ENV graph.volumes)" >> $GITHUB_OUTPUT
echo "prometheus_stack=$(./bin/tools/stack-config.sh $ENV monitoring.prometheus)" >> $GITHUB_OUTPUT
echo "grafana_stack=$(./bin/tools/stack-config.sh $ENV monitoring.grafana)" >> $GITHUB_OUTPUT
echo "bastion_stack=$(./bin/tools/stack-config.sh $ENV bastion)" >> $GITHUB_OUTPUT
echo "waf_stack=$(./bin/tools/stack-config.sh $ENV waf)" >> $GITHUB_OUTPUT
echo "audit_stack=$(./bin/tools/stack-config.sh $ENV audit)" >> $GITHUB_OUTPUT
echo "cloudtrail_stack=$(./bin/tools/stack-config.sh $ENV cloudtrail)" >> $GITHUB_OUTPUT
echo "security_stack=$(./bin/tools/stack-config.sh $ENV security)" >> $GITHUB_OUTPUT
echo "dagster_stack=$(./bin/tools/stack-config.sh $ENV dagster)" >> $GITHUB_OUTPUT
echo "worker_stack=$(./bin/tools/stack-config.sh $ENV workers)" >> $GITHUB_OUTPUT
echo "opensearch_stack=$(./bin/tools/stack-config.sh $ENV opensearch)" >> $GITHUB_OUTPUT
# Domain configuration - leave empty for isolated deployment (no custom domains)
echo "domain_name_root=${{ vars.API_DOMAIN_NAME_ROOT || '' }}" >> $GITHUB_OUTPUT
test:
needs: [runner]
uses: ./.github/workflows/test.yml
with:
runner_config: ${{ needs.runner.outputs.runner_config }}
secrets: inherit
build:
needs: [runner, create-deployment]
uses: ./.github/workflows/build.yml
with:
environment: ${{ vars.ENVIRONMENT_PROD || 'prod' }}
aws_region: ${{ vars.AWS_REGION || 'us-east-1' }}
ecr_repository: ${{ vars.AWS_ECR_REPOSITORY || 'robosystems' }}
# Docker Hub publishing: only when explicitly enabled AND deploying from a version tag or release branch
# Set DOCKERHUB_PUBLISHING_ENABLED to 'true' in GitHub variables when ready to publish
# Supports: refs/tags/v*.*.* (releases) and refs/heads/release/* (hotfixes)
publish_to_dockerhub: ${{ vars.DOCKERHUB_PUBLISHING_ENABLED == 'true' && (startsWith(github.ref, 'refs/tags/v') || startsWith(github.ref, 'refs/heads/release/')) }}
secrets:
DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_USERNAME }}
DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_TOKEN }}
create-deployment:
needs: [runner]
runs-on: ${{ fromJSON(needs.runner.outputs.runner_config) }}
# Every deploy-* job fans in through this one, so the production
# environment's protection rules gate the whole run here, once. It runs
# first so approval is immediate; tests run during the wait, build after.
environment: production
outputs:
deployment_id: ${{ steps.deployment.outputs.deployment_id }}
steps:
- name: Create GitHub Deployment
id: deployment
uses: chrnorm/deployment-action@500aa6a23c81ffa1acf71072aee3cfa2cc2e556a # v2
with:
token: ${{ github.token }}
environment: production
description: "Production Deployment Created"
- name: Update Deployment Status
uses: chrnorm/deployment-status@6df8d036fd2fee9eb82936733953da1f8382b41e # v2
with:
token: ${{ github.token }}
state: in_progress
deployment-id: ${{ steps.deployment.outputs.deployment_id }}
description: "Production Deployment In Progress"
deploy-vpc:
needs: [runner, create-deployment, stack-config, test]
uses: ./.github/workflows/deploy-vpc.yml
with:
# Stack & Repository Configuration
stack_name: ${{ needs.stack-config.outputs.vpc_stack }}
cloudtrail_stack_name: ${{ needs.stack-config.outputs.cloudtrail_stack }}
environment: shared
# GHA Runner Configuration
runner_config: ${{ needs.runner.outputs.runner_config }}
# AWS Configuration
aws_region: ${{ vars.AWS_REGION || 'us-east-1' }}
# VPC Configuration
endpoint_mode: ${{ vars.VPC_ENDPOINT_MODE || 'none' }}
max_availability_zones: ${{ vars.VPC_MAX_AVAILABILITY_ZONES || '2' }}
vpc_second_octet: ${{ vars.VPC_SECOND_OCTET || '0' }}
# VPC Flow Logs Configuration (VPC-level, not environment-specific)
vpc_flow_logs_enabled: ${{ vars.VPC_FLOW_LOGS_ENABLED || 'false' }}
vpc_flow_logs_retention_days: ${{ vars.VPC_FLOW_LOGS_RETENTION_DAYS || '30' }}
vpc_flow_logs_traffic_type: ${{ vars.VPC_FLOW_LOGS_TRAFFIC_TYPE || 'REJECT' }}
# CloudTrail Configuration (account-level, not environment-specific)
cloudtrail_enabled: ${{ vars.CLOUDTRAIL_ENABLED || 'false' }}
cloudtrail_log_retention_days: ${{ vars.CLOUDTRAIL_LOG_RETENTION_DAYS || '90' }}
cloudtrail_data_events_enabled: ${{ vars.CLOUDTRAIL_DATA_EVENTS_ENABLED || 'false' }}
# Security Baseline Configuration (account-level, not environment-specific)
security_enabled: ${{ vars.SECURITY_ENABLED || 'false' }}
security_stack_name: ${{ needs.stack-config.outputs.security_stack }}
security_config_enabled: ${{ vars.SECURITY_CONFIG_ENABLED || 'false' }}
secrets: inherit
deploy-s3:
needs: [runner, create-deployment, stack-config, test]
uses: ./.github/workflows/deploy-s3.yml
with:
# Stack & Repository Configuration
stack_name: ${{ needs.stack-config.outputs.s3_stack }}
environment: ${{ vars.ENVIRONMENT_PROD || 'prod' }}
# GHA Runner Configuration
runner_config: ${{ needs.runner.outputs.runner_config }}
# AWS Configuration
aws_region: ${{ vars.AWS_REGION || 'us-east-1' }}
# Namespace Configuration (always use account ID for globally unique bucket names)
namespace: ${{ vars.AWS_ACCOUNT_ID }}
# Domain Configuration
# Leave PUBLIC_DOMAIN_NAME_PROD empty for default CloudFront domain (isolated deployment)
public_domain_name: ${{ vars.PUBLIC_DOMAIN_NAME_PROD || '' }}
domain_name_root: ${{ vars.API_DOMAIN_NAME_ROOT || '' }}
secrets: inherit
package-scripts:
needs: [runner, deploy-s3]
runs-on: ${{ fromJSON(needs.runner.outputs.runner_config) }}
permissions:
id-token: write
contents: read
steps:
- name: Checkout
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
with:
repository: ${{ github.repository }}
ref: ${{ github.ref }}
token: ${{ github.token }}
- name: Configure AWS credentials
uses: aws-actions/configure-aws-credentials@e6de054238d6b7531b4efff3b6587d9aade6a06c # v6
with:
role-to-assume: ${{ vars.AWS_ROLE_ARN }}
aws-region: ${{ vars.AWS_REGION || 'us-east-1' }}
- name: Package and upload scripts
run: |
echo "📦 Packaging UserData scripts..."
./bin/tools/package-scripts.sh ${{ vars.ENVIRONMENT_PROD || 'prod' }}
echo "✅ Scripts packaged and uploaded successfully"
deploy-postgres:
needs: [runner, create-deployment, stack-config, build, deploy-vpc]
uses: ./.github/workflows/deploy-postgres.yml
with:
# Stack & Repository Configuration
stack_name: ${{ needs.stack-config.outputs.postgres_stack }}
environment: ${{ vars.ENVIRONMENT_PROD || 'prod' }}
# GHA Runner Configuration
runner_config: ${{ needs.runner.outputs.runner_config }}
# AWS Configuration
aws_region: ${{ vars.AWS_REGION || 'us-east-1' }}
vpc_id: ${{ needs.deploy-vpc.outputs.vpc_id }}
vpc_cidr: ${{ needs.deploy-vpc.outputs.vpc_cidr }}
subnet_ids: ${{ needs.deploy-vpc.outputs.private_subnet_ids }}
# Database Version Configuration
postgres_version: ${{ vars.DATABASE_POSTGRES_VERSION_PROD || '16.11' }}
# Instance Configuration
instance_size: ${{ vars.DATABASE_INSTANCE_SIZE_PROD || 'db.t4g.small' }}
allocated_storage: ${{ vars.DATABASE_ALLOCATED_STORAGE_PROD || '20' }}
allocated_max_storage: ${{ vars.DATABASE_MAX_ALLOCATED_STORAGE_PROD || '100' }}
# High Availability Configuration
multi_az_enabled: ${{ vars.DATABASE_MULTI_AZ_ENABLED_PROD || 'false' }}
# RDS Proxy Configuration (off by default - flip RDS_PROXY_ENABLED_PROD to 'true' to turn on)
rds_proxy_enabled: ${{ vars.RDS_PROXY_ENABLED_PROD || 'false' }}
rds_proxy_max_connections_percent: ${{ vars.RDS_PROXY_MAX_CONNECTIONS_PERCENT_PROD || '100' }}
rds_proxy_connection_borrow_timeout: ${{ vars.RDS_PROXY_CONNECTION_BORROW_TIMEOUT_PROD || '120' }}
# Performance Insights (Database Insights Standard mode - enabled via DATABASE_PI_ENABLED_PROD; free at 7-day retention)
performance_insights_enabled: ${{ vars.DATABASE_PI_ENABLED_PROD || 'false' }}
performance_insights_retention_days: ${{ vars.DATABASE_PI_RETENTION_DAYS_PROD || '7' }}
# Lambda Configuration (container-based)
lambda_image_uri: ${{ needs.build.outputs.lambda_image }}
# Other Configuration
aws_sns_alert_email: ${{ vars.AWS_SNS_ALERT_EMAIL }}
secrets: inherit
deploy-valkey:
needs: [runner, create-deployment, stack-config, build, deploy-vpc]
uses: ./.github/workflows/deploy-valkey.yml
with:
# Stack & Repository Configuration
stack_name: ${{ needs.stack-config.outputs.valkey_stack }}
environment: ${{ vars.ENVIRONMENT_PROD || 'prod' }}
# GHA Runner Configuration
runner_config: ${{ needs.runner.outputs.runner_config }}
# AWS Configuration
aws_region: ${{ vars.AWS_REGION || 'us-east-1' }}
vpc_id: ${{ needs.deploy-vpc.outputs.vpc_id }}
subnet_ids: ${{ needs.deploy-vpc.outputs.private_subnet_ids }}
# Instance Configuration
node_type: ${{ vars.VALKEY_NODE_TYPE_PROD || 'cache.t4g.micro' }}
num_cache_nodes: ${{ vars.VALKEY_NUM_NODES_PROD || '1' }}
# Valkey Version Configuration
valkey_version: ${{ vars.VALKEY_VERSION_PROD || '8.1' }}
# Security Configuration
encryption_enabled: ${{ vars.VALKEY_ENCRYPTION_ENABLED_PROD || 'true' }}
# Backup Configuration
snapshot_retention_days: ${{ vars.VALKEY_SNAPSHOT_RETENTION_DAYS_PROD || '7' }}
# Lambda Configuration (container-based)
lambda_image_uri: ${{ needs.build.outputs.lambda_image }}
# Other Configuration
aws_sns_alert_email: ${{ vars.AWS_SNS_ALERT_EMAIL }}
secrets: inherit
deploy-opensearch:
# Text search disabled by default - set OPENSEARCH_ENABLED_PROD=true to enable
if: vars.OPENSEARCH_ENABLED_PROD == 'true'
needs: [runner, create-deployment, stack-config, deploy-vpc, deploy-bastion]
uses: ./.github/workflows/deploy-opensearch.yml
with:
# Stack & Repository Configuration
stack_name: ${{ needs.stack-config.outputs.opensearch_stack }}
environment: ${{ vars.ENVIRONMENT_PROD || 'prod' }}
# GHA Runner Configuration
runner_config: ${{ needs.runner.outputs.runner_config }}
# AWS Configuration
aws_region: ${{ vars.AWS_REGION || 'us-east-1' }}
vpc_id: ${{ needs.deploy-vpc.outputs.vpc_id }}
subnet_ids: ${{ needs.deploy-vpc.outputs.private_subnet_ids }}
# Instance Configuration
instance_type: ${{ vars.OPENSEARCH_INSTANCE_TYPE_PROD || 't3.medium.search' }}
instance_count: ${{ vars.OPENSEARCH_INSTANCE_COUNT_PROD || '1' }}
zone_awareness_enabled: ${{ vars.OPENSEARCH_ZONE_AWARENESS_PROD || 'false' }}
availability_zone_count: ${{ vars.OPENSEARCH_AZ_COUNT_PROD || '2' }}
dedicated_master_enabled: ${{ vars.OPENSEARCH_DEDICATED_MASTER_PROD || 'false' }}
dedicated_master_type: ${{ vars.OPENSEARCH_MASTER_TYPE_PROD || 't3.small.search' }}
dedicated_master_count: ${{ vars.OPENSEARCH_MASTER_COUNT_PROD || '3' }}
ebs_volume_size: ${{ vars.OPENSEARCH_EBS_SIZE_PROD || '100' }}
# Engine Version Configuration
engine_version: ${{ vars.OPENSEARCH_VERSION_PROD || '2.19' }}
# Bastion Access
bastion_sg_id: ${{ needs.deploy-bastion.outputs.bastion_sg_id }}
# Other Configuration
aws_sns_alert_email: ${{ vars.AWS_SNS_ALERT_EMAIL }}
secrets: inherit
deploy-prometheus:
# Observability disabled by default - set OBSERVABILITY_ENABLED_PROD=true to enable
if: vars.OBSERVABILITY_ENABLED_PROD == 'true'
needs: [runner, create-deployment, stack-config, deploy-vpc]
uses: ./.github/workflows/deploy-prometheus.yml
with:
# Stack & Repository Configuration
stack_name: ${{ needs.stack-config.outputs.prometheus_stack }}
environment: ${{ vars.ENVIRONMENT_PROD || 'prod' }}
# GHA Runner Configuration
runner_config: ${{ needs.runner.outputs.runner_config }}
# AWS Configuration
aws_region: ${{ vars.AWS_REGION || 'us-east-1' }}
secrets: inherit
deploy-grafana:
# Observability disabled by default - set OBSERVABILITY_ENABLED_PROD=true to enable
if: vars.OBSERVABILITY_ENABLED_PROD == 'true'
needs: [runner, create-deployment, stack-config, test]
uses: ./.github/workflows/deploy-grafana.yml
with:
# Stack & Repository Configuration
stack_name: ${{ needs.stack-config.outputs.grafana_stack }}
environment: shared
# GHA Runner Configuration
runner_config: ${{ needs.runner.outputs.runner_config }}
# AWS Configuration
aws_region: ${{ vars.AWS_REGION || 'us-east-1' }}
secrets: inherit
deploy-bastion:
needs:
[
runner,
create-deployment,
stack-config,
deploy-vpc,
deploy-postgres,
deploy-valkey,
deploy-s3,
]
uses: ./.github/workflows/deploy-bastion.yml
with:
# Stack & Repository Configuration
stack_name: ${{ needs.stack-config.outputs.bastion_stack }}
environment: ${{ vars.ENVIRONMENT_PROD || 'prod' }}
# GHA Runner Configuration
runner_config: ${{ needs.runner.outputs.runner_config }}
# AWS Configuration
aws_region: ${{ vars.AWS_REGION || 'us-east-1' }}
vpc_id: ${{ needs.deploy-vpc.outputs.vpc_id }}
public_subnet_ids: ${{ needs.deploy-vpc.outputs.public_subnet_ids }}
# Security Groups (SSM-only access - no SSH parameters needed)
valkey_sg_id: ${{ needs.deploy-valkey.outputs.valkey_sg_id }}
# S3 Bucket Configuration (from S3 stack outputs)
deployment_bucket_arn: ${{ needs.deploy-s3.outputs.deployment_bucket_arn }}
secrets: inherit
# Graph Infrastructure - Step 1: Deploy DynamoDB, Secrets, Lambda monitoring
deploy-graph-infra:
needs:
[runner, create-deployment, stack-config, build, deploy-vpc, deploy-s3]
uses: ./.github/workflows/deploy-graph-infra.yml
with:
# Stack & Repository Configuration
stack_name: ${{ needs.stack-config.outputs.graph_infra_stack }}
environment: ${{ vars.ENVIRONMENT_PROD || 'prod' }}
# GHA Runner Configuration
runner_config: ${{ needs.runner.outputs.runner_config }}
# AWS Configuration
aws_region: ${{ vars.AWS_REGION || 'us-east-1' }}
# Lambda Configuration (container-based)
lambda_image_uri: ${{ needs.build.outputs.lambda_image }}
# Other Configuration
aws_sns_alert_email: ${{ vars.AWS_SNS_ALERT_EMAIL }}
secrets: inherit
# Graph Infrastructure - Step 2: Deploy EBS volume management
deploy-graph-volumes:
needs: [runner, stack-config, build, deploy-vpc, deploy-graph-infra]
uses: ./.github/workflows/deploy-graph-volumes.yml
with:
# Stack & Repository Configuration
stack_name: ${{ needs.stack-config.outputs.graph_volumes_stack }}
environment: ${{ vars.ENVIRONMENT_PROD || 'prod' }}
# GHA Runner Configuration
runner_config: ${{ needs.runner.outputs.runner_config }}
# AWS Configuration
aws_region: ${{ vars.AWS_REGION || 'us-east-1' }}
vpc_id: ${{ needs.deploy-vpc.outputs.vpc_id }}
subnet_ids: ${{ needs.deploy-vpc.outputs.private_subnet_ids }}
# Registry Configuration
volume_registry_table: ${{ needs.deploy-graph-infra.outputs.volume_registry_table }}
# Lambda Configuration (container-based)
lambda_image_uri: ${{ needs.build.outputs.lambda_image }}
# Secrets Configuration
graph_api_secret_arn: ${{ needs.deploy-graph-infra.outputs.secret_arn }}
# Other Configuration
aws_sns_alert_email: ${{ vars.AWS_SNS_ALERT_EMAIL }}
secrets: inherit
deploy-graph:
needs:
[
runner,
create-deployment,
build,
stack-config,
deploy-vpc,
deploy-postgres,
deploy-valkey,
deploy-s3,
deploy-graph-infra,
deploy-graph-volumes,
package-scripts,
]
uses: ./.github/workflows/deploy-graph.yml
with:
# Environment Configuration
environment: ${{ vars.ENVIRONMENT_PROD || 'prod' }}
# GHA Runner Configuration
runner_config: ${{ needs.runner.outputs.runner_config }}
# AWS Configuration
aws_region: ${{ vars.AWS_REGION || 'us-east-1' }}
vpc_id: ${{ needs.deploy-vpc.outputs.vpc_id }}
subnet_ids: ${{ needs.deploy-vpc.outputs.private_subnet_ids }}
public_subnet_ids: ${{ needs.deploy-vpc.outputs.public_subnet_ids }}
valkey_sg_id: ${{ needs.deploy-valkey.outputs.valkey_sg_id }}
valkey_url: ${{ needs.deploy-valkey.outputs.valkey_url }}
database_endpoint: ${{ needs.deploy-postgres.outputs.database_endpoint }}
database_port: ${{ needs.deploy-postgres.outputs.database_port }}
ami_id: ${{ needs.stack-config.outputs.graph_ami_id }}
# Container Configuration - use prod tag for EC2/ASG (container refresh handles updates)
ecr_image_tag: prod
# SEC shared replicas track the moving prod tag by default (no per-deploy
# ECRImageTag change → CloudFormation doesn't cycle the fleet; the daily
# shared_replicas_refresh is the only routine boot). Leave
# shared_replica_image_tag unset for that. Pin it ONLY during a
# storage-format-breaking engine upgrade, so a boot can't pull a new engine
# before the new-format sec.lbug is published — set a
# SHARED_REPLICA_IMAGE_TAG_PROD var for the migration window, then clear it.
shared_replica_image_tag: ${{ vars.SHARED_REPLICA_IMAGE_TAG_PROD || '' }}
# Infrastructure inputs from graph infrastructure jobs
secret_arn: ${{ needs.deploy-graph-infra.outputs.secret_arn }}
volume_registry_table: ${{ needs.deploy-graph-infra.outputs.volume_registry_table }}
volume_manager_function_arn: ${{ needs.deploy-graph-volumes.outputs.volume_manager_function_arn }}
volume_detachment_topic_arn: ${{ needs.deploy-graph-volumes.outputs.volume_detachment_topic_arn }}
# Graph Tier Configuration
lbug_large_enabled: ${{ vars.LBUG_LARGE_ENABLED_PROD }}
lbug_xlarge_enabled: ${{ vars.LBUG_XLARGE_ENABLED_PROD }}
lbug_shared_enabled: ${{ vars.LBUG_SHARED_ENABLED_PROD }}
# Graph Scaling Configuration
lbug_standard_min_instances: ${{ vars.LBUG_STANDARD_MIN_INSTANCES_PROD }}
lbug_standard_max_instances: ${{ vars.LBUG_STANDARD_MAX_INSTANCES_PROD }}
lbug_large_min_instances: ${{ vars.LBUG_LARGE_MIN_INSTANCES_PROD }}
lbug_large_max_instances: ${{ vars.LBUG_LARGE_MAX_INSTANCES_PROD }}
lbug_xlarge_min_instances: ${{ vars.LBUG_XLARGE_MIN_INSTANCES_PROD }}
lbug_xlarge_max_instances: ${{ vars.LBUG_XLARGE_MAX_INSTANCES_PROD }}
lbug_shared_min_instances: ${{ vars.LBUG_SHARED_MIN_INSTANCES_PROD }}
lbug_shared_max_instances: ${{ vars.LBUG_SHARED_MAX_INSTANCES_PROD }}
# Shared Replicas Configuration
shared_replicas_enabled: ${{ vars.SHARED_REPLICAS_ENABLED_PROD }}
shared_replicas_min_instances: ${{ vars.SHARED_REPLICAS_MIN_INSTANCES_PROD || '1' }}
shared_replicas_max_instances: ${{ vars.SHARED_REPLICAS_MAX_INSTANCES_PROD || '3' }}
shared_replicas_desired_capacity: ${{ vars.SHARED_REPLICAS_DESIRED_CAPACITY_PROD || '1' }}
shared_replicas_root_volume_size: ${{ vars.SHARED_REPLICAS_ROOT_VOLUME_SIZE_PROD || '100' }}
shared_repositories: ${{ vars.SHARED_REPOSITORIES_PROD || 'sec' }}
shared_replicas_cpu_target: ${{ vars.SHARED_REPLICAS_CPU_TARGET_PROD || '70' }}
shared_replicas_memory_target: ${{ vars.SHARED_REPLICAS_MEMORY_TARGET_PROD || '80' }}
shared_replicas_enable_response_time_scaling: ${{ vars.SHARED_REPLICAS_ENABLE_RESPONSE_TIME_SCALING_PROD || 'false' }}
shared_replicas_response_time_target: ${{ vars.SHARED_REPLICAS_RESPONSE_TIME_TARGET_PROD || '5' }}
shared_replicas_instance_warmup: ${{ vars.SHARED_REPLICAS_INSTANCE_WARMUP_PROD || '900' }}
shared_replicas_health_check_grace_period: ${{ vars.SHARED_REPLICAS_HEALTH_CHECK_GRACE_PERIOD_PROD || '900' }}
# Shared Replicas Spot Configuration
shared_replicas_spot_enabled: ${{ vars.SHARED_REPLICAS_SPOT_ENABLED_PROD || 'false' }}
shared_replicas_od_base: ${{ vars.SHARED_REPLICAS_OD_BASE_PROD || '0' }}
shared_replicas_spot_weight: ${{ vars.SHARED_REPLICAS_SPOT_WEIGHT_PROD || '0' }}
# Other Configuration
aws_sns_alert_email: ${{ vars.AWS_SNS_ALERT_EMAIL }}
secrets: inherit
deploy-api:
needs:
[
runner,
create-deployment,
stack-config,
build,
deploy-vpc,
deploy-valkey,
deploy-postgres,
deploy-s3,
deploy-grafana,
deploy-prometheus,
deploy-opensearch,
deploy-bastion,
]
if: ${{ !cancelled() && !contains(needs.*.result, 'cancelled') && !contains(needs.*.result, 'failure') }}
uses: ./.github/workflows/deploy-api.yml
with:
# Stack & Repository Configuration
stack_name: ${{ needs.stack-config.outputs.api_stack }}
environment: ${{ vars.ENVIRONMENT_PROD || 'prod' }}
# GHA Runner Configuration
runner_config: ${{ needs.runner.outputs.runner_config }}
# AWS Configuration
aws_region: ${{ vars.AWS_REGION || 'us-east-1' }}
vpc_id: ${{ needs.deploy-vpc.outputs.vpc_id }}
subnet_ids: ${{ needs.deploy-vpc.outputs.private_subnet_ids }}
public_subnet_ids: ${{ needs.deploy-vpc.outputs.public_subnet_ids }}
# Container & Application Configuration
ecr_repository_url: ${{ needs.build.outputs.ecr_repository_url }}
ecr_image_tag: ${{ needs.build.outputs.image_ref }}
# ECS & Compute Configuration
cpu: ${{ vars.API_CPU_PROD || '512' }}
memory: ${{ vars.API_MEMORY_PROD || '1024' }}
fargate_spot_weight: ${{ vars.API_FARGATE_SPOT_WEIGHT_PROD || '0' }}
fargate_base: ${{ vars.API_FARGATE_BASE_PROD || '0' }}
# Auto-scaling Configuration
min_capacity: ${{ vars.API_MIN_CAPACITY_PROD || '1' }}
max_capacity: ${{ vars.API_MAX_CAPACITY_PROD || '2' }}
cpu_target_value: ${{ vars.API_CPU_TARGET_PROD || '70' }}
memory_target_value: ${{ vars.API_MEMORY_TARGET_PROD || '80' }}
# Access Mode & Domain Configuration
# API_ACCESS_MODE_PROD: 'public' (HTTPS with domain) or 'internal' (SSM tunnel)
api_access_mode: ${{ vars.API_ACCESS_MODE_PROD || 'internal' }}
domain_name: ${{ vars.API_DOMAIN_NAME_PROD || '' }}
domain_name_root: ${{ needs.stack-config.outputs.domain_name_root }}
# Database Configuration (from Postgres stack outputs)
# Uses RDS Proxy endpoint when RDS_PROXY_ENABLED_PROD=true, direct endpoint otherwise.
# The proxy output is an empty string when the flag is off, which GHA's || coalesce treats as falsy.
database_endpoint: ${{ needs.deploy-postgres.outputs.database_proxy_endpoint || needs.deploy-postgres.outputs.database_endpoint }}
database_port: ${{ needs.deploy-postgres.outputs.database_port }}
# Lambda Configuration (container-based)
lambda_image_uri: ${{ needs.build.outputs.lambda_image }}
# Cache Configuration
valkey_url: ${{ needs.deploy-valkey.outputs.valkey_url }}
valkey_sg_id: ${{ needs.deploy-valkey.outputs.valkey_sg_id }}
bastion_sg_id: ${{ needs.deploy-bastion.outputs.bastion_sg_id }}
# OpenSearch Configuration (optional, empty when OPENSEARCH_ENABLED_PROD=false)
opensearch_endpoint: ${{ needs.deploy-opensearch.outputs.opensearch_endpoint || '' }}
opensearch_sg_id: ${{ needs.deploy-opensearch.outputs.opensearch_sg_id || '' }}
# WAF Configuration (environment-specific, skipped for internal-only deployments)
waf_enabled: ${{ vars.WAF_ENABLED_PROD || 'false' }}
waf_stack_name: ${{ needs.stack-config.outputs.waf_stack }}
waf_rate_limit_per_ip: ${{ vars.WAF_RATE_LIMIT_PER_IP || '3000' }}
waf_enable_geo_blocking: ${{ vars.WAF_GEO_BLOCKING_ENABLED || 'false' }}
waf_enable_aws_managed_rules: ${{ vars.WAF_AWS_MANAGED_RULES_ENABLED || 'true' }}
audit_enabled: ${{ vars.AUDIT_ENABLED_PROD || 'false' }}
audit_stack_name: ${{ needs.stack-config.outputs.audit_stack }}
audit_retention_days: ${{ vars.AUDIT_RETENTION_DAYS || '400' }}
# S3 Bucket Configuration (from S3 stack outputs)
shared_processed_bucket_arn: ${{ needs.deploy-s3.outputs.shared_processed_bucket_arn }}
deployment_bucket_arn: ${{ needs.deploy-s3.outputs.deployment_bucket_arn }}
user_data_bucket_arn: ${{ needs.deploy-s3.outputs.user_data_bucket_arn }}
# Other Configuration
prometheus_stack_name: ${{ needs.deploy-prometheus.outputs.prometheus_stack_name || '' }}
aws_sns_alert_email: ${{ vars.AWS_SNS_ALERT_EMAIL }}
robosystems_app_url: ${{ vars.ROBOSYSTEMS_APP_URL_PROD || 'https://robosystems.ai' }}
roboledger_app_url: ${{ vars.ROBOLEDGER_APP_URL_PROD || 'https://roboledger.ai' }}
roboinvestor_app_url: ${{ vars.ROBOINVESTOR_APP_URL_PROD || 'https://roboinvestor.ai' }}
secrets: inherit
deploy-dagster:
needs:
[
runner,
create-deployment,
stack-config,
build,
deploy-vpc,
deploy-valkey,
deploy-postgres,
deploy-s3,
deploy-grafana,
deploy-prometheus,
deploy-opensearch,
deploy-bastion,
]
if: ${{ !cancelled() && !contains(needs.*.result, 'cancelled') && !contains(needs.*.result, 'failure') }}
uses: ./.github/workflows/deploy-dagster.yml
with:
# Stack & Repository Configuration
stack_name: ${{ needs.stack-config.outputs.dagster_stack }}
environment: ${{ vars.ENVIRONMENT_PROD || 'prod' }}
# GHA Runner Configuration
runner_config: ${{ needs.runner.outputs.runner_config }}
# AWS Configuration
aws_region: ${{ vars.AWS_REGION || 'us-east-1' }}
vpc_id: ${{ needs.deploy-vpc.outputs.vpc_id }}
subnet_ids: ${{ needs.deploy-vpc.outputs.private_subnet_ids }}
# Container & Application Configuration
ecr_repository_url: ${{ needs.build.outputs.ecr_repository_url }}
ecr_image_tag: ${{ needs.build.outputs.image_ref }}
# Fargate Configuration
daemon_cpu: ${{ vars.DAGSTER_DAEMON_CPU_PROD || '1024' }}
daemon_memory: ${{ vars.DAGSTER_DAEMON_MEMORY_PROD || '2048' }}
webserver_cpu: ${{ vars.DAGSTER_WEBSERVER_CPU_PROD || '512' }}
webserver_memory: ${{ vars.DAGSTER_WEBSERVER_MEMORY_PROD || '1024' }}
webserver_desired_count: ${{ vars.DAGSTER_WEBSERVER_DESIRED_COUNT_PROD || '0' }}
# Run Job Profiles: hardcoded in deploy-dagster.yml (Light/Standard/Compute/Heavy)
max_concurrent_runs: ${{ vars.DAGSTER_MAX_CONCURRENT_RUNS_PROD || '20' }}
# Worker Configuration
worker_stack_name: ${{ needs.stack-config.outputs.worker_stack }}
graph_volumes_stack_name: ${{ needs.stack-config.outputs.graph_volumes_stack }}
worker_enabled: ${{ vars.WORKER_ENABLED_PROD || 'false' }}
worker_cpu: ${{ vars.WORKER_CPU_PROD || '512' }}
worker_memory: ${{ vars.WORKER_MEMORY_PROD || '1024' }}
worker_desired_count: ${{ vars.WORKER_DESIRED_COUNT_PROD || '1' }}
worker_min_count: ${{ vars.WORKER_MIN_COUNT_PROD || '1' }}
worker_max_count: ${{ vars.WORKER_MAX_COUNT_PROD || '5' }}
worker_autoscaling_enabled: ${{ vars.WORKER_AUTOSCALING_ENABLED_PROD || 'false' }}
aws_sns_alert_email: ${{ vars.AWS_SNS_ALERT_EMAIL }}
# Capacity Provider Configuration (Spot vs On-Demand)
daemon_fargate_spot_weight: ${{ vars.DAGSTER_DAEMON_FARGATE_SPOT_WEIGHT_PROD || '0' }}
webserver_fargate_spot_weight: ${{ vars.DAGSTER_WEBSERVER_FARGATE_SPOT_WEIGHT_PROD || '0' }}
daemon_fargate_base: ${{ vars.DAGSTER_DAEMON_FARGATE_BASE_PROD || '0' }}
webserver_fargate_base: ${{ vars.DAGSTER_WEBSERVER_FARGATE_BASE_PROD || '0' }}
# Database Configuration (from Postgres stack outputs)
# Uses RDS Proxy endpoint when RDS_PROXY_ENABLED_PROD=true, direct endpoint otherwise.
# Dagster daemon connections will pin on the proxy (LISTEN/NOTIFY, advisory locks) so
# they get no proxy benefit but pinning is neutral. The embedded worker stack benefits.
database_endpoint: ${{ needs.deploy-postgres.outputs.database_proxy_endpoint || needs.deploy-postgres.outputs.database_endpoint }}
database_port: ${{ needs.deploy-postgres.outputs.database_port }}
# Cache Configuration
valkey_url: ${{ needs.deploy-valkey.outputs.valkey_url }}
valkey_sg_id: ${{ needs.deploy-valkey.outputs.valkey_sg_id }}
# Database Security Group
database_sg_id: ${{ needs.deploy-postgres.outputs.database_sg_id }}
# OpenSearch Configuration (optional, empty when OPENSEARCH_ENABLED_PROD=false)
opensearch_endpoint: ${{ needs.deploy-opensearch.outputs.opensearch_endpoint || '' }}
opensearch_sg_id: ${{ needs.deploy-opensearch.outputs.opensearch_sg_id || '' }}
# Bastion Access (for internal UI access)
bastion_sg_id: ${{ needs.deploy-bastion.outputs.bastion_sg_id }}
# Monitoring Configuration
container_insights_enabled: ${{ vars.DAGSTER_CONTAINER_INSIGHTS_PROD || 'disabled' }}
# S3 Bucket Configuration (from S3 stack outputs)
shared_raw_bucket_arn: ${{ needs.deploy-s3.outputs.shared_raw_bucket_arn }}
shared_processed_bucket_arn: ${{ needs.deploy-s3.outputs.shared_processed_bucket_arn }}
deployment_bucket_arn: ${{ needs.deploy-s3.outputs.deployment_bucket_arn }}
user_data_bucket_arn: ${{ needs.deploy-s3.outputs.user_data_bucket_arn }}
public_data_bucket_arn: ${{ needs.deploy-s3.outputs.public_data_bucket_arn }}
logs_bucket_arn: ${{ needs.deploy-s3.outputs.logs_bucket_arn }}
public_data_cdn_url: ${{ needs.deploy-s3.outputs.public_data_cdn_url }}
robosystems_app_url: ${{ vars.ROBOSYSTEMS_APP_URL_PROD || 'https://robosystems.ai' }}
roboledger_app_url: ${{ vars.ROBOLEDGER_APP_URL_PROD || 'https://roboledger.ai' }}
roboinvestor_app_url: ${{ vars.ROBOINVESTOR_APP_URL_PROD || 'https://roboinvestor.ai' }}
secrets: inherit
# Service refresh - currently only graph containers (ECS services auto-deploy via version tags)
service-refresh:
needs: [runner, create-deployment, stack-config, deploy-graph]
if: ${{ !cancelled() && needs.deploy-graph.result == 'success' }}
uses: ./.github/workflows/service-refresh.yml
with:
environment: ${{ vars.ENVIRONMENT_PROD || 'prod' }}
runner_config: ${{ needs.runner.outputs.runner_config }}
aws_region: ${{ vars.AWS_REGION || 'us-east-1' }}
# Graph refresh (EC2 containers via SSM)
graph_refresh_enabled: ${{ inputs.graph_container_refresh != false }}
graph_node_types: "writer"
# Pre-refresh busy-counter wait (protects in-flight materialization
# on the shared-tier SEC master from mid-op cycling)
max_wait_minutes: ${{ inputs.graph_refresh_max_wait_minutes || '30' }}
force_ignore_busy: ${{ (inputs.graph_refresh_force_ignore_busy && 'true') || 'false' }}
# API/Dagster/Worker refresh disabled - ECS auto-deploys via version tags in task definitions
api_refresh_enabled: "false"
api_stack_name: ${{ needs.stack-config.outputs.api_stack }}
dagster_refresh_enabled: "false"
worker_refresh_enabled: "false"
secrets: inherit
deployment-successful:
needs:
[
runner,
test,
build,
stack-config,
create-deployment,
deploy-vpc,
deploy-graph,
deploy-valkey,
deploy-postgres,
deploy-s3,
deploy-grafana,
deploy-prometheus,
deploy-opensearch,
deploy-bastion,
deploy-api,
deploy-dagster,
service-refresh,
]
if: |
always() &&
needs.runner.result == 'success' &&
needs.test.result == 'success' &&
needs.build.result == 'success' &&
needs.stack-config.result == 'success' &&
needs.create-deployment.result == 'success' &&
needs.deploy-vpc.result == 'success' &&
needs.deploy-graph.result == 'success' &&
needs.deploy-valkey.result == 'success' &&
needs.deploy-postgres.result == 'success' &&
needs.deploy-s3.result == 'success' &&
(needs.deploy-grafana.result == 'success' || needs.deploy-grafana.result == 'skipped') &&
(needs.deploy-prometheus.result == 'success' || needs.deploy-prometheus.result == 'skipped') &&
(needs.deploy-opensearch.result == 'success' || needs.deploy-opensearch.result == 'skipped') &&
needs.deploy-bastion.result == 'success' &&
needs.deploy-api.result == 'success' &&
needs.deploy-dagster.result == 'success' &&
(needs.service-refresh.result == 'success' || needs.service-refresh.result == 'skipped')
runs-on: ${{ fromJSON(needs.runner.outputs.runner_config) }}
outputs:
deployment_id: ${{ needs.create-deployment.outputs.deployment_id }}
steps:
- name: Checkout Code
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
with:
repository: ${{ github.repository }}
ref: ${{ github.ref }}
token: ${{ github.token }}
- name: Generate Deployment Summary
env:
API_ACCESS_MODE: ${{ vars.API_ACCESS_MODE_PROD || 'internal' }}
API_DOMAIN: ${{ vars.API_DOMAIN_NAME_PROD || '' }}
LBUG_LARGE_ENABLED: ${{ vars.LBUG_LARGE_ENABLED_PROD || 'false' }}
LBUG_XLARGE_ENABLED: ${{ vars.LBUG_XLARGE_ENABLED_PROD || 'false' }}
LBUG_SHARED_ENABLED: ${{ vars.LBUG_SHARED_ENABLED_PROD || 'false' }}
run: |
cat >> $GITHUB_STEP_SUMMARY << EOF
## Production Deployment Successful
| Component | Details |
|-----------|---------|
| Environment | production |
| Image Tag | \`${{ needs.build.outputs.image_tag }}\` |
| Version | \`${{ needs.build.outputs.version_tag || 'N/A' }}\` |
| Image | \`${{ needs.build.outputs.image }}\` |
| Commit | \`${{ github.sha }}\` |
| Ref | \`${{ github.ref_name }}\` |
| Triggered by | @${{ github.actor }} |
### API Access
EOF
if [ "$API_ACCESS_MODE" = "public" ] && [ -n "$API_DOMAIN" ]; then
cat >> $GITHUB_STEP_SUMMARY << EOF
- **Mode**: Public (HTTPS)
- **URL**: https://${API_DOMAIN}
- **Admin CLI**: \`just admin prod <command>\` (requires \`just tunnel prod api-internal\`; the CLI targets the tunnel on :18000)
EOF
else
cat >> $GITHUB_STEP_SUMMARY << EOF
- **Mode**: Internal (SSM tunnel required)
- **Connect**: \`just tunnel prod all\`
- **API**: http://localhost:18000
- **Admin CLI**: \`just admin prod <command>\`
EOF
fi
cat >> $GITHUB_STEP_SUMMARY << EOF
### LadybugDB Graph Clusters
| Tier | Status |
|------|--------|
| Standard | Enabled |
EOF
if [ "$LBUG_LARGE_ENABLED" = "true" ]; then
echo "| Large | Enabled |" >> $GITHUB_STEP_SUMMARY
fi
if [ "$LBUG_XLARGE_ENABLED" = "true" ]; then
echo "| XLarge | Enabled |" >> $GITHUB_STEP_SUMMARY
fi
if [ "$LBUG_SHARED_ENABLED" = "true" ]; then
echo "| Shared | Enabled |" >> $GITHUB_STEP_SUMMARY
fi
cat >> $GITHUB_STEP_SUMMARY << EOF
### All Services
Connect to all services with one command:
\`\`\`
just tunnel prod all
\`\`\`
| Service | Local Port |
|---------|------------|
| API | http://localhost:18000 |
| Dagster UI | http://localhost:18002 |
| PostgreSQL | localhost:15432 |
| Valkey | localhost:16379 |
### Next Steps
- [Bootstrap Guide](https://github.com/RoboFinSystems/robosystems/wiki/Bootstrap-Guide) - Initial setup
- [Architecture Overview](https://github.com/RoboFinSystems/robosystems/wiki/Architecture-Overview) - System architecture
EOF
- name: Update Deployment Status
uses: chrnorm/deployment-status@6df8d036fd2fee9eb82936733953da1f8382b41e # v2
with:
token: ${{ github.token }}
state: success
deployment-id: ${{ needs.create-deployment.outputs.deployment_id }}
description: "Production Deployment Succeeded"
handle-deployment-failure:
needs:
[
runner,
test,
build,
create-deployment,
deploy-vpc,
deploy-graph,
deploy-valkey,
deploy-postgres,
deploy-s3,
deploy-grafana,
deploy-prometheus,
deploy-opensearch,
deploy-bastion,
deploy-api,
deploy-dagster,
service-refresh,
]
if: always() && contains(needs.*.result, 'failure')
runs-on: ${{ fromJSON(needs.runner.outputs.runner_config) }}
steps:
- name: Checkout Code
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
with:
repository: ${{ github.repository }}
ref: ${{ github.ref }}
token: ${{ github.token }}
- name: Handle Deployment Failure
uses: ./.github/actions/handle-deployment-failure
with:
deployment-id: ${{ needs.create-deployment.outputs.deployment_id }}
environment: production
github-token: ${{ github.token }}