refactor(graph): move container refresh onto the instance #10
Workflow file for this run
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Service Refresh | ||
|
Check failure on line 1 in .github/workflows/service-refresh.yml
|
||
| on: | ||
| workflow_dispatch: | ||
| inputs: | ||
| environment: | ||
| description: "Environment to refresh (staging, prod)" | ||
| required: true | ||
| type: choice | ||
| options: | ||
| - staging | ||
| - prod | ||
| graph_refresh_enabled: | ||
| description: "Refresh graph containers" | ||
| required: false | ||
| type: boolean | ||
| default: true | ||
| api_refresh_enabled: | ||
| description: "Refresh API ECS service" | ||
| required: false | ||
| type: boolean | ||
| default: true | ||
| dagster_refresh_enabled: | ||
| description: "Refresh Dagster ECS services" | ||
| required: false | ||
| type: boolean | ||
| default: true | ||
| worker_refresh_enabled: | ||
| description: "Refresh Worker ECS service" | ||
| required: false | ||
| type: boolean | ||
| default: true | ||
| graph_node_types: | ||
| description: "Graph node types to refresh" | ||
| required: false | ||
| type: choice | ||
| options: | ||
| - writer | ||
| - all | ||
| - shared | ||
| - shared-replicas | ||
| default: "writer" | ||
| max_wait_minutes: | ||
| description: "Minutes to wait for in-flight destructive ops to finish" | ||
| required: false | ||
| type: string | ||
| default: "30" | ||
| force_ignore_busy: | ||
| description: "Skip busy-counter check (emergency override)" | ||
| required: false | ||
| type: boolean | ||
| default: false | ||
| workflow_call: | ||
| inputs: | ||
| environment: | ||
| description: "Environment to refresh (staging, prod)" | ||
| required: true | ||
| type: string | ||
| runner_config: | ||
| description: "GitHub Actions runner configuration (JSON array)" | ||
| required: false | ||
| type: string | ||
| default: '["ubuntu-latest"]' | ||
| aws_region: | ||
| description: "AWS region" | ||
| required: false | ||
| type: string | ||
| default: "us-east-1" | ||
| # Graph refresh configuration | ||
| graph_refresh_enabled: | ||
| description: "Refresh graph containers" | ||
| required: false | ||
| type: string | ||
| default: "true" | ||
| graph_node_types: | ||
| description: "Graph node types to refresh (writer, all, shared, shared-replicas)" | ||
| required: false | ||
| type: string | ||
| default: "writer" | ||
| max_wait_minutes: | ||
| description: "Minutes to wait for in-flight destructive ops on the target instance" | ||
| required: false | ||
| type: string | ||
| default: "30" | ||
| force_ignore_busy: | ||
| description: "Skip busy-counter check (emergency override)" | ||
| required: false | ||
| type: string | ||
| default: "false" | ||
| # API refresh configuration | ||
| api_refresh_enabled: | ||
| description: "Refresh API ECS service" | ||
| required: false | ||
| type: string | ||
| default: "true" | ||
| api_stack_name: | ||
| description: "API CloudFormation stack name" | ||
| required: false | ||
| type: string | ||
| default: "" | ||
| # Dagster refresh configuration | ||
| dagster_refresh_enabled: | ||
| description: "Refresh Dagster ECS services" | ||
| required: false | ||
| type: string | ||
| default: "true" | ||
| worker_refresh_enabled: | ||
| description: "Refresh Worker ECS service" | ||
| required: false | ||
| type: string | ||
| default: "true" | ||
| # Minimal top-level permissions - jobs define their own | ||
| permissions: {} | ||
| jobs: | ||
| # ============================================ | ||
| # Graph Container Refresh (EC2 instances via SSM) | ||
| # ============================================ | ||
| collect-graph-instances: | ||
| if: ${{ inputs.graph_refresh_enabled == true || inputs.graph_refresh_enabled == 'true' }} | ||
| runs-on: ${{ github.event_name == 'workflow_dispatch' && 'ubuntu-latest' || fromJSON(inputs.runner_config) }} | ||
| permissions: | ||
| id-token: write | ||
| contents: read | ||
| outputs: | ||
| matrix: ${{ steps.collect.outputs.matrix }} | ||
| has_instances: ${{ steps.collect.outputs.has_instances }} | ||
| steps: | ||
| - name: Configure AWS credentials | ||
| uses: aws-actions/configure-aws-credentials@v6 | ||
| with: | ||
| role-to-assume: ${{ vars.AWS_ROLE_ARN }} | ||
| aws-region: ${{ github.event_name == 'workflow_dispatch' && (vars.AWS_REGION || 'us-east-1') || inputs.aws_region }} | ||
| - name: Collect instances to update | ||
| id: collect | ||
| run: | | ||
| echo "🔍 Collecting graph database instances for environment: ${{ inputs.environment }}" | ||
| NODE_TYPES="${{ inputs.graph_node_types }}" | ||
| INSTANCES_JSON="[]" | ||
| # Check what to update based on node_types input | ||
| if [ "$NODE_TYPES" == "all" ] || [ "$NODE_TYPES" == "writer" ] || [ "$NODE_TYPES" == "shared" ]; then | ||
| echo "📦 Collecting writer instances..." | ||
| # Get all graph writers using the LadybugRole tag. WriterTier comes | ||
| # out of this same response — reading it with a per-instance | ||
| # describe-tags call inside the loop below is O(n) API calls from a | ||
| # single runner and throttles long before a large fleet finishes. | ||
| LBUG_WRITERS=$(aws ec2 describe-instances \ | ||
| --filters \ | ||
| "Name=tag:Environment,Values=${{ inputs.environment }}" \ | ||
| "Name=tag:LadybugRole,Values=writer" \ | ||
| "Name=instance-state-name,Values=running" \ | ||
| --query "Reservations[].Instances[].[InstanceId, Tags[?Key=='WriterTier']|[0].Value]" \ | ||
| --output text \ | ||
| --region "${{ github.event_name == 'workflow_dispatch' && (vars.AWS_REGION || 'us-east-1') || inputs.aws_region }}" 2>/dev/null || echo "") | ||
| # Process graph writers (tab-separated "<instance-id> <tier>" rows) | ||
| while read -r INSTANCE TIER; do | ||
| [ -n "$INSTANCE" ] || continue | ||
| # An untagged instance renders as the literal "None" in text output | ||
| [ "$TIER" == "None" ] && TIER="" | ||
| # Filter instances based on node_types parameter | ||
| if [ "$NODE_TYPES" == "shared" ]; then | ||
| if [ "$TIER" != "shared" ]; then | ||
| continue | ||
| fi | ||
| elif [ "$NODE_TYPES" == "writer" ] || [ "$NODE_TYPES" == "all" ]; then | ||
| true | ||
| fi | ||
| # Determine node type based on tier | ||
| if [ "$TIER" == "shared" ]; then | ||
| NODE_TYPE="shared-writer" | ||
| echo " Found graph shared writer: $INSTANCE" | ||
| else | ||
| NODE_TYPE="writer" | ||
| echo " Found graph writer: $INSTANCE (tier: ${TIER:-standard})" | ||
| fi | ||
| INSTANCE_OBJ="{\"instance_id\":\"$INSTANCE\",\"node_type\":\"$NODE_TYPE\",\"backend\":\"ladybug\"}" | ||
| if [ "$INSTANCES_JSON" == "[]" ]; then | ||
| INSTANCES_JSON="[$INSTANCE_OBJ]" | ||
| else | ||
| INSTANCES_JSON="${INSTANCES_JSON%]},${INSTANCE_OBJ}]" | ||
| fi | ||
| done <<< "$LBUG_WRITERS" | ||
| fi | ||
| # Collect shared replicas (only if all or shared-replicas is selected) | ||
| if [ "$NODE_TYPES" == "all" ] || [ "$NODE_TYPES" == "shared-replicas" ]; then | ||
| echo "📦 Collecting shared replica instances..." | ||
| # Get shared replica instances by NodeType tag | ||
| SHARED_REPLICAS=$(aws ec2 describe-instances \ | ||
| --filters \ | ||
| "Name=tag:Environment,Values=${{ inputs.environment }}" \ | ||
| "Name=tag:NodeType,Values=shared_replica" \ | ||
| "Name=instance-state-name,Values=running" \ | ||
| --query "Reservations[].Instances[].InstanceId" \ | ||
| --output text \ | ||
| --region "${{ github.event_name == 'workflow_dispatch' && (vars.AWS_REGION || 'us-east-1') || inputs.aws_region }}" 2>/dev/null || echo "") | ||
| for INSTANCE in $SHARED_REPLICAS; do | ||
| echo " Found shared replica: $INSTANCE" | ||
| INSTANCE_OBJ="{\"instance_id\":\"$INSTANCE\",\"node_type\":\"shared-replica\",\"backend\":\"ladybug\"}" | ||
| if [ "$INSTANCES_JSON" == "[]" ]; then | ||
| INSTANCES_JSON="[$INSTANCE_OBJ]" | ||
| else | ||
| INSTANCES_JSON="${INSTANCES_JSON%]},${INSTANCE_OBJ}]" | ||
| fi | ||
| done | ||
| fi | ||
| echo "instances_json=$INSTANCES_JSON" | ||
| if [ "$INSTANCES_JSON" == "[]" ]; then | ||
| echo "ℹ️ No graph instances found to update" | ||
| echo "matrix={}" >> $GITHUB_OUTPUT | ||
| echo "has_instances=false" >> $GITHUB_OUTPUT | ||
| else | ||
| echo "✅ Found graph instances to update" | ||
| echo "matrix={\"include\":$INSTANCES_JSON}" >> $GITHUB_OUTPUT | ||
| echo "has_instances=true" >> $GITHUB_OUTPUT | ||
| # Display what will be updated | ||
| echo "📋 Graph instances to update:" | ||
| echo "$INSTANCES_JSON" | jq -r '.[] | " - \(.instance_id) (\(.backend)/\(.node_type))"' | ||
| fi | ||
| refresh-graph: | ||
| needs: [collect-graph-instances] | ||
| if: needs.collect-graph-instances.outputs.has_instances == 'true' | ||
| runs-on: ${{ github.event_name == 'workflow_dispatch' && 'ubuntu-latest' || fromJSON(inputs.runner_config) }} | ||
| # Derived from the busy-wait ceiling, not a fixed number: a static 15 was | ||
| # shorter than the 30-minute default wait, so a genuinely busy instance | ||
| # could never be waited out — the job was killed first. The +15 covers the | ||
| # pull, restart, and health check on top of the wait. | ||
| timeout-minutes: ${{ fromJSON(inputs.max_wait_minutes || '30') + 15 }} | ||
| permissions: | ||
| id-token: write | ||
| contents: read | ||
| strategy: | ||
| matrix: ${{ fromJSON(needs.collect-graph-instances.outputs.matrix) }} | ||
| max-parallel: 10 | ||
| fail-fast: false | ||
| steps: | ||
| - name: Checkout repository | ||
| uses: actions/checkout@v7 | ||
| with: | ||
| repository: ${{ github.repository }} | ||
| ref: ${{ github.ref }} | ||
| token: ${{ github.token }} | ||
| - name: Configure AWS credentials | ||
| uses: aws-actions/configure-aws-credentials@v6 | ||
| with: | ||
| role-to-assume: ${{ vars.AWS_ROLE_ARN }} | ||
| aws-region: ${{ github.event_name == 'workflow_dispatch' && (vars.AWS_REGION || 'us-east-1') || inputs.aws_region }} | ||
| - name: Update Graph Container | ||
| uses: ./.github/actions/refresh-graph-containers | ||
| with: | ||
| environment: ${{ inputs.environment }} | ||
| instance-id: ${{ matrix.instance_id }} | ||
| node-type: ${{ matrix.node_type }} | ||
| backend: ${{ matrix.backend }} | ||
| aws-region: ${{ github.event_name == 'workflow_dispatch' && (vars.AWS_REGION || 'us-east-1') || inputs.aws_region }} | ||
| max-wait-minutes: ${{ inputs.max_wait_minutes || '30' }} | ||
| force-ignore-busy: ${{ inputs.force_ignore_busy || 'false' }} | ||
| # ============================================ | ||
| # API ECS Service Refresh | ||
| # ============================================ | ||
| refresh-api: | ||
| if: ${{ inputs.api_refresh_enabled == true || inputs.api_refresh_enabled == 'true' }} | ||
| runs-on: ${{ github.event_name == 'workflow_dispatch' && 'ubuntu-latest' || fromJSON(inputs.runner_config) }} | ||
| timeout-minutes: 10 | ||
| permissions: | ||
| id-token: write | ||
| contents: read | ||
| steps: | ||
| - name: Checkout repository | ||
| uses: actions/checkout@v7 | ||
| with: | ||
| repository: ${{ github.repository }} | ||
| ref: ${{ github.ref }} | ||
| token: ${{ github.token }} | ||
| - name: Configure AWS credentials | ||
| uses: aws-actions/configure-aws-credentials@v6 | ||
| with: | ||
| role-to-assume: ${{ vars.AWS_ROLE_ARN }} | ||
| aws-region: ${{ github.event_name == 'workflow_dispatch' && (vars.AWS_REGION || 'us-east-1') || inputs.aws_region }} | ||
| - name: Determine API Stack Name | ||
| id: stack-name | ||
| run: | | ||
| # Use provided stack name or construct default | ||
| if [ -n "${{ inputs.api_stack_name }}" ]; then | ||
| STACK_NAME="${{ inputs.api_stack_name }}" | ||
| else | ||
| # Default naming convention | ||
| ENV="${{ inputs.environment }}" | ||
| if [ "$ENV" == "prod" ]; then | ||
| STACK_NAME="RoboSystemsAPIProd" | ||
| else | ||
| STACK_NAME="RoboSystemsAPIStaging" | ||
| fi | ||
| fi | ||
| echo "stack_name=$STACK_NAME" >> $GITHUB_OUTPUT | ||
| echo "📦 Using API stack: $STACK_NAME" | ||
| - name: Refresh API ECS Service | ||
| uses: ./.github/actions/refresh-ecs | ||
| with: | ||
| stack-name: ${{ steps.stack-name.outputs.stack_name }} | ||
| service-type: "api" | ||
| max-wait-time: "180" | ||
| service-stability-delay: "15" | ||
| # ============================================ | ||
| # Dagster Daemon ECS Service Refresh | ||
| # ============================================ | ||
| refresh-dagster-daemon: | ||
| if: ${{ inputs.dagster_refresh_enabled == true || inputs.dagster_refresh_enabled == 'true' }} | ||
| runs-on: ${{ github.event_name == 'workflow_dispatch' && 'ubuntu-latest' || fromJSON(inputs.runner_config) }} | ||
| timeout-minutes: 15 | ||
| permissions: | ||
| id-token: write | ||
| contents: read | ||
| steps: | ||
| - name: Checkout repository | ||
| uses: actions/checkout@v7 | ||
| with: | ||
| repository: ${{ github.repository }} | ||
| ref: ${{ github.ref }} | ||
| token: ${{ github.token }} | ||
| - name: Configure AWS credentials | ||
| uses: aws-actions/configure-aws-credentials@v6 | ||
| with: | ||
| role-to-assume: ${{ vars.AWS_ROLE_ARN }} | ||
| aws-region: ${{ github.event_name == 'workflow_dispatch' && (vars.AWS_REGION || 'us-east-1') || inputs.aws_region }} | ||
| - name: Refresh Dagster Daemon Service | ||
| uses: ./.github/actions/refresh-ecs | ||
| with: | ||
| cluster-name: robosystems-dagster-${{ inputs.environment }}-cluster | ||
| service-name: robosystems-dagster-daemon-${{ inputs.environment }} | ||
| service-type: daemon | ||
| max-wait-time: "600" | ||
| service-stability-delay: "30" | ||
| # ============================================ | ||
| # Dagster Webserver ECS Service Refresh | ||
| # ============================================ | ||
| refresh-dagster-webserver: | ||
| if: ${{ inputs.dagster_refresh_enabled == true || inputs.dagster_refresh_enabled == 'true' }} | ||
| runs-on: ${{ github.event_name == 'workflow_dispatch' && 'ubuntu-latest' || fromJSON(inputs.runner_config) }} | ||
| timeout-minutes: 10 | ||
| permissions: | ||
| id-token: write | ||
| contents: read | ||
| steps: | ||
| - name: Checkout repository | ||
| uses: actions/checkout@v7 | ||
| with: | ||
| repository: ${{ github.repository }} | ||
| ref: ${{ github.ref }} | ||
| token: ${{ github.token }} | ||
| - name: Configure AWS credentials | ||
| uses: aws-actions/configure-aws-credentials@v6 | ||
| with: | ||
| role-to-assume: ${{ vars.AWS_ROLE_ARN }} | ||
| aws-region: ${{ github.event_name == 'workflow_dispatch' && (vars.AWS_REGION || 'us-east-1') || inputs.aws_region }} | ||
| - name: Refresh Dagster Webserver Service | ||
| uses: ./.github/actions/refresh-ecs | ||
| with: | ||
| cluster-name: robosystems-dagster-${{ inputs.environment }}-cluster | ||
| service-name: robosystems-dagster-webserver-${{ inputs.environment }} | ||
| service-type: api | ||
| max-wait-time: "180" | ||
| service-stability-delay: "15" | ||
| # ============================================ | ||
| # Worker ECS Service Refresh | ||
| # ============================================ | ||
| refresh-worker: | ||
| if: ${{ inputs.worker_refresh_enabled == true || inputs.worker_refresh_enabled == 'true' }} | ||
| runs-on: ${{ github.event_name == 'workflow_dispatch' && 'ubuntu-latest' || fromJSON(inputs.runner_config) }} | ||
| timeout-minutes: 15 | ||
| permissions: | ||
| id-token: write | ||
| contents: read | ||
| steps: | ||
| - name: Checkout repository | ||
| uses: actions/checkout@v7 | ||
| with: | ||
| repository: ${{ github.repository }} | ||
| ref: ${{ github.ref }} | ||
| token: ${{ github.token }} | ||
| - name: Configure AWS credentials | ||
| uses: aws-actions/configure-aws-credentials@v6 | ||
| with: | ||
| role-to-assume: ${{ vars.AWS_ROLE_ARN }} | ||
| aws-region: ${{ github.event_name == 'workflow_dispatch' && (vars.AWS_REGION || 'us-east-1') || inputs.aws_region }} | ||
| - name: Check if Worker service exists | ||
| id: check-worker | ||
| run: | | ||
| CLUSTER="robosystems-dagster-${{ inputs.environment }}-cluster" | ||
| SERVICE="robosystems-worker-${{ inputs.environment }}" | ||
| # Check if the service exists before attempting refresh | ||
| SERVICE_STATUS=$(aws ecs describe-services \ | ||
| --cluster "$CLUSTER" \ | ||
| --services "$SERVICE" \ | ||
| --query "services[0].status" \ | ||
| --output text 2>/dev/null || echo "NOT_FOUND") | ||
| if [ "$SERVICE_STATUS" == "ACTIVE" ]; then | ||
| echo "exists=true" >> $GITHUB_OUTPUT | ||
| echo "✅ Worker service found: $SERVICE" | ||
| else | ||
| echo "exists=false" >> $GITHUB_OUTPUT | ||
| echo "ℹ️ Worker service not found in ${{ inputs.environment }} — skipping refresh" | ||
| fi | ||
| - name: Refresh Worker Service | ||
| if: steps.check-worker.outputs.exists == 'true' | ||
| uses: ./.github/actions/refresh-ecs | ||
| with: | ||
| cluster-name: robosystems-dagster-${{ inputs.environment }}-cluster | ||
| service-name: robosystems-worker-${{ inputs.environment }} | ||
| service-type: daemon | ||
