service-refresh.yml

name: Service Refresh

on:
  workflow_dispatch:
    inputs:
      environment:
        description: "Environment to refresh (staging, prod)"
        required: true
        type: choice
        options:
          - staging
          - prod
      graph_refresh_enabled:
        description: "Refresh graph containers"
        required: false
        type: boolean
        default: true
      api_refresh_enabled:
        description: "Refresh API ECS service"
        required: false
        type: boolean
        default: true
      dagster_refresh_enabled:
        description: "Refresh Dagster ECS services"
        required: false
        type: boolean
        default: true
      worker_refresh_enabled:
        description: "Refresh Worker ECS service"
        required: false
        type: boolean
        default: true
      graph_node_types:
        description: "Graph node types to refresh"
        required: false
        type: choice
        options:
          - writer
          - all
          - shared
          - shared-replicas
        default: "writer"
      max_wait_minutes:
        description: "Minutes to wait for in-flight destructive ops to finish"
        required: false
        type: string
        default: "30"
      force_ignore_busy:
        description: "Skip busy-counter check (emergency override)"
        required: false
        type: boolean
        default: false
      force_restart:
        description: "Restart containers even when the image is unchanged (for secret rotation)"
        required: false
        type: boolean
        default: false
  workflow_call:
    inputs:
      environment:
        description: "Environment to refresh (staging, prod)"
        required: true
        type: string
      runner_config:
        description: "GitHub Actions runner configuration (JSON array)"
        required: false
        type: string
        default: '["ubuntu-latest"]'
      aws_region:
        description: "AWS region"
        required: false
        type: string
        default: "us-east-1"
      # Graph refresh configuration
      graph_refresh_enabled:
        description: "Refresh graph containers"
        required: false
        type: string
        default: "true"
      graph_node_types:
        description: "Graph node types to refresh (writer, all, shared, shared-replicas)"
        required: false
        type: string
        default: "writer"
      max_wait_minutes:
        description: "Minutes to wait for in-flight destructive ops on the target instance"
        required: false
        type: string
        default: "30"
      force_ignore_busy:
        description: "Skip busy-counter check (emergency override)"
        required: false
        type: string
        default: "false"
      force_restart:
        description: >-
          Restart containers even when the image digest is unchanged. Set by
          secrets rotation, whose whole purpose is replacing the process so it
          re-reads rotated credentials — without this the digest-skip correctly
          finds nothing to pull and the refresh is a no-op.
        required: false
        type: string
        default: "false"
      # API refresh configuration
      api_refresh_enabled:
        description: "Refresh API ECS service"
        required: false
        type: string
        default: "true"
      api_stack_name:
        description: "API CloudFormation stack name"
        required: false
        type: string
        default: ""
      # Dagster refresh configuration
      dagster_refresh_enabled:
        description: "Refresh Dagster ECS services"
        required: false
        type: string
        default: "true"
      worker_refresh_enabled:
        description: "Refresh Worker ECS service"
        required: false
        type: string
        default: "true"
# Minimal top-level permissions - jobs define their own
permissions: {}

jobs:
  # Computes the refresh job's timeout (Actions expressions have no arithmetic).
  # It must exceed the poll budget (max_wait_minutes + 15), or the runner is
  # killed before the poll's graceful give-up branch can fire.
  graph-refresh-budget:
    if: ${{ inputs.graph_refresh_enabled == true || inputs.graph_refresh_enabled == 'true' }}
    runs-on: ubuntu-latest
    timeout-minutes: 5
    outputs:
      job_timeout_minutes: ${{ steps.budget.outputs.job_timeout_minutes }}
    steps:
      - name: Compute refresh job timeout
        id: budget
        run: |
          MAX_WAIT="${{ inputs.max_wait_minutes || '30' }}"
          if ! [[ "$MAX_WAIT" =~ ^[0-9]+$ ]]; then
            echo "::error::max_wait_minutes must be a positive integer, got '$MAX_WAIT'"
            exit 1
          fi
          # +20 covers the poll loop's own +15 headroom plus runner startup.
          echo "job_timeout_minutes=$((MAX_WAIT + 20))" >> $GITHUB_OUTPUT
          echo "Refresh job timeout: $((MAX_WAIT + 20))m (busy-wait ceiling ${MAX_WAIT}m)"

  # ============================================
  # Graph Container Refresh (fleet-wide, via SSM)
  # ============================================
  # One Lambda invocation, one tag-targeted SSM command per node-type group.
  # Not a job matrix: it caps at 256 jobs and would pay runner minutes per
  # instance through the busy-wait.
  refresh-graph:
    needs: [graph-refresh-budget]
    if: ${{ inputs.graph_refresh_enabled == true || inputs.graph_refresh_enabled == 'true' }}
    runs-on: ${{ github.event_name == 'workflow_dispatch' && 'ubuntu-latest' || fromJSON(inputs.runner_config) }}
    # Must exceed the poll budget computed inside the step below, or the runner
    # kills the job before the poll can report anything. Derived, never fixed —
    # see graph-refresh-budget.
    timeout-minutes: ${{ fromJSON(needs.graph-refresh-budget.outputs.job_timeout_minutes) }}
    permissions:
      id-token: write
      contents: read
    steps:
      - name: Checkout repository
        uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
        with:
          repository: ${{ github.repository }}
          ref: ${{ github.ref }}
          token: ${{ github.token }}

      - name: Configure AWS credentials
        uses: aws-actions/configure-aws-credentials@e6de054238d6b7531b4efff3b6587d9aade6a06c # v6
        with:
          role-to-assume: ${{ vars.AWS_ROLE_ARN }}
          aws-region: ${{ github.event_name == 'workflow_dispatch' && (vars.AWS_REGION || 'us-east-1') || inputs.aws_region }}

      # Stack name from stack-config.sh, function name from the stack's outputs;
      # never string-built here.
      - name: Resolve the refresh function
        id: fn
        env:
          REGION: ${{ github.event_name == 'workflow_dispatch' && (vars.AWS_REGION || 'us-east-1') || inputs.aws_region }}
          ENV_NAME: ${{ inputs.environment }}
        run: |
          set -uo pipefail
          STACK=$(bash bin/tools/stack-config.sh "$ENV_NAME" graph.infra)
          echo "graph-infra stack: $STACK"

          FUNCTION=$(aws cloudformation describe-stacks --stack-name "$STACK" \
            --query "Stacks[0].Outputs[?OutputKey=='GraphContainerRefreshFunctionName'].OutputValue" \
            --output text --region "$REGION" 2>/dev/null) || FUNCTION=""

          if [ -z "$FUNCTION" ] || [ "$FUNCTION" = "None" ]; then
            echo "::error::${STACK} has no GraphContainerRefreshFunctionName output. Deploy graph-infra before refreshing graph containers."
            exit 1
          fi
          echo "function=$FUNCTION" >> $GITHUB_OUTPUT

      - name: Refresh graph containers
        env:
          REGION: ${{ github.event_name == 'workflow_dispatch' && (vars.AWS_REGION || 'us-east-1') || inputs.aws_region }}
          ENV_NAME: ${{ inputs.environment }}
          NODE_TYPES: ${{ inputs.graph_node_types || 'writer' }}
          MAX_WAIT: ${{ inputs.max_wait_minutes || '30' }}
          FORCE_IGNORE_BUSY: ${{ inputs.force_ignore_busy || 'false' }}
          FORCE_RESTART: ${{ inputs.force_restart || 'false' }}
          FUNCTION: ${{ steps.fn.outputs.function }}
        run: |
          set -uo pipefail
          echo "Invoking ${FUNCTION}"

          PAYLOAD=$(jq -n \
            --arg env "$ENV_NAME" --arg nt "$NODE_TYPES" \
            --argjson wait "$MAX_WAIT" \
            --argjson ignore_busy "$([ "$FORCE_IGNORE_BUSY" = "true" ] && echo true || echo false)" \
            --argjson force_restart "$([ "$FORCE_RESTART" = "true" ] && echo true || echo false)" \
            '{action:"start", environment:$env, node_types:$nt, max_wait_minutes:$wait,
              force_ignore_busy:$ignore_busy, force_restart:$force_restart}')

          aws lambda invoke --function-name "$FUNCTION" \
            --cli-binary-format raw-in-base64-out \
            --payload "$PAYLOAD" --region "$REGION" /tmp/start.json >/dev/null || {
            echo "::error::Failed to invoke ${FUNCTION}"; exit 1; }

          cat /tmp/start.json | jq .
          # A propagated exception has no statusCode at all — its message is in
          # errorMessage (the Lambda raises operational failures so the stack's
          # errors alarm pages on them; see graph_container_refresh.handler).
          if [ "$(jq -r '.statusCode' /tmp/start.json)" != "200" ]; then
            echo "::error::Refresh dispatch failed: $(jq -r '.error // .errorMessage // "unknown"' /tmp/start.json)"
            exit 1
          fi

          IDS=$(jq -r '[.commands[].command_id | select(. != null)]' /tmp/start.json)
          if [ "$(jq -r 'length' <<<"$IDS")" = "0" ]; then
            echo "No graph instances matched — nothing to refresh."
            echo "### Graph refresh: no instances matched" >> "$GITHUB_STEP_SUMMARY"
            exit 0
          fi
          echo "Dispatched commands: $IDS"

          # Poll to terminal. The budget mirrors the instance-side ceiling plus
          # headroom for the pull, restart and health check, and the job's own
          # timeout is derived to exceed it (see graph-refresh-budget).
          DEADLINE=$(( $(date +%s) + MAX_WAIT * 60 + 900 ))
          echo '{}' > /tmp/status.json
          POLL_ERRORS=0
          while true; do
            if aws lambda invoke --function-name "$FUNCTION" \
                 --cli-binary-format raw-in-base64-out \
                 --payload "$(jq -n --argjson ids "$IDS" '{action:"status", command_ids:$ids}')" \
                 --region "$REGION" /tmp/status.json >/dev/null \
               && [ "$(jq -r '.statusCode // empty' /tmp/status.json)" = "200" ]; then
              POLL_ERRORS=0
            else
              # Ride out transient failures, but a poller that never reaches the
              # Lambda must not read as "still in progress" until the deadline.
              POLL_ERRORS=$((POLL_ERRORS + 1))
              echo "::warning::status poll failed (${POLL_ERRORS} consecutive)"
              if [ "$POLL_ERRORS" -ge 5 ]; then
                echo "::error::Cannot reach ${FUNCTION} to read refresh status after ${POLL_ERRORS} attempts. The refresh may still be running on the fleet — check the SSM command history."
                exit 1
              fi
            fi

            if [ "$(jq -r '.complete // false' /tmp/status.json)" = "true" ]; then break; fi
            if [ "$(date +%s)" -ge "$DEADLINE" ]; then
              echo "::warning::Refresh did not reach a terminal state within the budget; it may still be running on the fleet."
              break
            fi
            sleep 20
          done

          jq . /tmp/status.json

          # Per-instance outcomes, so a fleet stuck on skipped-* stays visible
          # behind a green job.
          {
            echo "### Graph container refresh (${ENV_NAME}, ${NODE_TYPES})"
            echo
            echo "| Outcome | Instances |"
            echo "| --- | --- |"
            jq -r '.refresh_results // {} | to_entries[] | "| \(.key) | \(.value) |"' /tmp/status.json
            echo
            echo "| Invocation status | Count |"
            echo "| --- | --- |"
            jq -r '.counts // {} | to_entries[] | "| \(.key) | \(.value) |"' /tmp/status.json
          } >> "$GITHUB_STEP_SUMMARY"

          # `failed` excludes skips (script not installed yet, or stale
          # /etc/environment): they exit 0 so they don't spend the SSM MaxErrors
          # budget, and are surfaced here as warnings rather than failing the deploy.
          SKIPPED=$(jq -r '.skipped // 0' /tmp/status.json)
          if [ "${SKIPPED:-0}" -gt 0 ]; then
            echo "::warning::${SKIPPED} instance(s) skipped — not yet cycled onto the new refresh scripts. Cycle them (graph-asg-refresh.yml) to pick them up."
          fi

          FAILED=$(jq -r '.failed // 0' /tmp/status.json)
          if [ "${FAILED:-0}" -gt 0 ]; then
            echo "::error::${FAILED} instance(s) failed to refresh"
            jq -r '.failures[]? | "  \(.instance_id): \(.status) (exit \(.response_code))"' /tmp/status.json
            exit 1
          fi
          echo "Refresh complete."

  # ============================================
  # API ECS Service Refresh
  # ============================================
  refresh-api:
    if: ${{ inputs.api_refresh_enabled == true || inputs.api_refresh_enabled == 'true' }}
    runs-on: ${{ github.event_name == 'workflow_dispatch' && 'ubuntu-latest' || fromJSON(inputs.runner_config) }}
    timeout-minutes: 10
    permissions:
      id-token: write
      contents: read
    steps:
      - name: Checkout repository
        uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
        with:
          repository: ${{ github.repository }}
          ref: ${{ github.ref }}
          token: ${{ github.token }}

      - name: Configure AWS credentials
        uses: aws-actions/configure-aws-credentials@e6de054238d6b7531b4efff3b6587d9aade6a06c # v6
        with:
          role-to-assume: ${{ vars.AWS_ROLE_ARN }}
          aws-region: ${{ github.event_name == 'workflow_dispatch' && (vars.AWS_REGION || 'us-east-1') || inputs.aws_region }}

      - name: Determine API Stack Name
        id: stack-name
        run: |
          # Use provided stack name or construct default
          if [ -n "${{ inputs.api_stack_name }}" ]; then
            STACK_NAME="${{ inputs.api_stack_name }}"
          else
            # Default naming convention
            ENV="${{ inputs.environment }}"
            if [ "$ENV" == "prod" ]; then
              STACK_NAME="RoboSystemsAPIProd"
            else
              STACK_NAME="RoboSystemsAPIStaging"
            fi
          fi
          echo "stack_name=$STACK_NAME" >> $GITHUB_OUTPUT
          echo "📦 Using API stack: $STACK_NAME"

      - name: Refresh API ECS Service
        uses: ./.github/actions/refresh-ecs
        with:
          stack-name: ${{ steps.stack-name.outputs.stack_name }}
          service-type: "api"
          max-wait-time: "180"
          service-stability-delay: "15"

  # ============================================
  # Dagster Daemon ECS Service Refresh
  # ============================================
  refresh-dagster-daemon:
    if: ${{ inputs.dagster_refresh_enabled == true || inputs.dagster_refresh_enabled == 'true' }}
    runs-on: ${{ github.event_name == 'workflow_dispatch' && 'ubuntu-latest' || fromJSON(inputs.runner_config) }}
    timeout-minutes: 15
    permissions:
      id-token: write
      contents: read
    steps:
      - name: Checkout repository
        uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
        with:
          repository: ${{ github.repository }}
          ref: ${{ github.ref }}
          token: ${{ github.token }}

      - name: Configure AWS credentials
        uses: aws-actions/configure-aws-credentials@e6de054238d6b7531b4efff3b6587d9aade6a06c # v6
        with:
          role-to-assume: ${{ vars.AWS_ROLE_ARN }}
          aws-region: ${{ github.event_name == 'workflow_dispatch' && (vars.AWS_REGION || 'us-east-1') || inputs.aws_region }}

      - name: Refresh Dagster Daemon Service
        uses: ./.github/actions/refresh-ecs
        with:
          cluster-name: robosystems-dagster-${{ inputs.environment }}-cluster
          service-name: robosystems-dagster-daemon-${{ inputs.environment }}
          service-type: daemon
          max-wait-time: "600"
          service-stability-delay: "30"

  # ============================================
  # Dagster Webserver ECS Service Refresh
  # ============================================
  refresh-dagster-webserver:
    if: ${{ inputs.dagster_refresh_enabled == true || inputs.dagster_refresh_enabled == 'true' }}
    runs-on: ${{ github.event_name == 'workflow_dispatch' && 'ubuntu-latest' || fromJSON(inputs.runner_config) }}
    timeout-minutes: 10
    permissions:
      id-token: write
      contents: read
    steps:
      - name: Checkout repository
        uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
        with:
          repository: ${{ github.repository }}
          ref: ${{ github.ref }}
          token: ${{ github.token }}

      - name: Configure AWS credentials
        uses: aws-actions/configure-aws-credentials@e6de054238d6b7531b4efff3b6587d9aade6a06c # v6
        with:
          role-to-assume: ${{ vars.AWS_ROLE_ARN }}
          aws-region: ${{ github.event_name == 'workflow_dispatch' && (vars.AWS_REGION || 'us-east-1') || inputs.aws_region }}

      - name: Refresh Dagster Webserver Service
        uses: ./.github/actions/refresh-ecs
        with:
          cluster-name: robosystems-dagster-${{ inputs.environment }}-cluster
          service-name: robosystems-dagster-webserver-${{ inputs.environment }}
          service-type: api
          max-wait-time: "180"
          service-stability-delay: "15"

  # ============================================
  # Worker ECS Service Refresh
  # ============================================
  refresh-worker:
    if: ${{ inputs.worker_refresh_enabled == true || inputs.worker_refresh_enabled == 'true' }}
    runs-on: ${{ github.event_name == 'workflow_dispatch' && 'ubuntu-latest' || fromJSON(inputs.runner_config) }}
    timeout-minutes: 15
    permissions:
      id-token: write
      contents: read
    steps:
      - name: Checkout repository
        uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
        with:
          repository: ${{ github.repository }}
          ref: ${{ github.ref }}
          token: ${{ github.token }}

      - name: Configure AWS credentials
        uses: aws-actions/configure-aws-credentials@e6de054238d6b7531b4efff3b6587d9aade6a06c # v6
        with:
          role-to-assume: ${{ vars.AWS_ROLE_ARN }}
          aws-region: ${{ github.event_name == 'workflow_dispatch' && (vars.AWS_REGION || 'us-east-1') || inputs.aws_region }}

      - name: Check if Worker service exists
        id: check-worker
        run: |
          CLUSTER="robosystems-dagster-${{ inputs.environment }}-cluster"
          SERVICE="robosystems-worker-${{ inputs.environment }}"

          # Check if the service exists before attempting refresh
          SERVICE_STATUS=$(aws ecs describe-services \
            --cluster "$CLUSTER" \
            --services "$SERVICE" \
            --query "services[0].status" \
            --output text 2>/dev/null || echo "NOT_FOUND")

          if [ "$SERVICE_STATUS" == "ACTIVE" ]; then
            echo "exists=true" >> $GITHUB_OUTPUT
            echo "✅ Worker service found: $SERVICE"
          else
            echo "exists=false" >> $GITHUB_OUTPUT
            echo "ℹ️ Worker service not found in ${{ inputs.environment }} — skipping refresh"
          fi

      - name: Refresh Worker Service
        if: steps.check-worker.outputs.exists == 'true'
        uses: ./.github/actions/refresh-ecs
        with:
          cluster-name: robosystems-dagster-${{ inputs.environment }}-cluster
          service-name: robosystems-worker-${{ inputs.environment }}
          service-type: daemon