refactor(graph): move container refresh onto the instance

.github/workflows/graph-maintenance.yml

@@ -274,7 +274,11 @@ jobs:
           for ENV in $ENVS; do
             echo "🔍 Discovering instances for $ENV..."
 
-            # Find LadybugDB writer instances by tag
+            # Find LadybugDB writer instances by tag. This covers every writer
+            # tier including shared: the launch template tags all of them
+            # LadybugRole=writer and distinguishes the tier with WriterTier.
+            # There is no LadybugRole=shared-writer — a second query for that
+            # value matched nothing and has been removed.
             WRITERS=$(aws ec2 describe-instances \
               --filters \
                 "Name=tag:Environment,Values=$ENV" \
@@ -283,16 +287,7 @@ jobs:
               --query "Reservations[].Instances[].InstanceId" \
               --output text 2>/dev/null || echo "")
 
-            # Find shared-writer instances
-            SHARED_WRITERS=$(aws ec2 describe-instances \
-              --filters \
-                "Name=tag:Environment,Values=$ENV" \
-                "Name=tag:LadybugRole,Values=shared-writer" \
-                "Name=instance-state-name,Values=running" \
-              --query "Reservations[].Instances[].InstanceId" \
-              --output text 2>/dev/null || echo "")
-
-            for ID in $WRITERS $SHARED_WRITERS; do
+            for ID in $WRITERS; do
               if [ -n "$ID" ]; then
                 INSTANCE_IDS="$INSTANCE_IDS $ID"
                 echo "  Found: $ID"

.github/workflows/prod.yml

@@ -705,7 +705,6 @@ jobs:
       # Graph refresh (EC2 containers via SSM)
       graph_refresh_enabled: ${{ inputs.graph_container_refresh != false }}
       graph_node_types: "writer"
-      graph_health_check_timeout: "30"
       # Pre-refresh busy-counter wait (protects in-flight materialization
       # on the shared-tier SEC master from mid-op cycling)
       max_wait_minutes: ${{ inputs.graph_refresh_max_wait_minutes || '30' }}

.github/workflows/service-refresh.yml

@@ -77,11 +77,6 @@ on:
         required: false
         type: string
         default: "writer"
-      graph_health_check_timeout:
-        description: "Seconds to wait for graph health check"
-        required: false
-        type: string
-        default: "30"
       max_wait_minutes:
         description: "Minutes to wait for in-flight destructive ops on the target instance"
         required: false
@@ -130,7 +125,29 @@ jobs:
     outputs:
       matrix: ${{ steps.collect.outputs.matrix }}
       has_instances: ${{ steps.collect.outputs.has_instances }}
+      job_timeout_minutes: ${{ steps.budget.outputs.job_timeout_minutes }}
     steps:
+      - name: Compute refresh job timeout
+        id: budget
+        run: |
+          # The refresh job must outlive the instance-side busy-wait, so this is
+          # derived from max_wait_minutes rather than hardcoded — a static 15 was
+          # shorter than the 30-minute default wait, so a genuinely busy instance
+          # could never be waited out.
+          #
+          # Computed in shell, NOT in a `${{ }}` expression: GitHub Actions
+          # expressions have no arithmetic operators (the operator set is
+          # grouping, index, dereference, !, comparisons, && and ||), so a `+`
+          # there is a workflow parse error, not a value.
+          MAX_WAIT="${{ inputs.max_wait_minutes || '30' }}"
+          if ! [[ "$MAX_WAIT" =~ ^[0-9]+$ ]]; then
+            echo "::error::max_wait_minutes must be a positive integer, got '$MAX_WAIT'"
+            exit 1
+          fi
+          # +15 covers the pull, restart and health check on top of the wait.
+          echo "job_timeout_minutes=$((MAX_WAIT + 15))" >> $GITHUB_OUTPUT
+          echo "Refresh job timeout: $((MAX_WAIT + 15))m (busy-wait ceiling ${MAX_WAIT}m)"
+
       - name: Configure AWS credentials
         uses: aws-actions/configure-aws-credentials@v6
         with:
@@ -149,25 +166,24 @@ jobs:
           if [ "$NODE_TYPES" == "all" ] || [ "$NODE_TYPES" == "writer" ] || [ "$NODE_TYPES" == "shared" ]; then
             echo "📦 Collecting writer instances..."
 
-            # Get all graph writers using the LadybugRole tag
+            # Get all graph writers using the LadybugRole tag. WriterTier comes
+            # out of this same response — reading it with a per-instance
+            # describe-tags call inside the loop below is O(n) API calls from a
+            # single runner and throttles long before a large fleet finishes.
             LBUG_WRITERS=$(aws ec2 describe-instances \
               --filters \
                 "Name=tag:Environment,Values=${{ inputs.environment }}" \
                 "Name=tag:LadybugRole,Values=writer" \
                 "Name=instance-state-name,Values=running" \
-              --query "Reservations[].Instances[].InstanceId" \
+              --query "Reservations[].Instances[].[InstanceId, Tags[?Key=='WriterTier']|[0].Value]" \
               --output text \
               --region "${{ github.event_name == 'workflow_dispatch' && (vars.AWS_REGION || 'us-east-1') || inputs.aws_region }}" 2>/dev/null || echo "")
 
-            # Process graph writers
-            for INSTANCE in $LBUG_WRITERS; do
-              TIER=$(aws ec2 describe-tags \
-                --filters \
-                  "Name=resource-id,Values=$INSTANCE" \
-                  "Name=key,Values=WriterTier" \
-                --query "Tags[0].Value" \
-                --output text \
-                --region "${{ github.event_name == 'workflow_dispatch' && (vars.AWS_REGION || 'us-east-1') || inputs.aws_region }}" 2>/dev/null || echo "")
+            # Process graph writers (tab-separated "<instance-id> <tier>" rows)
+            while read -r INSTANCE TIER; do
+              [ -n "$INSTANCE" ] || continue
+              # An untagged instance renders as the literal "None" in text output
+              [ "$TIER" == "None" ] && TIER=""
 
               # Filter instances based on node_types parameter
               if [ "$NODE_TYPES" == "shared" ]; then
@@ -193,7 +209,7 @@ jobs:
               else
                 INSTANCES_JSON="${INSTANCES_JSON%]},${INSTANCE_OBJ}]"
               fi
-            done
+            done <<< "$LBUG_WRITERS"
           fi
 
           # Collect shared replicas (only if all or shared-replicas is selected)
@@ -241,7 +257,11 @@ jobs:
     needs: [collect-graph-instances]
     if: needs.collect-graph-instances.outputs.has_instances == 'true'
     runs-on: ${{ github.event_name == 'workflow_dispatch' && 'ubuntu-latest' || fromJSON(inputs.runner_config) }}
-    timeout-minutes: 15
+    # Derived from the busy-wait ceiling by the collect job (see its "Compute
+    # refresh job timeout" step) rather than hardcoded here, so the two cannot
+    # drift. The `|| 45` is a floor for the impossible case of an empty output,
+    # since an empty timeout-minutes is not a valid value.
+    timeout-minutes: ${{ needs.collect-graph-instances.outputs.job_timeout_minutes || 45 }}
     permissions:
       id-token: write
       contents: read
@@ -263,13 +283,6 @@ jobs:
           role-to-assume: ${{ vars.AWS_ROLE_ARN }}
           aws-region: ${{ github.event_name == 'workflow_dispatch' && (vars.AWS_REGION || 'us-east-1') || inputs.aws_region }}
 
-      - name: Get AWS Account ID
-        id: aws-account
-        run: |
-          # Fetch from STS - validates OIDC connection is working
-          ACCOUNT_ID=$(aws sts get-caller-identity --query Account --output text)
-          echo "account_id=${ACCOUNT_ID}" >> $GITHUB_OUTPUT
-
       - name: Update Graph Container
         uses: ./.github/actions/refresh-graph-containers
         with:
@@ -278,8 +291,6 @@ jobs:
           node-type: ${{ matrix.node_type }}
           backend: ${{ matrix.backend }}
           aws-region: ${{ github.event_name == 'workflow_dispatch' && (vars.AWS_REGION || 'us-east-1') || inputs.aws_region }}
-          aws-account-id: ${{ steps.aws-account.outputs.account_id }}
-          health-check-timeout: ${{ inputs.graph_health_check_timeout || '30' }}
           max-wait-minutes: ${{ inputs.max_wait_minutes || '30' }}
           force-ignore-busy: ${{ inputs.force_ignore_busy || 'false' }}
 

.github/workflows/staging.yml

@@ -722,7 +722,6 @@ jobs:
       # Graph refresh (EC2 containers via SSM)
       graph_refresh_enabled: ${{ inputs.graph_container_refresh != false }}
       graph_node_types: "writer"
-      graph_health_check_timeout: "30"
       # Pre-refresh busy-counter wait (protects in-flight materialization
       # on the shared-tier SEC master from mid-op cycling)
       max_wait_minutes: ${{ inputs.graph_refresh_max_wait_minutes || '30' }}

bin/lambda/graph_volume_monitor.py

@@ -227,7 +227,21 @@ def monitor_all_instances(expand_immediately: bool = False) -> dict[str, Any]:
 
 
 def discover_lbug_instances() -> list[dict]:
-  """Discover all running Graph instances"""
+  """Discover all running Graph writer instances.
+
+  `writer` is the only value the writer launch template ever puts on
+  `LadybugRole`, across every tier — the tier is carried separately on
+  `WriterTier`. This filter previously also listed `shared_master` and
+  `shared_replica`, which are `NODE_TYPE` values and never appear on
+  `LadybugRole`, so they matched nothing.
+
+  Do not widen this to match any `LadybugRole` value. Replicas carry no
+  `LadybugRole` today and are slated to get `replica` (deliberately not
+  `shared_replica`) so that a tag expression can select the whole graph fleet.
+  Either way they must stay out of this function: its callers drive volume
+  expansion and replicas carry no data volume to manage. See
+  `discover_replica_instance_ids`, which enumerates them separately.
+  """
 
   instances = []
 
@@ -236,10 +250,7 @@ def discover_lbug_instances() -> list[dict]:
     response = ec2.describe_instances(
       Filters=[
         {"Name": "tag:Service", "Values": ["RoboSystems"]},
-        {
-          "Name": "tag:LadybugRole",
-          "Values": ["writer", "shared_master", "shared_replica"],
-        },
+        {"Name": "tag:LadybugRole", "Values": ["writer"]},
         {"Name": "instance-state-name", "Values": ["running"]},
         {"Name": "tag:Environment", "Values": [ENVIRONMENT]},
       ]

bin/userdata/common/graph-health-check.sh

@@ -23,17 +23,21 @@ set -e
 TOKEN=$(curl -s -X PUT "http://169.254.169.254/latest/api/token" -H "X-aws-ec2-metadata-token-ttl-seconds: 21600")
 INSTANCE_ID=$(curl -s -H "X-aws-ec2-metadata-token: $TOKEN" http://169.254.169.254/latest/meta-data/instance-id)
 
-# Validate the runtime type and derive the container name from the instance role
+# Validate the runtime type
 if [ "${DATABASE_TYPE}" != "ladybug" ]; then
   echo "ERROR: Unsupported DATABASE_TYPE: ${DATABASE_TYPE}"
   exit 1
 fi
 
-if [ "${NODE_TYPE}" = "shared_master" ] || [ "${NODE_TYPE}" = "shared_replica" ]; then
-  CONTAINER_NAME="graph-api-shared"
-else
-  CONTAINER_NAME="graph-api"
-fi
+# Ask run-graph-container.sh for the container name rather than re-deriving the
+# NODE_TYPE mapping a third time. Both scripts are downloaded from S3 in the same
+# userdata block, so they are always the same vintage. If it is missing, this
+# check cannot identify the container OR restart it (see below), so failing here
+# is more honest than guessing a name and reporting health for it.
+CONTAINER_NAME=$(/usr/local/bin/run-graph-container.sh --print-container-name) || {
+  echo "[$(date)] ERROR: could not determine container name from run-graph-container.sh"
+  exit 1
+}
 
 # Check container status
 if docker ps | grep -q $CONTAINER_NAME; then

bin/userdata/common/graph-lifecycle.sh

@@ -20,11 +20,12 @@ INSTANCE_REGISTRY_TABLE="${INSTANCE_REGISTRY_TABLE:-robosystems-graph-${ENVIRONM
 # ==================================================================================
 # DATABASE-SPECIFIC CONFIGURATION
 # ==================================================================================
-if [ "${NODE_TYPE}" = "shared_master" ] || [ "${NODE_TYPE}" = "shared_replica" ]; then
-    CONTAINER_NAME="graph-api-shared"
-else
-    CONTAINER_NAME="graph-api"
-fi
+# Ask run-graph-container.sh for the container name rather than re-deriving the
+# NODE_TYPE mapping. It owns that mapping; every other copy of it drifted.
+CONTAINER_NAME=$(/usr/local/bin/run-graph-container.sh --print-container-name) || {
+    echo "ERROR: could not determine container name from run-graph-container.sh" >&2
+    exit 1
+}
 GRAPH_API_PORT="8001"
 DRAIN_ENDPOINT="http://localhost:${GRAPH_API_PORT}/admin/drain"
 CONNECTIONS_ENDPOINT="http://localhost:${GRAPH_API_PORT}/admin/connections"