fix(aws-lambda): isolate smoke cleanup ownership

Author: jrusso1020

examples/aws-lambda/scripts/_aws-isolation.sh

@@ -1,6 +1,20 @@
 #!/usr/bin/env bash
 # AWS resource-name isolation and failed-deploy discovery helpers.
 
+hf_new_smoke_run_id() {
+  local seconds seed digest
+  seconds=$(date +%s)
+  seed="${seconds}:$$:${RANDOM}:${BASHPID:-$$}"
+  digest=$(printf '%s' "$seed" | sha256sum | awk '{print substr($1,1,16)}')
+  printf '%s-%s\n' "$seconds" "$digest"
+}
+
+hf_sam_deploy_bucket_name() {
+  local account_id="$1" region="$2" run_id="$3" digest
+  digest=$(printf '%s' "$run_id" | sha256sum | awk '{print substr($1,1,20)}')
+  printf 'hf-sam-%s-%s-%s\n' "$account_id" "$region" "$digest"
+}
+
 hf_derive_project_name() {
   local stack_name="$1" prefix digest
   prefix=$(printf '%s' "$stack_name" |
@@ -83,6 +97,52 @@ hf_assert_deploy_isolation() {
       --query "logGroups[?logGroupName=='$states_log'].logGroupName" --output text
 }
 
+# Atomically reserve the exact stack name before SAM can create or update it.
+# CloudFormation's create-stack call is the compare-and-set: only one concurrent
+# smoke run can acquire a name that both preflight checks observed as absent.
+hf_reserve_smoke_stack() {
+  local stack_name="$1" run_id="$2"
+  aws cloudformation create-stack \
+    --stack-name "$stack_name" \
+    --template-body \
+      '{"Resources":{"SmokeOwnershipHandle":{"Type":"AWS::CloudFormation::WaitConditionHandle"}}}' \
+    --tags "Key=HyperframesSmokeRun,Value=$run_id" >/dev/null &&
+    aws cloudformation wait stack-create-complete --stack-name "$stack_name"
+}
+
+# Print "owned" when the stack has this run's ownership tag and
+# "absent" when there is no stack. Any foreign/missing tag or AWS API error
+# fails closed so a cleanup trap cannot delete a concurrent run's resources.
+hf_stack_ownership_status() {
+  local stack_name="$1" run_id="$2" output_file error_file owner status detail
+  output_file=$(mktemp)
+  error_file=$(mktemp)
+  if aws cloudformation describe-stacks \
+    --stack-name "$stack_name" \
+    --query "Stacks[0].Tags[?Key=='HyperframesSmokeRun'].Value | [0]" \
+    --output text >"$output_file" 2>"$error_file"; then
+    owner=$(tr -d '\r\n' <"$output_file")
+    rm -f "$output_file" "$error_file"
+    if [ "$owner" != "$run_id" ]; then
+      echo "ERROR: refusing cleanup: stack ownership is '${owner:-missing}', expected '$run_id'" >&2
+      return 3
+    fi
+    printf 'owned\n'
+    return
+  else
+    status=$?
+  fi
+  if hf_known_absent "does not exist" "$error_file"; then
+    rm -f "$output_file" "$error_file"
+    printf 'absent\n'
+    return
+  fi
+  detail=$(tr '\n' ' ' <"$error_file" | cut -c1-240)
+  echo "ERROR: could not verify stack ownership (exit=$status): $detail" >&2
+  rm -f "$output_file" "$error_file"
+  return 2
+}
+
 # Return a JSON object with any physical resources CloudFormation managed to
 # create, even when stack outputs were never populated. A genuinely absent
 # stack is an empty result; auth/network/query failures are errors.

examples/aws-lambda/scripts/aws-isolation.test.sh

@@ -37,6 +37,19 @@ case "$MOCK_MODE:$operation" in
     echo "AccessDenied: credentials expired" >&2
     exit 253
     ;;
+  owned:"cloudformation describe-stacks")
+    printf 'run-a\n'
+    ;;
+  foreign:"cloudformation describe-stacks")
+    printf 'run-b\n'
+    ;;
+  ownership-missing:"cloudformation describe-stacks")
+    echo "ValidationError: Stack with id smoke does not exist" >&2
+    exit 255
+    ;;
+  reserve:"cloudformation create-stack"|reserve:"cloudformation wait")
+    exit 0
+    ;;
   discovery:"cloudformation list-stack-resources")
     cat <<'JSON'
 {"StackResourceSummaries":[
@@ -62,6 +75,14 @@ name_b=$(hf_derive_project_name "hyperframes-lambda-smoke-a-very-long-shared-pre
 [ "$name_a" != "$name_b" ]
 [ "${#name_a}" -le 49 ]
 [ "${#name_b}" -le 49 ]
+run_a=$(hf_new_smoke_run_id)
+run_b=$(hf_new_smoke_run_id)
+[ "$run_a" != "$run_b" ]
+bucket_a=$(hf_sam_deploy_bucket_name "767398024897" "us-east-2" "$run_a")
+bucket_b=$(hf_sam_deploy_bucket_name "767398024897" "us-east-2" "$run_b")
+[ "$bucket_a" != "$bucket_b" ]
+[ "${#bucket_a}" -le 63 ]
+[ "${#bucket_b}" -le 63 ]
 
 MOCK_MODE=absent PATH="$WORK/bin:$PATH" \
   hf_assert_deploy_isolation "smoke" "$name_a"
@@ -79,6 +100,19 @@ if MOCK_MODE=auth PATH="$WORK/bin:$PATH" \
 fi
 grep -q "could not prove" "$WORK/auth-error"
 
+[ "$(MOCK_MODE=owned PATH="$WORK/bin:$PATH" \
+  hf_stack_ownership_status "smoke" "run-a")" = "owned" ]
+[ "$(MOCK_MODE=ownership-missing PATH="$WORK/bin:$PATH" \
+  hf_stack_ownership_status "smoke" "run-a")" = "absent" ]
+if MOCK_MODE=foreign PATH="$WORK/bin:$PATH" \
+  hf_stack_ownership_status "smoke" "run-a" 2>"$WORK/foreign-error"; then
+  echo "expected foreign stack ownership to fail closed" >&2
+  exit 1
+fi
+grep -q "refusing cleanup" "$WORK/foreign-error"
+MOCK_MODE=reserve PATH="$WORK/bin:$PATH" \
+  hf_reserve_smoke_stack "smoke" "run-a"
+
 discovered=$(MOCK_MODE=discovery PATH="$WORK/bin:$PATH" \
   hf_discover_stack_resources "smoke")
 [ "$(jq -r .renderBucket <<<"$discovered")" = "physical-render-bucket" ]

examples/aws-lambda/scripts/smoke.sh

@@ -27,7 +27,7 @@
 #   --fixture <name>           (default: mp4-h264-sdr)
 #   --chunk-counts <list>      (default: 2,4,8)
 #   --psnr-threshold <db>      (default: fixture meta.json minPsnr)
-#   --stack-name <name>        (default: hyperframes-lambda-smoke-<timestamp>)
+#   --stack-name <name>        (default: hyperframes-lambda-smoke-<unique-run-id>)
 #   --region <region>          (default: $AWS_REGION or us-east-1)
 #   --profile <name>           (default: $AWS_PROFILE, otherwise the AWS
 #                               default profile resolution chain)
@@ -78,7 +78,8 @@ CHUNK_COUNTS="${CHUNK_COUNTS:-2,4,8}"
 # calibrated for that content/runtime boundary. Override it via
 # --psnr-threshold (or PSNR_THRESHOLD) for a stricter experiment.
 PSNR_THRESHOLD="${PSNR_THRESHOLD-}"
-STACK_NAME="${STACK_NAME:-hyperframes-lambda-smoke-$(date +%s)}"
+SMOKE_RUN_ID="${HYPERFRAMES_SMOKE_RUN_ID:-$(hf_new_smoke_run_id)}"
+STACK_NAME="${STACK_NAME:-hyperframes-lambda-smoke-${SMOKE_RUN_ID}}"
 AWS_REGION="${AWS_REGION:-us-east-1}"
 AWS_PROFILE="${AWS_PROFILE:-}"
 PLAN_PROTOCOL="${PLAN_PROTOCOL:-v1}"
@@ -106,7 +107,7 @@ Flags:
   --fixture <name>              fixture under packages/producer/tests/distributed/ (default: mp4-h264-sdr)
   --chunk-counts <list>         comma-separated chunk counts to benchmark (default: 2,4,8)
   --psnr-threshold <db>         PSNR floor (default: fixture meta.json minPsnr)
-  --stack-name <name>           SAM stack name (default: hyperframes-lambda-smoke-<timestamp>)
+  --stack-name <name>           SAM stack name (default: hyperframes-lambda-smoke-<unique-run-id>)
   --region <region>             AWS region (default: $AWS_REGION or us-east-1)
   --profile <name>              AWS profile (default: $AWS_PROFILE)
   --plan-protocol <v1|v2|both>  plan transport(s) to compare (default: v1)
@@ -241,42 +242,57 @@ cleanup_and_exit() {
   else
     echo "→ Tearing down stack $STACK_NAME"
     local cleanup_identity_ok=true
+    local stack_cleanup_allowed=false
+    local ownership_status=""
     local discovery_errors=()
     if ! aws sts get-caller-identity >/dev/null; then
       cleanup_identity_ok=false
       echo "ERROR: AWS identity check failed before cleanup; absence cannot be trusted" >&2
     fi
-    local discovered
-    if discovered=$(hf_discover_stack_resources "$STACK_NAME"); then
-      if [ -z "$BUCKET" ]; then
-        BUCKET=$(jq -r '.renderBucket' <<<"$discovered")
+    if [ "$cleanup_identity_ok" = true ]; then
+      if ownership_status=$(hf_stack_ownership_status "$STACK_NAME" "$SMOKE_RUN_ID"); then
+        if [ "$ownership_status" = "owned" ]; then
+          stack_cleanup_allowed=true
+        else
+          echo "→ Stack is absent; skipping stack-scoped destructive cleanup"
+        fi
+      else
+        discovery_errors+=("verification-error:cloudformation-stack-ownership")
       fi
-      if [ -z "$STATE_MACHINE_ARN" ]; then
-        STATE_MACHINE_ARN=$(jq -r '.stateMachineArn' <<<"$discovered")
-      fi
-    else
-      discovery_errors+=("verification-error:cloudformation-resource-discovery")
     fi
-    if [ -n "$BUCKET" ]; then
-      if ! hf_delete_s3_bucket_completely "$BUCKET"; then
-        echo "WARN: failed to purge/delete retained render bucket s3://$BUCKET" >&2
+    if [ "$stack_cleanup_allowed" = true ]; then
+      local discovered
+      if discovered=$(hf_discover_stack_resources "$STACK_NAME"); then
+        if [ -z "$BUCKET" ]; then
+          BUCKET=$(jq -r '.renderBucket' <<<"$discovered")
+        fi
+        if [ -z "$STATE_MACHINE_ARN" ]; then
+          STATE_MACHINE_ARN=$(jq -r '.stateMachineArn' <<<"$discovered")
+        fi
+      else
+        discovery_errors+=("verification-error:cloudformation-resource-discovery")
       fi
-    fi
-    if ! (cd "$SAM_DIR" && sam delete \
-      --stack-name "$STACK_NAME" \
-      --region "$AWS_REGION" \
-      --no-prompts); then
-      echo "WARN: sam delete failed for $STACK_NAME" >&2
-    fi
-    if ! aws cloudformation wait stack-delete-complete --stack-name "$STACK_NAME"; then
-      echo "WARN: CloudFormation did not confirm stack deletion for $STACK_NAME" >&2
-    fi
-    if aws logs describe-log-groups \
-      --log-group-name-prefix "/aws/lambda/${PROJECT_NAME}-render" \
-      --query "logGroups[?logGroupName=='/aws/lambda/${PROJECT_NAME}-render'].logGroupName" \
-      --output text | grep -q .; then
-      if ! aws logs delete-log-group --log-group-name "/aws/lambda/${PROJECT_NAME}-render"; then
-        echo "WARN: failed to delete Lambda log group" >&2
+      if [ -n "$BUCKET" ]; then
+        if ! hf_delete_s3_bucket_completely "$BUCKET"; then
+          echo "WARN: failed to purge/delete retained render bucket s3://$BUCKET" >&2
+        fi
+      fi
+      if ! (cd "$SAM_DIR" && sam delete \
+        --stack-name "$STACK_NAME" \
+        --region "$AWS_REGION" \
+        --no-prompts); then
+        echo "WARN: sam delete failed for $STACK_NAME" >&2
+      fi
+      if ! aws cloudformation wait stack-delete-complete --stack-name "$STACK_NAME"; then
+        echo "WARN: CloudFormation did not confirm stack deletion for $STACK_NAME" >&2
+      fi
+      if aws logs describe-log-groups \
+        --log-group-name-prefix "/aws/lambda/${PROJECT_NAME}-render" \
+        --query "logGroups[?logGroupName=='/aws/lambda/${PROJECT_NAME}-render'].logGroupName" \
+        --output text | grep -q .; then
+        if ! aws logs delete-log-group --log-group-name "/aws/lambda/${PROJECT_NAME}-render"; then
+          echo "WARN: failed to delete Lambda log group" >&2
+        fi
       fi
     fi
     if [ -n "$SAM_DEPLOY_BUCKET" ]; then
@@ -382,18 +398,25 @@ if ! aws sts get-caller-identity --output text >/dev/null 2>&1; then
   exit 1
 fi
 
-# This check runs before cleanup is armed or any resource is created. A name
-# collision must never route through cleanup, because cleanup is destructive.
+# This check runs before cleanup is armed or any resource is created.
 echo "→ Pre-flight: proving exact AWS resource names are unused"
 if ! hf_assert_deploy_isolation "$STACK_NAME" "$PROJECT_NAME"; then
   echo "ERROR: refusing to reuse or clean resources not created by this run." >&2
   exit 1
 fi
 
-# From this point onward, unexpected failures must tear down resources created
-# by this exact, preflight-isolated run.
+# Arm cleanup before atomically reserving the stack name. If another smoke run
+# wins the create-stack race, ownership verification prevents this run from
+# touching it. If this run wins, every later destructive action requires the
+# same ownership tag.
 trap 'cleanup_and_exit $?' EXIT
 
+echo "→ Pre-flight: atomically reserving stack name for this smoke run"
+if ! hf_reserve_smoke_stack "$STACK_NAME" "$SMOKE_RUN_ID"; then
+  echo "ERROR: could not reserve stack name; another run may have won the race." >&2
+  cleanup_and_exit 1
+fi
+
 mkdir -p "$ARTIFACT_DIR/renders"
 
 # ── 1. Build the handler ZIP ──────────────────────────────────────────────
@@ -419,7 +442,7 @@ echo "→ SAM deploy (stack=$STACK_NAME, region=$AWS_REGION)"
 # at its default makes concurrent smoke stacks overwrite/collide. A dedicated
 # SAM bucket also lets teardown remove every object created by this run.
 ACCOUNT_ID=$(aws sts get-caller-identity --query Account --output text)
-SAM_DEPLOY_BUCKET="hf-sam-${ACCOUNT_ID}-${AWS_REGION}-$(date +%s)-$$"
+SAM_DEPLOY_BUCKET=$(hf_sam_deploy_bucket_name "$ACCOUNT_ID" "$AWS_REGION" "$SMOKE_RUN_ID")
 if [ "$AWS_REGION" = "us-east-1" ]; then
   aws s3api create-bucket --bucket "$SAM_DEPLOY_BUCKET" >/dev/null
 else
@@ -434,6 +457,7 @@ if ! (cd "$SAM_DIR" && sam deploy \
         --capabilities CAPABILITY_IAM \
         --no-confirm-changeset \
         --no-fail-on-empty-changeset \
+        --tags "HyperframesSmokeRun=$SMOKE_RUN_ID" \
         --parameter-overrides \
           "ProjectName=$PROJECT_NAME" \
           ChromeSource=sparticuz \