feat: harden AWS integration security and deployment evidence

.env.example

@@ -1,4 +1,9 @@
 API_PORT=8000
 KAFKA_BROKER=localhost:9092
-DB_URL=postgres://user:password@localhost:5432/rides
-REDIS_URL=redis://localhost:6379
+DRIVER_LOCATION_REDIS_URL=redis://localhost:6379/0
+DRIVER_LOCATION_REDIS_SECRET_ID=
+AUTH_REQUIRED=false
+AUTH_ISSUER=
+AUTH_AUDIENCE=
+AUTH_JWKS_URL=
+AUTH_TOKEN_USE=access

.github/workflows/aws-msk-integration.yml

@@ -0,0 +1,109 @@
+name: Real AWS MSK Integration Evidence
+
+on:
+  workflow_dispatch:
+    inputs:
+      aws_region:
+        description: AWS region containing the development VPC
+        required: true
+        default: us-east-1
+        type: string
+      vpc_id:
+        description: Development VPC ID
+        required: true
+        type: string
+      subnet_ids_json:
+        description: JSON array of at least two private subnet IDs
+        required: true
+        type: string
+      samples:
+        description: Number of sequential end-to-end probes
+        required: true
+        default: "20"
+        type: string
+      destroy_after:
+        description: Destroy the temporary MSK integration environment after evidence collection
+        required: true
+        default: true
+        type: boolean
+
+permissions:
+  contents: read
+  id-token: write
+
+jobs:
+  real-msk-evidence:
+    name: Real MSK end-to-end evidence
+    runs-on: [self-hosted, linux, aws-msk-dev]
+    environment: development
+    timeout-minutes: 90
+    env:
+      AWS_REGION: ${{ inputs.aws_region }}
+      TF_VAR_aws_region: ${{ inputs.aws_region }}
+      TF_VAR_create_dev_msk_cluster: "true"
+      TF_VAR_enable_integration_probe: "true"
+      TF_VAR_dev_msk_vpc_id: ${{ inputs.vpc_id }}
+      TF_VAR_dev_msk_subnet_ids: ${{ inputs.subnet_ids_json }}
+      LAMBDA_REQUESTS_PER_MILLION_USD: ${{ vars.LAMBDA_REQUESTS_PER_MILLION_USD }}
+      SQS_REQUESTS_PER_MILLION_USD: ${{ vars.SQS_REQUESTS_PER_MILLION_USD }}
+      SNS_PUBLISHES_PER_MILLION_USD: ${{ vars.SNS_PUBLISHES_PER_MILLION_USD }}
+      MSK_CLUSTER_HOURLY_USD: ${{ vars.MSK_CLUSTER_HOURLY_USD }}
+
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+
+      - name: Configure AWS credentials with OIDC
+        uses: aws-actions/configure-aws-credentials@v4
+        with:
+          role-to-assume: ${{ secrets.AWS_MSK_INTEGRATION_ROLE_ARN }}
+          aws-region: ${{ inputs.aws_region }}
+
+      - name: Set up Terraform
+        uses: hashicorp/setup-terraform@v3
+        with:
+          terraform_version: "1.9.8"
+
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: "3.11"
+
+      - name: Install integration dependencies
+        run: python -m pip install -r requirements.txt -r requirements-dev.txt
+
+      - name: Apply development integration environment
+        working-directory: infra/aws
+        run: |
+          terraform init
+          terraform apply -auto-approve
+
+      - name: Allow event source mapping to become active
+        run: sleep 60
+
+      - name: Capture Terraform outputs
+        id: terraform
+        working-directory: infra/aws
+        run: |
+          echo "cluster_arn=$(terraform output -raw effective_msk_cluster_arn)" >> "$GITHUB_OUTPUT"
+          echo "probe_queue_url=$(terraform output -raw integration_probe_queue_url)" >> "$GITHUB_OUTPUT"
+
+      - name: Measure real end-to-end path
+        run: |
+          python scripts/aws_msk_e2e.py \
+            --cluster-arn '${{ steps.terraform.outputs.cluster_arn }}' \
+            --probe-queue-url '${{ steps.terraform.outputs.probe_queue_url }}' \
+            --samples '${{ inputs.samples }}' \
+            --output evidence/aws-msk-integration-results.json
+
+      - name: Upload measured evidence
+        uses: actions/upload-artifact@v4
+        with:
+          name: aws-msk-integration-evidence-${{ github.sha }}
+          path: evidence/aws-msk-integration-results.json
+          if-no-files-found: error
+
+      - name: Destroy temporary integration environment
+        if: ${{ always() && inputs.destroy_after }}
+        working-directory: infra/aws
+        run: terraform destroy -auto-approve

.github/workflows/aws-serverless.yml

@@ -5,22 +5,30 @@ on:
     paths:
       - "serverless/**"
       - "infra/aws/**"
+      - "scripts/aws_msk_e2e.py"
       - "tests/test_aws_lambda_handlers.py"
+      - "tests/test_aws_security_boundaries.py"
+      - "tests/test_release_evidence_tools.py"
       - ".github/workflows/aws-serverless.yml"
+      - ".github/workflows/aws-msk-integration.yml"
   push:
     branches: ["main"]
     paths:
       - "serverless/**"
       - "infra/aws/**"
+      - "scripts/aws_msk_e2e.py"
       - "tests/test_aws_lambda_handlers.py"
+      - "tests/test_aws_security_boundaries.py"
+      - "tests/test_release_evidence_tools.py"
       - ".github/workflows/aws-serverless.yml"
+      - ".github/workflows/aws-msk-integration.yml"
 
 permissions:
   contents: read
 
 jobs:
-  lambda-unit-tests:
-    name: Lambda unit tests
+  aws-unit-tests:
+    name: AWS credential-free unit tests
     runs-on: ubuntu-latest
 
     steps:
@@ -41,8 +49,12 @@ jobs:
           python -m pip install --upgrade pip
           python -m pip install -r requirements.txt -r requirements-dev.txt
 
-      - name: Run Lambda handler tests
-        run: pytest -q tests/test_aws_lambda_handlers.py
+      - name: Run AWS and evidence unit tests
+        run: |
+          pytest -q \
+            tests/test_aws_lambda_handlers.py \
+            tests/test_aws_security_boundaries.py \
+            tests/test_release_evidence_tools.py
 
   terraform-validate:
     name: Terraform format and validate

.github/workflows/kubernetes-promotion.yml

@@ -0,0 +1,130 @@
+name: Kubernetes Promotion Evidence
+
+on:
+  workflow_dispatch:
+    inputs:
+      environment:
+        description: Target environment
+        required: true
+        type: choice
+        options: [dev, staging, prod]
+      image:
+        description: Full image reference; production requires @sha256 digest
+        required: true
+        type: string
+      deploy:
+        description: Apply to the configured cluster instead of render-only evidence
+        required: true
+        default: false
+        type: boolean
+
+permissions:
+  contents: read
+
+jobs:
+  promote:
+    runs-on: ubuntu-latest
+    environment: ${{ inputs.environment }}
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Set up Kustomize
+        uses: imranismail/setup-kustomize@v2
+
+      - name: Set up kubectl
+        uses: azure/setup-kubectl@v4
+
+      - name: Render immutable candidate manifest
+        env:
+          TARGET_ENV: ${{ inputs.environment }}
+          IMAGE: ${{ inputs.image }}
+        run: |
+          set -euo pipefail
+          cd "infra/kubernetes/overlays/${TARGET_ENV}"
+          kustomize edit set image "api-gateway=${IMAGE}"
+          kustomize build . > "$RUNNER_TEMP/rendered.yaml"
+          grep -n "image:" "$RUNNER_TEMP/rendered.yaml"
+
+      - name: Enforce promotion policy before apply
+        env:
+          TARGET_ENV: ${{ inputs.environment }}
+          IMAGE: ${{ inputs.image }}
+          DEPLOY: ${{ inputs.deploy }}
+        run: |
+          set -euo pipefail
+          if [[ "$TARGET_ENV" == "prod" && "$IMAGE" != *@sha256:* ]]; then
+            echo "Production promotion requires an immutable @sha256 image digest." >&2
+            exit 1
+          fi
+          if [[ "$DEPLOY" == "true" && "$TARGET_ENV" != "dev" ]] && \
+             grep -q "replace-me" "$RUNNER_TEMP/rendered.yaml"; then
+            echo "Staging/prod OIDC placeholders must be replaced before an actual deployment." >&2
+            exit 1
+          fi
+
+      - name: Client-side manifest validation
+        run: kubectl apply --dry-run=client --validate=false -f "$RUNNER_TEMP/rendered.yaml"
+
+      - name: Configure cluster credentials
+        if: ${{ inputs.deploy }}
+        env:
+          KUBE_CONFIG_DATA: ${{ secrets.KUBE_CONFIG_DATA }}
+        run: |
+          test -n "$KUBE_CONFIG_DATA"
+          printf '%s' "$KUBE_CONFIG_DATA" | base64 --decode > "$RUNNER_TEMP/kubeconfig"
+          echo "KUBECONFIG=$RUNNER_TEMP/kubeconfig" >> "$GITHUB_ENV"
+
+      - name: Capture current revision
+        if: ${{ inputs.deploy }}
+        id: before
+        run: |
+          revision=$(kubectl rollout history deployment/api-gateway | awk 'NR>2 && $1 ~ /^[0-9]+$/ {print $1}' | tail -n 1)
+          echo "revision=${revision:-unknown}" >> "$GITHUB_OUTPUT"
+
+      - name: Apply and verify rollout
+        if: ${{ inputs.deploy }}
+        run: |
+          kubectl apply -f "$RUNNER_TEMP/rendered.yaml"
+          kubectl rollout status deployment/api-gateway --timeout=180s
+
+      - name: Capture promoted revision
+        if: ${{ inputs.deploy }}
+        id: after
+        run: |
+          revision=$(kubectl rollout history deployment/api-gateway | awk 'NR>2 && $1 ~ /^[0-9]+$/ {print $1}' | tail -n 1)
+          echo "revision=${revision:-unknown}" >> "$GITHUB_OUTPUT"
+
+      - name: Record render-only evidence
+        if: ${{ !inputs.deploy }}
+        run: |
+          python scripts/k8s_release_evidence.py \
+            --action promote \
+            --environment '${{ inputs.environment }}' \
+            --image '${{ inputs.image }}' \
+            --git-sha '${{ github.sha }}' \
+            --manifest "$RUNNER_TEMP/rendered.yaml" \
+            --deployment-status rendered \
+            --output evidence/kubernetes-deployment-results.json
+
+      - name: Record applied promotion evidence
+        if: ${{ inputs.deploy }}
+        run: |
+          python scripts/k8s_release_evidence.py \
+            --action promote \
+            --environment '${{ inputs.environment }}' \
+            --image '${{ inputs.image }}' \
+            --git-sha '${{ github.sha }}' \
+            --manifest "$RUNNER_TEMP/rendered.yaml" \
+            --deployment-status applied \
+            --from-revision '${{ steps.before.outputs.revision }}' \
+            --to-revision '${{ steps.after.outputs.revision }}' \
+            --output evidence/kubernetes-deployment-results.json
+
+      - name: Upload promotion evidence
+        uses: actions/upload-artifact@v4
+        with:
+          name: kubernetes-promotion-${{ inputs.environment }}-${{ github.sha }}
+          path: |
+            ${{ runner.temp }}/rendered.yaml
+            evidence/kubernetes-deployment-results.json
+          if-no-files-found: error

.github/workflows/kubernetes-rollback.yml

@@ -0,0 +1,75 @@
+name: Kubernetes Rollback Evidence
+
+on:
+  workflow_dispatch:
+    inputs:
+      environment:
+        description: Environment to roll back
+        required: true
+        type: choice
+        options: [dev, staging, prod]
+      revision:
+        description: Deployment revision to restore
+        required: true
+        type: string
+
+permissions:
+  contents: read
+
+jobs:
+  rollback:
+    runs-on: ubuntu-latest
+    environment: ${{ inputs.environment }}
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Set up kubectl
+        uses: azure/setup-kubectl@v4
+
+      - name: Configure cluster credentials
+        env:
+          KUBE_CONFIG_DATA: ${{ secrets.KUBE_CONFIG_DATA }}
+        run: |
+          test -n "$KUBE_CONFIG_DATA"
+          printf '%s' "$KUBE_CONFIG_DATA" | base64 --decode > "$RUNNER_TEMP/kubeconfig"
+          echo "KUBECONFIG=$RUNNER_TEMP/kubeconfig" >> "$GITHUB_ENV"
+
+      - name: Capture current revision
+        id: before
+        run: |
+          revision=$(kubectl rollout history deployment/api-gateway | awk 'NR>2 && $1 ~ /^[0-9]+$/ {print $1}' | tail -n 1)
+          echo "revision=${revision:-unknown}" >> "$GITHUB_OUTPUT"
+
+      - name: Roll back and verify
+        run: |
+          kubectl rollout undo deployment/api-gateway --to-revision='${{ inputs.revision }}'
+          kubectl rollout status deployment/api-gateway --timeout=180s
+          kubectl get deployment api-gateway -o yaml > "$RUNNER_TEMP/rollback-deployment.yaml"
+
+      - name: Capture rollback image
+        id: image
+        run: |
+          image=$(kubectl get deployment api-gateway -o jsonpath='{.spec.template.spec.containers[?(@.name=="api-gateway")].image}')
+          echo "image=$image" >> "$GITHUB_OUTPUT"
+
+      - name: Record rollback evidence
+        run: |
+          python scripts/k8s_release_evidence.py \
+            --action rollback \
+            --environment '${{ inputs.environment }}' \
+            --image '${{ steps.image.outputs.image }}' \
+            --git-sha '${{ github.sha }}' \
+            --manifest "$RUNNER_TEMP/rollback-deployment.yaml" \
+            --deployment-status applied \
+            --from-revision '${{ steps.before.outputs.revision }}' \
+            --to-revision '${{ inputs.revision }}' \
+            --output evidence/kubernetes-deployment-results.json
+
+      - name: Upload rollback evidence
+        uses: actions/upload-artifact@v4
+        with:
+          name: kubernetes-rollback-${{ inputs.environment }}-${{ github.sha }}
+          path: |
+            ${{ runner.temp }}/rollback-deployment.yaml
+            evidence/kubernetes-deployment-results.json
+          if-no-files-found: error