Skip to content

Commit 4eca332

Browse files
CoreyLeath-codeCoreyLeath-code
authored
Merge pull request #22 from CoreyLeath-code/feat/aws-production-hardening
feat: harden AWS integration security and deployment evidence
2 parents d0df3de + c8f0229 commit 4eca332

55 files changed

Lines changed: 2358 additions & 638 deletions

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

‎.env.example‎

Lines changed: 7 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,9 @@
11
API_PORT=8000
22
KAFKA_BROKER=localhost:9092
3-
DB_URL=postgres://user:password@localhost:5432/rides
4-
REDIS_URL=redis://localhost:6379
3+
DRIVER_LOCATION_REDIS_URL=redis://localhost:6379/0
4+
DRIVER_LOCATION_REDIS_SECRET_ID=
5+
AUTH_REQUIRED=false
6+
AUTH_ISSUER=
7+
AUTH_AUDIENCE=
8+
AUTH_JWKS_URL=
9+
AUTH_TOKEN_USE=access

‎.github/workflows/aws-msk-integration.yml‎

Lines changed: 109 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,109 @@
1+
name: Real AWS MSK Integration Evidence
2+
3+
on:
4+
workflow_dispatch:
5+
inputs:
6+
aws_region:
7+
description: AWS region containing the development VPC
8+
required: true
9+
default: us-east-1
10+
type: string
11+
vpc_id:
12+
description: Development VPC ID
13+
required: true
14+
type: string
15+
subnet_ids_json:
16+
description: JSON array of at least two private subnet IDs
17+
required: true
18+
type: string
19+
samples:
20+
description: Number of sequential end-to-end probes
21+
required: true
22+
default: "20"
23+
type: string
24+
destroy_after:
25+
description: Destroy the temporary MSK integration environment after evidence collection
26+
required: true
27+
default: true
28+
type: boolean
29+
30+
permissions:
31+
contents: read
32+
id-token: write
33+
34+
jobs:
35+
real-msk-evidence:
36+
name: Real MSK end-to-end evidence
37+
runs-on: [self-hosted, linux, aws-msk-dev]
38+
environment: development
39+
timeout-minutes: 90
40+
env:
41+
AWS_REGION: ${{ inputs.aws_region }}
42+
TF_VAR_aws_region: ${{ inputs.aws_region }}
43+
TF_VAR_create_dev_msk_cluster: "true"
44+
TF_VAR_enable_integration_probe: "true"
45+
TF_VAR_dev_msk_vpc_id: ${{ inputs.vpc_id }}
46+
TF_VAR_dev_msk_subnet_ids: ${{ inputs.subnet_ids_json }}
47+
LAMBDA_REQUESTS_PER_MILLION_USD: ${{ vars.LAMBDA_REQUESTS_PER_MILLION_USD }}
48+
SQS_REQUESTS_PER_MILLION_USD: ${{ vars.SQS_REQUESTS_PER_MILLION_USD }}
49+
SNS_PUBLISHES_PER_MILLION_USD: ${{ vars.SNS_PUBLISHES_PER_MILLION_USD }}
50+
MSK_CLUSTER_HOURLY_USD: ${{ vars.MSK_CLUSTER_HOURLY_USD }}
51+
52+
steps:
53+
- name: Checkout repository
54+
uses: actions/checkout@v4
55+
56+
- name: Configure AWS credentials with OIDC
57+
uses: aws-actions/configure-aws-credentials@v4
58+
with:
59+
role-to-assume: ${{ secrets.AWS_MSK_INTEGRATION_ROLE_ARN }}
60+
aws-region: ${{ inputs.aws_region }}
61+
62+
- name: Set up Terraform
63+
uses: hashicorp/setup-terraform@v3
64+
with:
65+
terraform_version: "1.9.8"
66+
67+
- name: Set up Python
68+
uses: actions/setup-python@v5
69+
with:
70+
python-version: "3.11"
71+
72+
- name: Install integration dependencies
73+
run: python -m pip install -r requirements.txt -r requirements-dev.txt
74+
75+
- name: Apply development integration environment
76+
working-directory: infra/aws
77+
run: |
78+
terraform init
79+
terraform apply -auto-approve
80+
81+
- name: Allow event source mapping to become active
82+
run: sleep 60
83+
84+
- name: Capture Terraform outputs
85+
id: terraform
86+
working-directory: infra/aws
87+
run: |
88+
echo "cluster_arn=$(terraform output -raw effective_msk_cluster_arn)" >> "$GITHUB_OUTPUT"
89+
echo "probe_queue_url=$(terraform output -raw integration_probe_queue_url)" >> "$GITHUB_OUTPUT"
90+
91+
- name: Measure real end-to-end path
92+
run: |
93+
python scripts/aws_msk_e2e.py \
94+
--cluster-arn '${{ steps.terraform.outputs.cluster_arn }}' \
95+
--probe-queue-url '${{ steps.terraform.outputs.probe_queue_url }}' \
96+
--samples '${{ inputs.samples }}' \
97+
--output evidence/aws-msk-integration-results.json
98+
99+
- name: Upload measured evidence
100+
uses: actions/upload-artifact@v4
101+
with:
102+
name: aws-msk-integration-evidence-${{ github.sha }}
103+
path: evidence/aws-msk-integration-results.json
104+
if-no-files-found: error
105+
106+
- name: Destroy temporary integration environment
107+
if: ${{ always() && inputs.destroy_after }}
108+
working-directory: infra/aws
109+
run: terraform destroy -auto-approve

‎.github/workflows/aws-serverless.yml‎

Lines changed: 16 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -5,22 +5,30 @@ on:
55
paths:
66
- "serverless/**"
77
- "infra/aws/**"
8+
- "scripts/aws_msk_e2e.py"
89
- "tests/test_aws_lambda_handlers.py"
10+
- "tests/test_aws_security_boundaries.py"
11+
- "tests/test_release_evidence_tools.py"
912
- ".github/workflows/aws-serverless.yml"
13+
- ".github/workflows/aws-msk-integration.yml"
1014
push:
1115
branches: ["main"]
1216
paths:
1317
- "serverless/**"
1418
- "infra/aws/**"
19+
- "scripts/aws_msk_e2e.py"
1520
- "tests/test_aws_lambda_handlers.py"
21+
- "tests/test_aws_security_boundaries.py"
22+
- "tests/test_release_evidence_tools.py"
1623
- ".github/workflows/aws-serverless.yml"
24+
- ".github/workflows/aws-msk-integration.yml"
1725

1826
permissions:
1927
contents: read
2028

2129
jobs:
22-
lambda-unit-tests:
23-
name: Lambda unit tests
30+
aws-unit-tests:
31+
name: AWS credential-free unit tests
2432
runs-on: ubuntu-latest
2533

2634
steps:
@@ -41,8 +49,12 @@ jobs:
4149
python -m pip install --upgrade pip
4250
python -m pip install -r requirements.txt -r requirements-dev.txt
4351
44-
- name: Run Lambda handler tests
45-
run: pytest -q tests/test_aws_lambda_handlers.py
52+
- name: Run AWS and evidence unit tests
53+
run: |
54+
pytest -q \
55+
tests/test_aws_lambda_handlers.py \
56+
tests/test_aws_security_boundaries.py \
57+
tests/test_release_evidence_tools.py
4658
4759
terraform-validate:
4860
name: Terraform format and validate

‎.github/workflows/kubernetes-promotion.yml‎

Lines changed: 130 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,130 @@
1+
name: Kubernetes Promotion Evidence
2+
3+
on:
4+
workflow_dispatch:
5+
inputs:
6+
environment:
7+
description: Target environment
8+
required: true
9+
type: choice
10+
options: [dev, staging, prod]
11+
image:
12+
description: Full image reference; production requires @sha256 digest
13+
required: true
14+
type: string
15+
deploy:
16+
description: Apply to the configured cluster instead of render-only evidence
17+
required: true
18+
default: false
19+
type: boolean
20+
21+
permissions:
22+
contents: read
23+
24+
jobs:
25+
promote:
26+
runs-on: ubuntu-latest
27+
environment: ${{ inputs.environment }}
28+
steps:
29+
- uses: actions/checkout@v4
30+
31+
- name: Set up Kustomize
32+
uses: imranismail/setup-kustomize@v2
33+
34+
- name: Set up kubectl
35+
uses: azure/setup-kubectl@v4
36+
37+
- name: Render immutable candidate manifest
38+
env:
39+
TARGET_ENV: ${{ inputs.environment }}
40+
IMAGE: ${{ inputs.image }}
41+
run: |
42+
set -euo pipefail
43+
cd "infra/kubernetes/overlays/${TARGET_ENV}"
44+
kustomize edit set image "api-gateway=${IMAGE}"
45+
kustomize build . > "$RUNNER_TEMP/rendered.yaml"
46+
grep -n "image:" "$RUNNER_TEMP/rendered.yaml"
47+
48+
- name: Enforce promotion policy before apply
49+
env:
50+
TARGET_ENV: ${{ inputs.environment }}
51+
IMAGE: ${{ inputs.image }}
52+
DEPLOY: ${{ inputs.deploy }}
53+
run: |
54+
set -euo pipefail
55+
if [[ "$TARGET_ENV" == "prod" && "$IMAGE" != *@sha256:* ]]; then
56+
echo "Production promotion requires an immutable @sha256 image digest." >&2
57+
exit 1
58+
fi
59+
if [[ "$DEPLOY" == "true" && "$TARGET_ENV" != "dev" ]] && \
60+
grep -q "replace-me" "$RUNNER_TEMP/rendered.yaml"; then
61+
echo "Staging/prod OIDC placeholders must be replaced before an actual deployment." >&2
62+
exit 1
63+
fi
64+
65+
- name: Client-side manifest validation
66+
run: kubectl apply --dry-run=client --validate=false -f "$RUNNER_TEMP/rendered.yaml"
67+
68+
- name: Configure cluster credentials
69+
if: ${{ inputs.deploy }}
70+
env:
71+
KUBE_CONFIG_DATA: ${{ secrets.KUBE_CONFIG_DATA }}
72+
run: |
73+
test -n "$KUBE_CONFIG_DATA"
74+
printf '%s' "$KUBE_CONFIG_DATA" | base64 --decode > "$RUNNER_TEMP/kubeconfig"
75+
echo "KUBECONFIG=$RUNNER_TEMP/kubeconfig" >> "$GITHUB_ENV"
76+
77+
- name: Capture current revision
78+
if: ${{ inputs.deploy }}
79+
id: before
80+
run: |
81+
revision=$(kubectl rollout history deployment/api-gateway | awk 'NR>2 && $1 ~ /^[0-9]+$/ {print $1}' | tail -n 1)
82+
echo "revision=${revision:-unknown}" >> "$GITHUB_OUTPUT"
83+
84+
- name: Apply and verify rollout
85+
if: ${{ inputs.deploy }}
86+
run: |
87+
kubectl apply -f "$RUNNER_TEMP/rendered.yaml"
88+
kubectl rollout status deployment/api-gateway --timeout=180s
89+
90+
- name: Capture promoted revision
91+
if: ${{ inputs.deploy }}
92+
id: after
93+
run: |
94+
revision=$(kubectl rollout history deployment/api-gateway | awk 'NR>2 && $1 ~ /^[0-9]+$/ {print $1}' | tail -n 1)
95+
echo "revision=${revision:-unknown}" >> "$GITHUB_OUTPUT"
96+
97+
- name: Record render-only evidence
98+
if: ${{ !inputs.deploy }}
99+
run: |
100+
python scripts/k8s_release_evidence.py \
101+
--action promote \
102+
--environment '${{ inputs.environment }}' \
103+
--image '${{ inputs.image }}' \
104+
--git-sha '${{ github.sha }}' \
105+
--manifest "$RUNNER_TEMP/rendered.yaml" \
106+
--deployment-status rendered \
107+
--output evidence/kubernetes-deployment-results.json
108+
109+
- name: Record applied promotion evidence
110+
if: ${{ inputs.deploy }}
111+
run: |
112+
python scripts/k8s_release_evidence.py \
113+
--action promote \
114+
--environment '${{ inputs.environment }}' \
115+
--image '${{ inputs.image }}' \
116+
--git-sha '${{ github.sha }}' \
117+
--manifest "$RUNNER_TEMP/rendered.yaml" \
118+
--deployment-status applied \
119+
--from-revision '${{ steps.before.outputs.revision }}' \
120+
--to-revision '${{ steps.after.outputs.revision }}' \
121+
--output evidence/kubernetes-deployment-results.json
122+
123+
- name: Upload promotion evidence
124+
uses: actions/upload-artifact@v4
125+
with:
126+
name: kubernetes-promotion-${{ inputs.environment }}-${{ github.sha }}
127+
path: |
128+
${{ runner.temp }}/rendered.yaml
129+
evidence/kubernetes-deployment-results.json
130+
if-no-files-found: error

‎.github/workflows/kubernetes-rollback.yml‎

Lines changed: 75 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,75 @@
1+
name: Kubernetes Rollback Evidence
2+
3+
on:
4+
workflow_dispatch:
5+
inputs:
6+
environment:
7+
description: Environment to roll back
8+
required: true
9+
type: choice
10+
options: [dev, staging, prod]
11+
revision:
12+
description: Deployment revision to restore
13+
required: true
14+
type: string
15+
16+
permissions:
17+
contents: read
18+
19+
jobs:
20+
rollback:
21+
runs-on: ubuntu-latest
22+
environment: ${{ inputs.environment }}
23+
steps:
24+
- uses: actions/checkout@v4
25+
26+
- name: Set up kubectl
27+
uses: azure/setup-kubectl@v4
28+
29+
- name: Configure cluster credentials
30+
env:
31+
KUBE_CONFIG_DATA: ${{ secrets.KUBE_CONFIG_DATA }}
32+
run: |
33+
test -n "$KUBE_CONFIG_DATA"
34+
printf '%s' "$KUBE_CONFIG_DATA" | base64 --decode > "$RUNNER_TEMP/kubeconfig"
35+
echo "KUBECONFIG=$RUNNER_TEMP/kubeconfig" >> "$GITHUB_ENV"
36+
37+
- name: Capture current revision
38+
id: before
39+
run: |
40+
revision=$(kubectl rollout history deployment/api-gateway | awk 'NR>2 && $1 ~ /^[0-9]+$/ {print $1}' | tail -n 1)
41+
echo "revision=${revision:-unknown}" >> "$GITHUB_OUTPUT"
42+
43+
- name: Roll back and verify
44+
run: |
45+
kubectl rollout undo deployment/api-gateway --to-revision='${{ inputs.revision }}'
46+
kubectl rollout status deployment/api-gateway --timeout=180s
47+
kubectl get deployment api-gateway -o yaml > "$RUNNER_TEMP/rollback-deployment.yaml"
48+
49+
- name: Capture rollback image
50+
id: image
51+
run: |
52+
image=$(kubectl get deployment api-gateway -o jsonpath='{.spec.template.spec.containers[?(@.name=="api-gateway")].image}')
53+
echo "image=$image" >> "$GITHUB_OUTPUT"
54+
55+
- name: Record rollback evidence
56+
run: |
57+
python scripts/k8s_release_evidence.py \
58+
--action rollback \
59+
--environment '${{ inputs.environment }}' \
60+
--image '${{ steps.image.outputs.image }}' \
61+
--git-sha '${{ github.sha }}' \
62+
--manifest "$RUNNER_TEMP/rollback-deployment.yaml" \
63+
--deployment-status applied \
64+
--from-revision '${{ steps.before.outputs.revision }}' \
65+
--to-revision '${{ inputs.revision }}' \
66+
--output evidence/kubernetes-deployment-results.json
67+
68+
- name: Upload rollback evidence
69+
uses: actions/upload-artifact@v4
70+
with:
71+
name: kubernetes-rollback-${{ inputs.environment }}-${{ github.sha }}
72+
path: |
73+
${{ runner.temp }}/rollback-deployment.yaml
74+
evidence/kubernetes-deployment-results.json
75+
if-no-files-found: error

0 commit comments

Comments
 (0)