diff --git a/.github/workflows/simulation-deploy.yml b/.github/workflows/simulation-deploy.yml index 5e2a9f533..a4f33808f 100644 --- a/.github/workflows/simulation-deploy.yml +++ b/.github/workflows/simulation-deploy.yml @@ -99,7 +99,12 @@ jobs: entrypoint_revision_prefix: s- entrypoint_min_instances: 0 entrypoint_max_instances: 2 - promote_entrypoint: false + # Staging's stable URL must serve the revision this deployment tested, + # because staging callers such as the Public API's staging service use + # that URL. Only a run from main promotes: a manual run from another + # branch tests its candidate but leaves stable traffic and older + # revisions alone. Rule: sim.deploy.staging-promotes-tested-revision. + promote_entrypoint: ${{ github.ref == 'refs/heads/main' }} run_full_integration: true deploy_existing_stack: true deploy_stage12_v2: true diff --git a/.github/workflows/stage12-deploy.yml b/.github/workflows/stage12-deploy.yml index cecb6b097..a5b252e3f 100644 --- a/.github/workflows/stage12-deploy.yml +++ b/.github/workflows/stage12-deploy.yml @@ -53,7 +53,9 @@ jobs: entrypoint_revision_prefix: s- entrypoint_min_instances: 0 entrypoint_max_instances: 2 - promote_entrypoint: false + # Like the main deployment, promote the tested staging revision, and + # only from main. Rule: sim.deploy.staging-promotes-tested-revision. + promote_entrypoint: ${{ github.ref == 'refs/heads/main' }} # This workflow deploys and validates only the independent Stage 12 path. # The main API deployment continues to run the existing-stack suite. run_full_integration: false diff --git a/AGENTS.md b/AGENTS.md index d173f588a..40cf004b1 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -132,7 +132,7 @@ To add or change a rule, read `docs/engineering/skills/rules-registry.md`. ### Deployment pipeline -- Checked: `sim.deploy.container-context-minimal`, `sim.deploy.docker-images-build`, `sim.deploy.gcloud-without-terraform`, `sim.deploy.modal-image-smoke`, `sim.deploy.numerical-sentinel-gate`, `sim.deploy.permission-enforcement-wired`, `sim.deploy.promotion-order-explicit`, `sim.deploy.release-commit-deploys-once`, `sim.deploy.release-identity-smoke`, `sim.deploy.revision-tags-valid`, `sim.deploy.routing-publication-journaled`, `sim.deploy.shell-scripts-valid`, `sim.deploy.smoke-tests-strict`, `sim.deploy.traffic-changes-exact`, `sim.deploy.workflow-shell-safety` +- Checked: `sim.deploy.container-context-minimal`, `sim.deploy.docker-images-build`, `sim.deploy.gcloud-without-terraform`, `sim.deploy.modal-image-smoke`, `sim.deploy.numerical-sentinel-gate`, `sim.deploy.permission-enforcement-wired`, `sim.deploy.promotion-order-explicit`, `sim.deploy.release-commit-deploys-once`, `sim.deploy.release-identity-smoke`, `sim.deploy.revision-tags-valid`, `sim.deploy.routing-publication-journaled`, `sim.deploy.shell-scripts-valid`, `sim.deploy.smoke-tests-strict`, `sim.deploy.staging-promotes-tested-revision`, `sim.deploy.traffic-changes-exact`, `sim.deploy.workflow-shell-safety` - `sim.deploy.image-smoke-covers-every-image-lib`: The image smoke path filter includes every library mounted into a Modal image, so a change to any of them runs the smoke. (doc-only, not checked: follow it yourself) ### API contracts and integration diff --git a/README.md b/README.md index f34ebaa49..308e475fb 100644 --- a/README.md +++ b/README.md @@ -96,13 +96,21 @@ push deploys one complete stack at a time (see three parallel service jobs; 3. publish routing only after all three service jobs pass; 4. run generated-client and live authentication tests through the complete - three-service request path; and -5. begin the `prod` deployment only after every `beta` deployment and - qualification job succeeds; -6. assign stable production traffic to the exact tested entrypoint revision - after the complete production suite passes; and -7. verify the stable endpoint, restoring its previous revision if immediate - post-promotion checks fail. + three-service request path; +5. assign the environment's stable traffic to the exact tested entrypoint + revision after that environment's tests pass, in `beta` as in `prod` + (`beta` only for a run from `main`); +6. verify the stable endpoint, restoring its previous revision if immediate + post-promotion checks fail; and +7. begin the `prod` deployment only after every `beta` deployment, + qualification, promotion and revision clean-up job succeeds, so a failed + staging clean-up also stops the production deployment. + +The staging service's stable URL therefore serves the revision that passed the +staging tests, so staging callers of that URL, such as the Public API's staging +service, reach the build the deployment tested. After each successful +promotion, the deployment deletes the service's older revisions, keeping only +the promoted revision and the one it replaced, for rollback. A successful deployment of a release commit publishes the API client from the exact deployed commit. A manual dispatch of the workflow deploys without a diff --git a/changelog.d/893.changed.md b/changelog.d/893.changed.md new file mode 100644 index 000000000..72ccab588 --- /dev/null +++ b/changelog.d/893.changed.md @@ -0,0 +1 @@ +Promoted each staging deployment's tested Simulation Entrypoint revision to the staging service's stable traffic, as production deployments already do, in both the main deployment's beta job and the Stage 12 comparison path's staging deployment. The staging stable URL now serves the build that passed the staging tests; promotion runs the same stable-URL checks as production's and restores the previous revision when they fail, and the existing clean-up then deletes the staging service's older revisions except the promoted one and the one it replaced. Only a run from main promotes: a manual staging run from another branch tests its candidate but leaves the staging service's Cloud Run traffic and older revisions unchanged. The new rule sim.deploy.staging-promotes-tested-revision requires it. diff --git a/docs/generated/rules.md b/docs/generated/rules.md index d9b80a709..53cdb5b83 100644 --- a/docs/generated/rules.md +++ b/docs/generated/rules.md @@ -8,11 +8,11 @@ Every rule that governs `policyengine-sim-api` is listed here with how it is enf | Tier | enforced | ratcheted | advisory | doc-only | Total | | --- | ---: | ---: | ---: | ---: | ---: | -| mechanical | 56 | 2 | 0 | 14 | 72 | +| mechanical | 57 | 2 | 0 | 14 | 73 | | metric | 1 | 0 | 0 | 0 | 1 | | judgment | 0 | 0 | 0 | 0 | 0 | -**Blocking:** 59 of 73 rules. +**Blocking:** 60 of 74 rules. ## Gaps @@ -156,6 +156,7 @@ flowchart LR | `sim.deploy.routing-publication-journaled` | Each routing publication is named by its run, attempt and environment and journaled before it is applied; an always-running finalizer resolves it from the journal with the release-gate outcome, a release reports success only after that finalizer completes, and routing jobs never tolerate a failed command. | mechanical | enforced | yes | pytest `projects/policyengine-simulation-entry/tests/test_deployment_assets_release_gates.py`, `projects/policyengine-simulation-entry/tests/test_deployment_assets_routing.py` | `pr.yml:test` | **unproven** | | `sim.deploy.shell-scripts-valid` | Deployment shell scripts have valid shell syntax. | mechanical | enforced | yes | pytest `projects/policyengine-simulation-entry/tests/test_deployment_assets.py` | `pr.yml:test` | **unproven** | | `sim.deploy.smoke-tests-strict` | Candidate and stable smoke tests reject a revision mismatch without retrying, and fail if the previous revision does not converge. | mechanical | enforced | yes | pytest `projects/policyengine-simulation-entry/tests/test_deployment_assets.py` | `pr.yml:test` | **unproven** | +| `sim.deploy.staging-promotes-tested-revision` | Every staging deployment passes promote_entrypoint: ${{ github.ref == 'refs/heads/main' }}, so a run from main moves the staging service's stable Cloud Run traffic to the entrypoint revision that passed the staging tests, and an accidental manual run from another branch neither moves that traffic nor deletes older revisions. | mechanical | enforced | yes | pytest `projects/policyengine-simulation-entry/tests/test_deployment_assets_promotion.py` | `pr.yml:test` | yes | | `sim.deploy.traffic-changes-exact` | Traffic changes target an exact, validated revision, are reversible, and refuse to run over an intervening promotion. | mechanical | enforced | yes | pytest `projects/policyengine-simulation-entry/tests/test_deployment_assets_traffic.py` | `pr.yml:test` | **unproven** | | `sim.deploy.workflow-shell-safety` | Cloud Run deployment never interpolates GitHub expressions into shell, escapes environment values, and pins secret versions. | mechanical | enforced | yes | pytest `projects/policyengine-simulation-entry/tests/test_deployment_assets_cloud_run.py` | `pr.yml:test` | **unproven** | @@ -189,6 +190,7 @@ flowchart LR e_pytest_projects_policyengine_simulation_entry_tests_test_deployment_assets_release_gates_py__projects_policyengine_simulation_entry_tests_test_deployment_assets_routing_py["pytest projects/policyengine-simulation-entry/tests/test_deployment_assets_release_gates.py, projects/policyengine-simulation-entry/tests/test_deployment_assets_routing.py"] r_sim_deploy_shell_scripts_valid("sim.deploy.shell-scripts-valid (enforced)") r_sim_deploy_smoke_tests_strict("sim.deploy.smoke-tests-strict (enforced)") + r_sim_deploy_staging_promotes_tested_revision("sim.deploy.staging-promotes-tested-revision (enforced)") r_sim_deploy_traffic_changes_exact("sim.deploy.traffic-changes-exact (enforced)") e_pytest_projects_policyengine_simulation_entry_tests_test_deployment_assets_traffic_py["pytest projects/policyengine-simulation-entry/tests/test_deployment_assets_traffic.py"] r_sim_deploy_workflow_shell_safety("sim.deploy.workflow-shell-safety (enforced)") @@ -216,6 +218,7 @@ flowchart LR e_pytest_projects_policyengine_simulation_entry_tests_test_deployment_assets_release_gates_py__projects_policyengine_simulation_entry_tests_test_deployment_assets_routing_py --> j_CI_pr_yml_test r_sim_deploy_shell_scripts_valid --> e_pytest_projects_policyengine_simulation_entry_tests_test_deployment_assets_py r_sim_deploy_smoke_tests_strict --> e_pytest_projects_policyengine_simulation_entry_tests_test_deployment_assets_py + r_sim_deploy_staging_promotes_tested_revision --> e_pytest_projects_policyengine_simulation_entry_tests_test_deployment_assets_promotion_py r_sim_deploy_traffic_changes_exact --> e_pytest_projects_policyengine_simulation_entry_tests_test_deployment_assets_traffic_py e_pytest_projects_policyengine_simulation_entry_tests_test_deployment_assets_traffic_py --> j_CI_pr_yml_test r_sim_deploy_workflow_shell_safety --> e_pytest_projects_policyengine_simulation_entry_tests_test_deployment_assets_cloud_run_py diff --git a/projects/policyengine-simulation-entry/DEPLOYMENT.md b/projects/policyengine-simulation-entry/DEPLOYMENT.md index 9783ef1cc..76038ce31 100644 --- a/projects/policyengine-simulation-entry/DEPLOYMENT.md +++ b/projects/policyengine-simulation-entry/DEPLOYMENT.md @@ -33,17 +33,29 @@ qualification uses Cloud Run's generated tagged revision URL and does not require a custom domain. The `prod` environment follows the same sequence automatically, but cannot -begin until every `beta` deployment, routing, integration, and authentication -job has succeeded. Production tests also use the tagged candidate URL, so the -currently serving revision remains unchanged during qualification. +begin until every `beta` deployment, routing, integration, authentication, +promotion, and revision clean-up job has succeeded, so a failed staging +clean-up also stops the production deployment. A run from a branch other than +`main` skips `beta`'s promotion and clean-up, and the `prod` environment then +refuses the run, since it accepts only `main`. Both environments' tests use +the tagged candidate URL, so the currently serving revision remains unchanged +during qualification. ## Promotion and rollback -After the complete production suite passes, the workflow verifies that the -exact tested revision belongs to the production service and is ready. It also +Both `beta` and `prod` promote, `beta` only for a run from `main`. After an +environment's complete suite passes, the workflow verifies that the exact +tested revision belongs to that environment's service and is ready. It also checks that stable traffic still points to the revision recorded before -deployment. If those guards pass, it assigns all stable service traffic to the -tested production revision. +deployment. If those checks pass, it assigns all stable service traffic to the +tested revision. + +In `beta`, promotion keeps the staging service's stable URL on the revision +that passed the staging tests, so staging callers of that URL, such as the +Public API's staging service, reach the build the deployment tested. The +staging deployment of the **Deploy Stage 12 comparison path** workflow promotes +in the same way. Rule `sim.deploy.staging-promotes-tested-revision` requires +both. The workflow then checks the stable generated Cloud Run URL. It uses the runtime revision header to prove that the stable URL reaches the exact revision @@ -51,6 +63,31 @@ that passed candidate tests, and it repeats the deployed authentication checks. If an immediate post-promotion check fails, the workflow restores the exact previous revision recorded before deployment and fails visibly. +After a successful promotion and deployment, the workflow deletes the +service's older revisions. It keeps only the promoted revision and the one it +replaced, which `simulation-rollback.yml` can route back to. + +Both workflows can also be started manually, and the `beta` environment does +not restrict which branch a run may come from. Both therefore pass +`promote_entrypoint: ${{ github.ref == 'refs/heads/main' }}`. A manual run from +a branch other than `main` deploys and tests its candidate revision, but leaves +the staging service's Cloud Run traffic where it was and deletes no revisions, +so the last revision deployed from `main` stays available for rollback. + +Promotion covers only the Cloud Run entrypoint. A branch run still deploys the +branch's Modal services to the staging Modal environment and publishes their +routing there, so callers of the staging stable URL can reach them: the main +deployment workflow deploys the gateway, the executor and the Stage 12 workers, +and the Stage 12 workflow deploys the Stage 12 workers. A later deployment from +`main` replaces the apps it deploys, but routes and versions that it doesn't +deploy can stay. + +The check reads the branch's own copy of the workflow: it stops an accidental +branch run from promoting, not a branch that edits it. Only a branch +restriction on the `beta` environment would enforce it, and that would also +refuse every branch run's candidate deployment and tests, since each of the +reusable workflow's deployment jobs runs in that environment. + Deployment concurrency prevents two releases from changing the entrypoint service simultaneously. Promotion never targets `LATEST`. @@ -82,10 +119,10 @@ The deployment workflow passes each environment's unset or empty variable deploys `0`, which records callers that lack a route's permission but lets them through. `1` rejects those callers with 403. Any other value fails the deployment before Cloud Run changes. A new value takes effect -at the next entrypoint deployment. In `prod` that deployment is promoted to -stable traffic. `beta` never promotes its candidate revision, so a `beta` value -applies only to the tagged candidate that the deployment tests call. The route -permissions are listed in `README.md`. +at the next entrypoint deployment, which `prod`, and `beta` for a run from +`main`, promote to stable traffic once their tests pass. A `beta` value +therefore applies to every caller of the staging stable URL as well as to the +deployment tests. The route permissions are listed in `README.md`. ### Auth0 prerequisites @@ -117,11 +154,12 @@ variable to `1`: 1. Deploy with the variable unset or `0`, and complete the Auth0 setup. 2. Watch the `prod` service for `simulation_entry_permission_missing` events with `enforced=false`. Each one names the missing permission and the - caller's `client_id`. Only `prod` serves real callers such as API v1: a - `beta` candidate receives only the deployment tests' own requests, so `beta` - events cannot show a missing API v1 grant. Access tokens issued before a - grant do not carry it, so wait at least one access-token lifetime after the - last grant with no new events. + caller's `client_id`. Only `prod` is certain to receive requests from every + real caller: the staging stable URL receives requests only from the staging + callers configured to use it, and tagged candidates only from the + deployment tests, so `beta` events cannot show every missing grant. Access + tokens issued before a grant do not carry it, so wait at least one + access-token lifetime after the last grant with no new events. 3. Set the `beta` variable to `1` and deploy. With enforcement on, the authenticated deployment tests also check that the test client's token is rejected on `GET /internal/stage12/reports/{evaluation_id}`, which requires diff --git a/projects/policyengine-simulation-entry/README.md b/projects/policyengine-simulation-entry/README.md index 3a3f72815..272f2cbf0 100644 --- a/projects/policyengine-simulation-entry/README.md +++ b/projects/policyengine-simulation-entry/README.md @@ -21,8 +21,9 @@ Cloud Run's generated revision name is returned in a response header so deployment checks can prove which revision a tagged or stable URL serves. Live caller/backend authentication checks are isolated in `authenticated_tests`. -They qualify tagged Cloud Run candidates before production traffic changes and -verify the stable production endpoint immediately after promotion. +They qualify tagged Cloud Run candidates before an environment's stable traffic +changes and verify its stable endpoint immediately after promotion, in both +staging and production. ## Caller authentication and permissions diff --git a/projects/policyengine-simulation-entry/tests/test_deployment_assets.py b/projects/policyengine-simulation-entry/tests/test_deployment_assets.py index a47600cbc..2b7c4485c 100644 --- a/projects/policyengine-simulation-entry/tests/test_deployment_assets.py +++ b/projects/policyengine-simulation-entry/tests/test_deployment_assets.py @@ -9,6 +9,7 @@ AUTHENTICATED_TESTS, CLOUD_RUN_DEPLOY_SCRIPT, DEPLOY_WORKFLOW, + MAIN_ONLY_PROMOTION, REPOSITORY_ROOT, REUSABLE_DEPLOY_WORKFLOW, SMOKE_SCRIPT, @@ -198,8 +199,10 @@ def _assert_beta_then_prod_release( assert "deployment_environment: staging" in deploy_workflow assert "deployment_environment: production" in deploy_workflow assert deploy_workflow.count("deploy_existing_stack: true") == 2 - assert "promote_entrypoint: false" in deploy_workflow - assert "promote_entrypoint: true" in deploy_workflow + # prod always promotes; beta promotes only a run from main. + assert deploy_workflow.count("promote_entrypoint: true") == 1 + assert f"promote_entrypoint: {MAIN_ONLY_PROMOTION}" in deploy_workflow + assert "promote_entrypoint: false" not in deploy_workflow assert "entrypoint_environment" not in deploy_workflow assert "entrypoint_environment" not in reusable_workflow assert "entrypoint_public_url" not in deploy_workflow diff --git a/projects/policyengine-simulation-entry/tests/test_deployment_assets_promotion.py b/projects/policyengine-simulation-entry/tests/test_deployment_assets_promotion.py index 30f3b9eda..a49783756 100644 --- a/projects/policyengine-simulation-entry/tests/test_deployment_assets_promotion.py +++ b/projects/policyengine-simulation-entry/tests/test_deployment_assets_promotion.py @@ -1,14 +1,20 @@ -"""Tests for promotion order, Stage 12-only deployments and the sentinel.""" +"""Tests for promotion order, Stage 12-only runs, the sentinel and staging promotion.""" from __future__ import annotations +import copy import re from test_deployment_assets_support import ( DEPLOY_WORKFLOW, DEPLOYMENT_READY_SCRIPT, + MAIN_ONLY_PROMOTION, REUSABLE_DEPLOY_WORKFLOW, + REUSABLE_DEPLOY_WORKFLOW_REFERENCE, STAGE12_DEPLOY_WORKFLOW, + STAGING_PROMOTION_FIXTURES, + WORKFLOWS_DIRECTORY, + evaluate_github_expression, job_section, sentinel_check_step, workflow_jobs, @@ -267,3 +273,93 @@ def test_deployed_stack_must_reproduce_the_numerical_sentinel(): for index, name in enumerate(names) if name.startswith("Restore previous revision") ) + + +def _staging_deployment_job_ids(jobs: dict) -> list[str]: + """Return the jobs that run the reusable deployment workflow for staging.""" + return [ + job_id + for job_id, job in jobs.items() + if job.get("uses") == REUSABLE_DEPLOY_WORKFLOW_REFERENCE + and (job.get("with") or {}).get("deployment_environment") == "staging" + ] + + +def staging_promotion_violations(jobs: dict) -> list[str]: + """Return each staging deployment job that doesn't promote exactly from main. + + Rule sim.deploy.staging-promotes-tested-revision: a staging deployment must + pass ``promote_entrypoint: ${{ github.ref == 'refs/heads/main' }}`` to the + reusable deployment workflow. Any other value counts as a violation: + ``false`` leaves the stable URL on a revision the deployment never tested, + and ``true`` lets a run from another branch promote an unmerged build. + """ + return [ + f"{job_id} must pass promote_entrypoint: {MAIN_ONLY_PROMOTION}" + for job_id in _staging_deployment_job_ids(jobs) + if jobs[job_id]["with"].get("promote_entrypoint") != MAIN_ONLY_PROMOTION + ] + + +def test_staging_deployments_promote_their_tested_entrypoint_revision(): + """Staging's stable URL serves the revision that passed staging's tests. + + Callers such as the Public API's staging service use the staging stable + URL, so a staging deployment that left stable traffic on an older revision + would leave them calling a build that the deployment did not test. + """ + staging_jobs_by_workflow = {} + violations = [] + for workflow_path in sorted(WORKFLOWS_DIRECTORY.glob("*.yml")): + jobs = workflow_jobs(workflow_path) + staging_job_ids = _staging_deployment_job_ids(jobs) + if staging_job_ids: + staging_jobs_by_workflow[workflow_path.name] = staging_job_ids + violations.extend( + f"{workflow_path.name}: {violation}" + for violation in staging_promotion_violations(jobs) + ) + + # The check must find the two known staging deployments, so that a renamed + # input or reusable workflow cannot make it pass by finding nothing. + assert "beta" in staging_jobs_by_workflow.get(DEPLOY_WORKFLOW.name, []) + assert "deploy-stage12-staging" in staging_jobs_by_workflow.get( + STAGE12_DEPLOY_WORKFLOW.name, [] + ) + for workflow_name, staging_job_ids in staging_jobs_by_workflow.items(): + jobs = workflow_jobs(WORKFLOWS_DIRECTORY / workflow_name) + for job_id in staging_job_ids: + assert ( + jobs[job_id]["with"]["entrypoint_service"] + == "policyengine-simulation-entry-staging" + ) + assert violations == [] + + +def test_rule_proof_staging_promotes_tested_revision(): + """Proof for rule sim.deploy.staging-promotes-tested-revision. + + The bad fixture is the `beta` deployment as it was before the rule: it + tested a tagged candidate but never moved staging's stable traffic to it. + Passing a literal ``true`` is also a violation, since a run from another + branch would then promote that branch's build. + """ + bad_jobs = workflow_jobs(STAGING_PROMOTION_FIXTURES / "bad_simulation_deploy.yml") + good_jobs = workflow_jobs(STAGING_PROMOTION_FIXTURES / "good_simulation_deploy.yml") + any_branch_jobs = copy.deepcopy(good_jobs) + any_branch_jobs["beta"]["with"]["promote_entrypoint"] = True + + violation = f"beta must pass promote_entrypoint: {MAIN_ONLY_PROMOTION}" + assert staging_promotion_violations(bad_jobs) == [violation] + assert staging_promotion_violations(any_branch_jobs) == [violation] + assert staging_promotion_violations(good_jobs) == [] + # The required value is true for a run from main and false for any other + # branch or for a tag. + for ref, promotes in ( + ("refs/heads/main", True), + ("refs/heads/feat/stage12-segmentation-cache", False), + ("refs/heads/main-backport", False), + ("refs/tags/v1.0.0", False), + ): + context = {"github": {"ref": ref}} + assert evaluate_github_expression(MAIN_ONLY_PROMOTION, context) is promotes diff --git a/projects/policyengine-simulation-entry/tests/test_deployment_assets_support.py b/projects/policyengine-simulation-entry/tests/test_deployment_assets_support.py index ae004c799..70ae911b0 100644 --- a/projects/policyengine-simulation-entry/tests/test_deployment_assets_support.py +++ b/projects/policyengine-simulation-entry/tests/test_deployment_assets_support.py @@ -34,10 +34,24 @@ DEPLOYMENT_READY_SCRIPT = ( REPOSITORY_ROOT / ".github" / "scripts" / "simulation-deployment-ready.sh" ) +WORKFLOWS_DIRECTORY = REPOSITORY_ROOT / ".github" / "workflows" DEPLOY_WORKFLOW = REPOSITORY_ROOT / ".github" / "workflows" / "simulation-deploy.yml" REUSABLE_DEPLOY_WORKFLOW = ( REPOSITORY_ROOT / ".github" / "workflows" / "simulation-deploy.reusable.yml" ) +REUSABLE_DEPLOY_WORKFLOW_REFERENCE = ( + "./.github/workflows/simulation-deploy.reusable.yml" +) +# The promotion input every staging deployment passes: true only for a run +# from main, so an accidental manual run from another branch never moves +# staging's stable traffic or deletes its older revisions. +MAIN_ONLY_PROMOTION = "${{ github.ref == 'refs/heads/main' }}" +STAGING_PROMOTION_FIXTURES = ( + REPOSITORY_ROOT + / "rules" + / "fixtures" + / "sim.deploy.staging-promotes-tested-revision" +) ROLLBACK_WORKFLOW = ( REPOSITORY_ROOT / ".github" / "workflows" / "simulation-rollback.yml" ) diff --git a/rules/fixtures/sim.deploy.staging-promotes-tested-revision/bad_simulation_deploy.yml b/rules/fixtures/sim.deploy.staging-promotes-tested-revision/bad_simulation_deploy.yml new file mode 100644 index 000000000..fbff8dc6a --- /dev/null +++ b/rules/fixtures/sim.deploy.staging-promotes-tested-revision/bad_simulation_deploy.yml @@ -0,0 +1,44 @@ +# Proof fixture: the deployment workflow's jobs as they were before the rule. +# "beta" deploys and tests a tagged staging candidate but passes +# promote_entrypoint: false, so the staging stable URL keeps serving whatever +# revision it served before, which the deployment never tested. +name: Deploy Simulation API + +on: + push: + branches: + - main + +jobs: + beta: + name: Complete beta deployment + uses: ./.github/workflows/simulation-deploy.reusable.yml + with: + release_environment: beta + deployment_environment: staging + modal_environment: staging + entrypoint_service: policyengine-simulation-entry-staging + entrypoint_revision_prefix: s- + entrypoint_min_instances: 0 + entrypoint_max_instances: 2 + promote_entrypoint: false + run_full_integration: true + deploy_existing_stack: true + secrets: inherit + + prod: + name: Complete prod deployment + needs: beta + uses: ./.github/workflows/simulation-deploy.reusable.yml + with: + release_environment: prod + deployment_environment: production + modal_environment: main + entrypoint_service: policyengine-simulation-entry + entrypoint_revision_prefix: p- + entrypoint_min_instances: 1 + entrypoint_max_instances: 20 + promote_entrypoint: true + run_full_integration: false + deploy_existing_stack: true + secrets: inherit diff --git a/rules/fixtures/sim.deploy.staging-promotes-tested-revision/good_simulation_deploy.yml b/rules/fixtures/sim.deploy.staging-promotes-tested-revision/good_simulation_deploy.yml new file mode 100644 index 000000000..2b6a33257 --- /dev/null +++ b/rules/fixtures/sim.deploy.staging-promotes-tested-revision/good_simulation_deploy.yml @@ -0,0 +1,44 @@ +# Proof fixture: the deployment workflow's jobs with the rule satisfied. +# "beta" promotes its tested staging candidate when the run is from main, so the +# staging stable URL serves the revision that passed the staging tests, and a +# manual run from another branch leaves it unchanged. +name: Deploy Simulation API + +on: + push: + branches: + - main + +jobs: + beta: + name: Complete beta deployment + uses: ./.github/workflows/simulation-deploy.reusable.yml + with: + release_environment: beta + deployment_environment: staging + modal_environment: staging + entrypoint_service: policyengine-simulation-entry-staging + entrypoint_revision_prefix: s- + entrypoint_min_instances: 0 + entrypoint_max_instances: 2 + promote_entrypoint: ${{ github.ref == 'refs/heads/main' }} + run_full_integration: true + deploy_existing_stack: true + secrets: inherit + + prod: + name: Complete prod deployment + needs: beta + uses: ./.github/workflows/simulation-deploy.reusable.yml + with: + release_environment: prod + deployment_environment: production + modal_environment: main + entrypoint_service: policyengine-simulation-entry + entrypoint_revision_prefix: p- + entrypoint_min_instances: 1 + entrypoint_max_instances: 20 + promote_entrypoint: true + run_full_integration: false + deploy_existing_stack: true + secrets: inherit diff --git a/rules/registry.toml b/rules/registry.toml index f02e2ba7f..1b314011d 100644 --- a/rules/registry.toml +++ b/rules/registry.toml @@ -377,6 +377,31 @@ workflow = ".github/workflows/pr.yml" job = "test" invocation = "uv run --frozen pytest tests/" +[[rule]] +id = "sim.deploy.staging-promotes-tested-revision" +category = "deployment" +statement = "Every staging deployment passes promote_entrypoint: ${{ github.ref == 'refs/heads/main' }}, so a run from main moves the staging service's stable Cloud Run traffic to the entrypoint revision that passed the staging tests, and an accidental manual run from another branch neither moves that traffic nor deletes older revisions." +rationale = "projects/policyengine-simulation-entry/DEPLOYMENT.md#promotion-and-rollback" +scope = [".github/workflows/*.yml"] +tier = "mechanical" +status = "enforced" +autofix = false +[rule.enforcer] +kind = "pytest" +project = "projects/policyengine-simulation-entry" +nodes = [ + "tests/test_deployment_assets_promotion.py::test_staging_deployments_promote_their_tested_entrypoint_revision", +] +[[rule.ci]] +workflow = ".github/workflows/pr.yml" +job = "test" +invocation = "uv run --frozen pytest tests/" +[rule.proof] +bad_fixture = "rules/fixtures/sim.deploy.staging-promotes-tested-revision/bad_simulation_deploy.yml" +good_fixture = "rules/fixtures/sim.deploy.staging-promotes-tested-revision/good_simulation_deploy.yml" +project = "projects/policyengine-simulation-entry" +test = "tests/test_deployment_assets_promotion.py::test_rule_proof_staging_promotes_tested_revision" + [[rule]] id = "sim.deploy.smoke-tests-strict" category = "deployment"