cloud calibration updates

.gitignore

@@ -1,5 +1,8 @@
 **/profiling.json
+artifacts/
 experiments/**/simulations/
+experiments/**/calibration/
+.codex
 
 # IDE and Platform
 .vscode
@@ -45,6 +48,8 @@ experiments/**/simulations/
 # !your_data_directory/
 !input/people_test.csv
 !input/symptom_delay_median_iqr.csv
+!packages/importation/src/importation/data/
+!packages/importation/src/importation/data/*.csv
 !tests/data/*
 
 

.pre-commit-config.yaml

@@ -11,8 +11,10 @@ repos:
     -   id: check-yaml
     -   id: check-toml
     -   id: end-of-file-fixer
+        exclude: \.patch$
     -   id: mixed-line-ending
     -   id: trailing-whitespace
+        exclude: \.patch$
 #####
 # Python
 - repo: https://github.com/astral-sh/ruff-pre-commit
@@ -49,7 +51,11 @@ repos:
     rev: v0.4.3.9017
     hooks:
     -   id: style-files
+        files: \.(R|Rmd|qmd)$
+        stages: [manual]
     -   id: lintr
+        files: \.(R|Rmd|qmd)$
+        stages: [manual]
 #####
 # Java
 - repo: https://github.com/macisamuele/language-formatters-pre-commit-hooks

Dockerfile.cloud

@@ -0,0 +1,41 @@
+FROM rust:1.90-bookworm AS rust-builder
+
+WORKDIR /app
+COPY Cargo.toml Cargo.lock ./
+COPY benches ./benches
+COPY src ./src
+RUN cargo build --release --bin ixa-epi-covid
+
+FROM ghcr.io/astral-sh/uv:python3.12-bookworm-slim AS python-builder
+
+WORKDIR /app
+RUN apt-get update \
+    && apt-get install -y --no-install-recommends git \
+    && rm -rf /var/lib/apt/lists/*
+COPY pyproject.toml uv.lock README.md ./
+COPY packages ./packages
+COPY src ./src
+COPY experiments/phase1/input/default_params.json ./experiments/phase1/input/default_params.json
+COPY ixa_epi_covid.mrp.task.toml ./ixa_epi_covid.mrp.task.toml
+RUN uv sync --frozen --no-dev
+
+FROM python:3.12-slim-bookworm
+
+WORKDIR /app
+
+RUN useradd --create-home --shell /bin/bash appuser
+
+COPY --from=python-builder /app /app
+COPY --from=rust-builder /app/target/release/ixa-epi-covid /usr/local/bin/ixa-epi-covid
+RUN mkdir -p /app/target/release
+COPY --from=rust-builder /app/target/release/ixa-epi-covid /app/target/release/ixa-epi-covid
+COPY ixa_epi_covid.mrp.task.toml /app/ixa_epi_covid.mrp.task.toml
+COPY experiments/phase1/input/default_params.json /app/default_params.json
+
+RUN ln -sf /app/.venv/bin/mrp /usr/local/bin/mrp
+
+ENV PATH="/app/.venv/bin:${PATH}"
+
+USER appuser
+
+CMD ["mrp", "run", "/app/ixa_epi_covid.mrp.task.toml"]

Makefile

@@ -1,33 +1,114 @@
 STATE ?=WY
 SIZE ?= 1000
+RUFF_VERSION ?= 0.14.6
 
 # Normalize SIZE to use underscores (e.g., 1000 -> 1_000, 1000000 -> 1_000_000)
 NORMALIZED_SIZE := $(shell python3 -c "print(f'{int(\"$(SIZE)\".replace(\"_\",\"\")):_}')")
 CLEAN_STATE_PATTERN := $(if $(filter command line environment environment override,$(origin STATE)),$(STATE),*)
 CLEAN_SIZE_PATTERN := $(if $(filter command line environment environment override,$(origin SIZE)),$(NORMALIZED_SIZE),*)
 
-.PHONY: all uv-sync synthetic-population test-syn-pop clean-synthetic-population run profile
+.PHONY: all help test lint format-check typecheck check uv-sync uv-sync-cloud
+.PHONY: synthetic-population test-syn-pop clean-synthetic-population
+.PHONY: run run-small run-large run-xl profile profile-small profile-large profile-xl
+.PHONY: bench bench-compare build-rust-release docker-build-cloud-image
+.PHONY: calibrate-phase-1 calibrate-phase-1-docker calibrate-phase-1-cloud
+.PHONY: cloud-list cloud-cleanup-plan cloud-cleanup
+.PHONY: cloud-cleanup-user-plan cloud-cleanup-user
+.PHONY: projections-phase-1 plot-phase-1-projection
+.PHONY: calibrate-phase-1-smc projections-phase-1-smc plot-phase-1-projection-smc
+.PHONY: calibrate-phase-1-dev projections-phase-1-dev plot-phase-1-projection-dev
 all: uv-sync
 
+help:
+	@printf '%s\n' \
+		'Available targets:' \
+		'  all                         Initialize the default uv environment' \
+		'  test                        Run the Python test suite' \
+		'  lint                        Run ruff checks' \
+		'  format-check                Check Python formatting' \
+		'  typecheck                   Run ty type checks' \
+		'  check                       Run lint, format check, type check, and tests' \
+		'  uv-sync                     Initialize the uv environment for local development' \
+		'  uv-sync-cloud               Initialize uv with Azure/cloud dependencies' \
+		'  synthetic-population        Generate a synthetic population (STATE=..., SIZE=...)' \
+		'  test-syn-pop                Run synthetic population generator tests' \
+		'  clean-synthetic-population  Remove generated synthetic population CSVs' \
+		'  run                         Run the model with a synthetic population' \
+		'  run-small                   Run the model with SIZE=10_000' \
+		'  run-large                   Run the model with SIZE=1_000_000' \
+		'  run-xl                      Run the model with SIZE=10_000_000' \
+		'  profile                     Profile the model with samply' \
+		'  profile-small               Profile with SIZE=10_000' \
+		'  profile-large               Profile with SIZE=1_000_000' \
+		'  profile-xl                  Profile with SIZE=10_000_000' \
+		'  bench                       Run infection loop benchmarks' \
+		'  bench-compare               Compare benchmarks against BASE (default: HEAD)' \
+		'  build-rust-release          Build the Rust binary in release mode' \
+		'  docker-build-cloud-image    Build the local cloud task image' \
+		'  calibrate-phase-1           Run phase-1 production calibration' \
+		'  calibrate-phase-1-docker    Run phase-1 calibration via local Docker MRP' \
+		'  calibrate-phase-1-cloud     Run phase-1 calibration via Azure/cloud MRP' \
+		'  cloud-list                  List current user Azure cloud resources' \
+		'  cloud-cleanup-plan          Dry-run cleanup for SESSION_ID=...' \
+		'  cloud-cleanup               Delete cloud resources for SESSION_ID=...' \
+		'  cloud-cleanup-user-plan     Dry-run cleanup for CLOUD_USER' \
+		'  cloud-cleanup-user          Delete all cloud sessions for CLOUD_USER' \
+		'  projections-phase-1         Run phase-1 projections' \
+		'  plot-phase-1-projection     Plot phase-1 projections' \
+		'  calibrate-phase-1-smc       Run phase-1 SMC calibration' \
+		'  projections-phase-1-smc     Run phase-1 SMC projections' \
+		'  plot-phase-1-projection-smc Plot phase-1 SMC projections' \
+		'  calibrate-phase-1-dev       Run phase-1 dev calibration' \
+		'  projections-phase-1-dev     Run phase-1 dev projections' \
+		'  plot-phase-1-projection-dev Plot phase-1 dev projections' \
+		'' \
+		'Variables:' \
+		'  STATE=WY                    Synthetic population state' \
+		'  SIZE=1000                   Synthetic population size' \
+		'  BASE=HEAD                   Benchmark comparison base ref' \
+		'  MAX_WORKERS=4               Calibration/projection worker count' \
+		'  SESSION_ID=...              Cloud cleanup session ID' \
+		'  CLOUD_USER=<current user>  Cloud cleanup username filter' \
+		'  CLOUD_CONFIG=...            Cloud cleanup/calibration config path'
+
+# Run the Python test suite
+test:
+	uv run pytest
+
+lint:
+	uv run --with ruff==$(RUFF_VERSION) ruff check --line-length 79 .
+
+format-check:
+	uv run --with ruff==$(RUFF_VERSION) ruff format --check --line-length 79 .
+
+typecheck:
+	uv run --with ty ty check --ignore=unresolved-import
+
+check: lint format-check typecheck test ## Run lint, format check, type check, and tests.
+
 # Initialize the uv environment for Python scripts
 uv-sync:
 	uv sync --all-packages --all-extras --dev --locked
 
+# Initialize the uv environment including Azure/cloud dependencies
+uv-sync-cloud:
+	uv sync --all-packages --all-extras --dev --group cloudops --locked
+
 # Generate a synthetic population (configure with STATE and SIZE)
 synthetic-population:
 	uv run python -m create_synthetic_population.run --state $(STATE) --size $(NORMALIZED_SIZE)
 
 # Run synthetic population generator tests
 test-syn-pop:
-	uv run pytest scripts/test_create_synthetic_population.py
+	uv run pytest packages/create_synthetic_population/tests/test_create_synthetic_population.py
 
 # Remove generated synthetic population CSVs. Optionally narrow with STATE and/or SIZE.
 clean-synthetic-population:
 	rm -f input/synth_pop_people_$(CLEAN_STATE_PATTERN)_$(CLEAN_SIZE_PATTERN).csv
 	rm -f input/synth_pop_region_$(CLEAN_STATE_PATTERN)_$(CLEAN_SIZE_PATTERN).csv
 
 input/synth_pop_people_%.csv:
-	uv run scripts/create_synthetic_population.py --state $(shell echo "$*" | sed 's/_.*//')  --size $(shell echo "$*" | sed 's/^[A-Z]*_//')
+	uv run python -m create_synthetic_population.run --state $(shell echo "$*" | sed 's/_.*//') --size $(shell echo "$*" | sed 's/^[A-Z]*_//')
 
 # Run the model with a synthetic population (e.g., make run SIZE=1_000_000)
 # Generates the population file if it doesn't exist.
@@ -70,11 +151,67 @@ bench-compare: input/synth_pop_people_WY_10_000.csv input/synth_pop_people_WY_10
 	uv run scripts/bench_compare.py $(BASE)
 
 MAX_WORKERS ?= 4
+AUTO_SIZE ?=
+CLOUD_CONFIG ?= ixa_epi_covid.cloud_config.toml
+CURRENT_CLOUD_USER := $(shell python3 -c 'import getpass, os, re; user = os.environ.get("USER") or os.environ.get("LOGNAME") or getpass.getuser(); print(re.sub("-+", "-", re.sub("[^a-z0-9-]+", "-", user.lower())).strip("-"))')
+CLOUD_USER ?= $(CURRENT_CLOUD_USER)
+CLOUD_CLEANUP_ACR ?= --skip-acr
+CLOUD_CLEANUP_FILTERS = $(if $(SESSION_ID),--session-id $(SESSION_ID)) $(if $(CLOUD_USER),--user $(CLOUD_USER)) $(if $(IMAGE_TAG),--image-tag $(IMAGE_TAG)) $(CLOUD_CLEANUP_ACR)
+PHASE1_PROD_CONFIG ?= ./experiments/phase1/input/prod-config.yaml
+PHASE1_DEV_CONFIG ?= ./experiments/phase1/input/dev-config.yaml
+CLOUD_ARTIFACTS_DIR ?= ./experiments/phase1/calibration/artifacts
 TARGET_RESULTS = ./experiments/phase1/calibration/output_indiana/results.pkl
+
+define require_session_id
+$(if $(SESSION_ID),,$(error SESSION_ID is required, e.g. make $@ SESSION_ID=<session-id>))
+endef
+
+define require_user
+$(if $(CLOUD_USER),,$(error CLOUD_USER is required, e.g. make $@ CLOUD_USER=<username>))
+endef
+
+build-rust-release:
+	uv run cargo build -r
+
+# Build the cloud task image locally. This is the image used by the docker-backed
+# MRP config and the same Dockerfile the cloud runner uploads to Azure.
+docker-build-cloud-image:
+	docker build -t ixa-epi-covid-cloud:latest -f Dockerfile.cloud .
+
 calibrate-phase-1: $(TARGET_RESULTS)
 $(TARGET_RESULTS): ./experiments/phase1/input/priors.json ./experiments/phase1/input/default_params.json
-	uv run cargo build -r
-	uv run python ./scripts/phase_1_calibration.py -c ./experiments/phase1/input/prod-config.yaml -o ./experiments/phase1/calibration/output_indiana --max-workers $(MAX_WORKERS)
+	$(MAKE) build-rust-release
+	uv run python ./scripts/phase_1_calibration.py -c $(PHASE1_PROD_CONFIG) -o ./experiments/phase1/calibration/output_indiana --max-workers $(MAX_WORKERS)
+
+# Run phase-1 calibration through the local Docker-backed MRP path. The Python
+# wrapper still orchestrates calibration, but each particle evaluation runs
+# through the cloud-task container image.
+calibrate-phase-1-docker: docker-build-cloud-image
+	uv run python ./scripts/phase_1_calibration.py -c $(PHASE1_PROD_CONFIG) -o ./experiments/phase1/calibration/output_indiana_docker --max-workers $(MAX_WORKERS) --docker
+
+# Run phase-1 calibration through the Azure/cloud-backed MRP path. This target
+# requires the cloudops dependency group, so bootstrap with uv-sync-cloud first.
+calibrate-phase-1-cloud: uv-sync-cloud build-rust-release
+	uv run python ./scripts/phase_1_calibration.py -c $(PHASE1_PROD_CONFIG) -o ./experiments/phase1/calibration/output_indiana_cloud --max-workers $(MAX_WORKERS) --cloud --cloud-config $(CLOUD_CONFIG) $(AUTO_SIZE) --artifacts-dir $(CLOUD_ARTIFACTS_DIR)
+
+cloud-list: uv-sync-cloud
+	uv run python -m calibrationtools.cloud.cleanup --config $(CLOUD_CONFIG) --list $(CLOUD_CLEANUP_FILTERS)
+
+cloud-cleanup-plan: uv-sync-cloud
+	$(call require_session_id)
+	uv run python -m calibrationtools.cloud.cleanup --config $(CLOUD_CONFIG) --session-id $(SESSION_ID) --user $(CLOUD_USER) --dry-run $(if $(IMAGE_TAG),--image-tag $(IMAGE_TAG)) $(CLOUD_CLEANUP_ACR)
+
+cloud-cleanup: uv-sync-cloud
+	$(call require_session_id)
+	uv run python -m calibrationtools.cloud.cleanup --config $(CLOUD_CONFIG) --session-id $(SESSION_ID) --user $(CLOUD_USER) $(if $(IMAGE_TAG),--image-tag $(IMAGE_TAG)) $(CLOUD_CLEANUP_ACR)
+
+cloud-cleanup-user-plan: uv-sync-cloud
+	$(call require_user)
+	uv run python -m calibrationtools.cloud.cleanup --config $(CLOUD_CONFIG) --user $(CLOUD_USER) --all-sessions-for-user --dry-run $(if $(IMAGE_TAG),--image-tag $(IMAGE_TAG)) $(CLOUD_CLEANUP_ACR)
+
+cloud-cleanup-user: uv-sync-cloud
+	$(call require_user)
+	uv run python -m calibrationtools.cloud.cleanup --config $(CLOUD_CONFIG) --user $(CLOUD_USER) --all-sessions-for-user $(if $(IMAGE_TAG),--image-tag $(IMAGE_TAG)) $(CLOUD_CLEANUP_ACR)
 
 projections-phase-1: $(TARGET_RESULTS)
 	uv run python ./scripts/phase_1_projection.py -d output_indiana --max-workers $(MAX_WORKERS)
@@ -83,7 +220,7 @@ plot-phase-1-projection:
 	uv run python ./scripts/plot_phase_1_projection.py -d output_indiana
 
 calibrate-phase-1-smc: ./experiments/phase1/input/priors.json ./experiments/phase1/input/default_params.json
-	uv run cargo build -r
+	$(MAKE) build-rust-release
 	uv run python ./scripts/phase_1_calibration.py -c ./experiments/phase1/input/prod-smc-config.yaml -o ./experiments/phase1/calibration/smc --max-workers $(MAX_WORKERS)
 
 projections-phase-1-smc: ./experiments/phase1/calibration/smc/results.pkl
@@ -93,8 +230,8 @@ plot-phase-1-projection-smc: projections-phase-1-smc
 	uv run python ./scripts/plot_phase_1_projection.py -d smc
 
 calibrate-phase-1-dev: ./experiments/phase1/input/priors.json ./experiments/phase1/input/default_params.json
-	uv run cargo build -r
-	uv run python ./scripts/phase_1_calibration.py -c ./experiments/phase1/input/dev-config.yaml -o ./experiments/phase1/calibration/dev_$(NORMALIZED_SIZE) --default-population-size-dev $(NORMALIZED_SIZE) --max-workers $(MAX_WORKERS)
+	$(MAKE) build-rust-release
+	uv run python ./scripts/phase_1_calibration.py -c $(PHASE1_DEV_CONFIG) -o ./experiments/phase1/calibration/dev_$(NORMALIZED_SIZE) --default-population-size-dev $(NORMALIZED_SIZE) --max-workers $(MAX_WORKERS)
 
 projections-phase-1-dev: ./experiments/phase1/calibration/dev_$(NORMALIZED_SIZE)/results.pkl
 	uv run python ./scripts/phase_1_projection.py -d dev_$(NORMALIZED_SIZE) --max-workers $(MAX_WORKERS) -f --plot-distances

README.md

@@ -92,6 +92,25 @@ This command can be called without calling the others explicitly.
 
 Production code follows the same format, but drops the `dev` suffix (for example the calibration command is `make calibrate-phase-1`). To run this, please ensure that you have specified a path to a valid synthetic population in the `.env` file under `SYNTH_POP_FILE`.
 
+Cloud calibration uses the model-facing cloud config:
+
+```bash
+uv run python -m ixa_epi_covid.phase1.calibrate \
+  --config_file experiments/phase1/input/prod-config.yaml \
+  --output-dir experiments/phase1/calibration/output_indiana_cloud \
+  --cloud \
+  --cloud-config ixa_epi_covid.cloud_config.toml
+```
+
+Cleanup uses the shared calibration-tools cleanup CLI:
+
+```bash
+python -m calibrationtools.cloud.cleanup \
+  --cloud-config ixa_epi_covid.cloud_config.toml \
+  --session-id <session-id> \
+  --dry-run
+```
+
 ## General Disclaimer
 This repository was created for use by CDC programs to collaborate on public health related projects in support of the [CDC mission](https://www.cdc.gov/about/organization/mission.htm).  GitHub is not hosted by the CDC, but is a third party website used by CDC and its partners to share information and collaborate on software. CDC use of GitHub does not imply an endorsement of any one particular service, product, or enterprise.
 

experiments/phase1/README.md

@@ -1,5 +1,5 @@
 # Running the calibration routine
-In order to generate the results analysis report from the `reports/` directory, first calibrate the model by using the phase 1 calibration script. Ensure that the uv environment is synced and then run the calibration to projection pipeline for the phase 1 routine
+In order to generate the results analysis report from the `reports/` directory, first calibrate the model by using the phase 1 calibration script. Ensure that the uv environment is synced and then run the calibration to projection pipeline for the phase 1 routine.
 
 ```
 uv sync --all-packages

experiments/phase1/reports/calibration.qmd

@@ -19,7 +19,7 @@ import matplotlib.pyplot as plt
 import numpy as np
 import polars as pl
 import json
-from particle_reader import ParticleReader
+from calibrationtools import ParticleReader
 import tempfile
 import polars as pl
 import os