Finish scheduled maintenance safety checks

AGENTS.md

@@ -748,6 +748,9 @@ example, `storage.bunnycdn.com` or `uk.storage.bunnycdn.com`).
 - `STORAGE_ZONE_KEY` - Bunny CDN storage zone access key (required for image uploads)
 - `BACKUP_PAGE_SIZE` - Rows read per keyset page when dumping a table for backup (default 500). Each page is one libsql response, so this bounds the response size to stay under libsqld's "Response is too large" payload cap. Used by `deno task backup` and the admin Backups page; migrations no longer back up inline (the edge subrequest budget can't fit a full dump), so backups are taken out-of-band.
 - `MAIN_INSTANCE_KEY` - Shared secret authorizing the inter-instance site-credentials endpoint (`POST /instance/site-credentials`). When set on a builder/main instance, that endpoint returns built sites' DB URL + token to a caller presenting this key as a bearer token, so the upgrade workflow can back each site up to the builder's storage before deploying. The returned token is each site's own full-access credential (the same one the site runs with) — callers only read, but must treat the response as write-capable production secrets. The caller passes the release tier it is publishing as `?tier=alpha|beta|release` (a tier-less call defaults to `release` ⇒ the whole fleet, which is what the single-site `backup-site` action relies on); each site carries an `updates` channel and only the sites at that tier or more eager are returned (a `release` deploy reaches every site, `beta` reaches beta + alpha sites, `alpha` only alpha sites — an unknown tier is a 400). The response echoes the applied `tier` so a caller can confirm the server actually filtered: a pre-tier build ignores the query string and omits it, letting the canary workflow fail closed instead of fanning a non-release deploy out to the whole fleet. Unset `MAIN_INSTANCE_KEY` ⇒ the endpoint is disabled (404). The upgrade workflow receives the key as a run-time input, not a stored GitHub secret.
+- `DENO_DEPLOY_TOKEN` - Deno Deploy organization access token. Required with `DENO_DEPLOY_ORG_ID` and `DENO_DEPLOY_ORG_SLUG` to build sites on Deno Deploy.
+- `DENO_DEPLOY_ORG_ID` - Deno Deploy organization ID used by the app creation API.
+- `DENO_DEPLOY_ORG_SLUG` - Deno Deploy organization slug used in each app's managed `<app>.<organization>.deno.net` production domain.
 - `BUNNY_DNS_ZONE_ID` - Bunny DNS zone ID for subdomain registration (enables subdomain feature when set with `BUNNY_API_KEY`)
 - `BUNNY_DNS_SUBDOMAIN_SUFFIX` - Suffix appended to user-chosen subdomain (e.g. `.tickets`)
 - `NTFY_URL` - Ntfy endpoint URL for error notifications (e.g. `https://ntfy.sh/your-topic`). Sends domain and error code only, no personal or encrypted data.

scripts/mutation/equivalent-mutants.txt

@@ -46,10 +46,11 @@ src/ui/templates/admin/holidays.tsx:79:45  ?? → ||   # values is a render-valu
 src/shared/checkout-pricing.ts:85:35  ?? → ||    # Map<number,number> sum: 0 ?? 0 === 0 || 0
 src/shared/checkout-pricing.ts:334:41  ?? → ||   # intent.modifiers: an array is always truthy
 src/shared/logistics-filter.ts:22:35  ?? → ||    # raw: string|null, only falsy string "" === fallback ""
-src/shared/config.ts:150:39  ?? → ||    # getEnv(): string|undefined, only falsy string "" === fallback ""
-src/shared/config.ts:163:61  ?? → ||    # getEnv(): string|undefined, only falsy string "" === fallback ""
-src/shared/config.ts:170:33  ?? → ||    # getEnv(): string|undefined, only falsy string "" === fallback ""
-src/shared/config.ts:177:33  ?? → ||    # getEnv(): string|undefined, only falsy string "" === fallback ""
+src/shared/config.ts:166:39  ?? → ||    # getEnv(): string|undefined, only falsy string "" === fallback ""
+src/shared/config.ts:36:5  ?? → ||    # providerValue returns boolean or null, and false stays false with either fallback operator
+src/shared/config.ts:179:61  ?? → ||    # getEnv(): string|undefined, only falsy string "" === fallback ""
+src/shared/config.ts:186:33  ?? → ||    # getEnv(): string|undefined, only falsy string "" === fallback ""
+src/shared/config.ts:193:33  ?? → ||    # getEnv(): string|undefined, only falsy string "" === fallback ""
 src/shared/qr-token.ts:60:18  ?? → ||    # input.date?: string, only falsy string "" === fallback ""
 src/shared/qr-token.ts:62:18  ?? → ||    # input.name?: string, only falsy string "" === fallback ""
 src/shared/app-forms.ts:71:18  ?? → ||    # config.auth is an AuthPolicy object when present, so it is always truthy
@@ -79,7 +80,7 @@ src/shared/accounting/manual-entries.ts:209:29  ?? → ||   # guard error messag
 src/shared/checkout-ledger.ts:35:72  ?? → ||   # find(...)?.amount: number|undefined; the only falsy-non-null amount is 0 and 0 ?? 0 === 0 || 0
 src/features/settings-bundles.ts:247:40  ?? → ||   # bundle is a readonly string[] or undefined; every present array, including [], is truthy
 src/shared/scheduled-access.ts:16:20  ?? → ||   # the optional regex capture is non-empty because the capture uses +, so only undefined reaches the null fallback
-src/shared/maintenance/runner.ts:113:40  ?? → ||   # wakePolicy is a non-empty MaintenanceWakePolicy value or undefined, so only undefined reaches the fallback
+src/shared/maintenance/runner.ts:181:23  ?? → ||   # wakePolicy is a non-empty MaintenanceWakePolicy value or undefined, so only undefined reaches the fallback
 src/shared/builder.ts:201:44  ?? → ||   # dbToken is string|undefined and the fallback is "", so its only falsy string already equals the fallback
 src/shared/builder.ts:202:35  ?? → ||   # dbProvider is a non-empty provider value or undefined, so only undefined reaches the fallback
 src/shared/builder.ts:207:36  ?? → ||   # dbProvider is a non-empty provider value or undefined, so only undefined reaches the fallback
@@ -88,7 +89,7 @@ src/features/url.ts:58:35  ?? → ||   # URLSearchParams.get returns null or a s
 src/shared/storage.ts:376:18  application/octet-stream → ""   # the pinned Bunny SDK normalizes an empty contentType to application/octet-stream, as the request-level upload test proves
 src/shared/site-build.ts:39:21  = → +=   # retainedId starts at 0 and buildSite invokes retain exactly once, so both assignments store row.id
 src/shared/db/built-site-scheduler.ts:11:36  ?? → ||   # scheduledTaskKey is null or a non-empty canonical key, so both operators select the candidate only when no key exists
-src/shared/deno-deploy-api.ts:85:34  ?? → ||   # env_vars is a record object when present, and every object is truthy
+src/shared/deno-deploy-api.ts:139:30  ?? → ||   # failure_reason is null or a non-empty picklist value, so both use the status only for null
 src/shared/bunny-cdn.ts:477:42  ?? → ||   # DefaultHostname is string|undefined; its only falsy string is already the same empty fallback
 src/shared/bunny-cdn.ts:529:50  ?? → ||   # Secrets is an array|null, and every present array is truthy while null takes the fallback either way
 src/shared/subrequest-budget.ts:38:40  ?? → ||   # scoped counts is an object when present, and every object is truthy

src/shared/config.ts

@@ -212,9 +212,11 @@ export const isInstanceApiEnabled = (): boolean =>
 /** The shared secret authorizing the inter-instance site-credentials endpoint. */
 export const getMainInstanceKey = (): string => requireEnv("MAIN_INSTANCE_KEY");
 
-/** Check if Deno Deploy hosting is enabled (requires DENO_DEPLOY_TOKEN and DENO_DEPLOY_ORG_ID). */
+/** Check if Deno Deploy hosting has its token, organization ID, and domain slug. */
 export const isDenoDeployEnabled = (): boolean =>
-  !!getEnv("DENO_DEPLOY_TOKEN") && !!getEnv("DENO_DEPLOY_ORG_ID");
+  !!getEnv("DENO_DEPLOY_TOKEN") &&
+  !!getEnv("DENO_DEPLOY_ORG_ID") &&
+  !!getEnv("DENO_DEPLOY_ORG_SLUG");
 
 /** Get the Deno Deploy API token from environment. */
 export const getDenoDeployToken = (): string => requireEnv("DENO_DEPLOY_TOKEN");
@@ -223,6 +225,10 @@ export const getDenoDeployToken = (): string => requireEnv("DENO_DEPLOY_TOKEN");
 export const getDenoDeployOrgId = (): string =>
   requireEnv("DENO_DEPLOY_ORG_ID");
 
+/** Get the Deno Deploy organization slug used in managed production domains. */
+export const getDenoDeployOrgSlug = (): string =>
+  requireEnv("DENO_DEPLOY_ORG_SLUG");
+
 /** Get the default database provider from DEFAULT_DB_HOST env var. Returns "turso" when set to "turso", "bunny" otherwise. */
 export const getDefaultDbProvider = (): "bunny" | "turso" =>
   getEnv("DEFAULT_DB_HOST") === "turso" ? "turso" : "bunny";

src/shared/db/built-sites/blob.ts

@@ -35,8 +35,11 @@ const siteDataFields = {
   u: v.string(),
 };
 
-const SiteDataBlobSchema = v.variant("v", [
-  v.strictObject({ ...siteDataFields, v: v.literal(1) }),
+const SiteDataBlobSchema = v.union([
+  v.strictObject({
+    ...siteDataFields,
+    v: v.optional(v.literal(1), 1),
+  }),
   v.strictObject({
     ...siteDataFields,
     sk: v.optional(v.pipe(v.string(), v.check(isScheduledTaskKey))),

src/shared/deno-deploy-api.ts

@@ -6,40 +6,54 @@
  * Auth: Authorization: Bearer {DENO_DEPLOY_TOKEN}
  */
 
+import * as v from "valibot";
 /* jscpd:ignore-start */
 import {
   getDenoDeployOrgId,
+  getDenoDeployOrgSlug,
   getDenoDeployToken,
   slugifyForProvider,
 } from "#shared/config.ts";
+import {
+  DenoAppEnvVarsSchema,
+  DenoAppIdentitySchema,
+  type DenoRevision,
+  DenoRevisionSchema,
+} from "#shared/deno-deploy-schema.ts";
+import { errorMessage } from "#shared/error-message.ts";
 import { fetchText, parseApiError } from "#shared/fetch.ts";
 import type {
   HostingProviderApi,
   PrepareSiteFn,
 } from "#shared/provider-types.ts";
 import { errorResult, okResult, type Result } from "#shared/result.ts";
+import { retryWithBackoff } from "#shared/retry.ts";
 
 /* jscpd:ignore-end */
 
 const DENO_API_BASE = "https://api.deno.com/v2";
 
-interface CreateAppResponse {
-  id: string;
-  slug: string;
-}
-
-interface GetAppResponse {
-  env_vars?: Record<string, { value: string; is_secret: boolean }>;
-  id: string;
-  slug: string;
-}
-
 /** Headers for all Deno Deploy API requests. */
 const denoApiHeaders = (): Record<string, string> => ({
   Authorization: `Bearer ${getDenoDeployToken()}`,
   "Content-Type": "application/json",
 });
 
+const parseDenoRevision = (text: string): DenoRevision =>
+  v.parse(DenoRevisionSchema, JSON.parse(text));
+
+const getDenoApi = async <T>(
+  path: string,
+  label: string,
+  parse: (text: string) => T,
+): Promise<Result<T>> => {
+  const res = await fetchText(`${DENO_API_BASE}/${path}`, {
+    headers: denoApiHeaders(),
+  });
+  if (!res.ok) return parseApiError(res, label);
+  return okResult(parse(res.text));
+};
+
 /**
  * Sanitize a site name into a valid Deno Deploy slug.
  * Rules: 3–32 chars, lowercase letters/numbers/hyphens, no leading/trailing hyphens.
@@ -66,21 +80,21 @@ const createAppImpl = async (
 
   if (!res.ok) return parseApiError(res, "Create app");
 
-  const data: CreateAppResponse = JSON.parse(res.text);
+  const data = v.parse(DenoAppIdentitySchema, JSON.parse(res.text));
   return okResult({ appId: data.id, slug: data.slug });
 };
 
 /** Fetch the current env vars for a Deno Deploy app. */
-const fetchAppEnvVars = async (
+const fetchAppEnvVarNames = async (
   appId: string,
-): Promise<Result<Record<string, { value: string; is_secret: boolean }>>> => {
-  const res = await fetchText(
-    `${DENO_API_BASE}/apps/${encodeURIComponent(appId)}`,
-    { headers: denoApiHeaders() },
+): Promise<Result<string[]>> => {
+  const result = await getDenoApi(
+    `apps/${encodeURIComponent(appId)}`,
+    "Get app",
+    (text) => v.parse(DenoAppEnvVarsSchema, JSON.parse(text)),
   );
-  if (!res.ok) return parseApiError(res, "Get app");
-  const data: GetAppResponse = JSON.parse(res.text);
-  return okResult(data.env_vars ?? {});
+  if (!result.ok) return result;
+  return okResult(result.value.env_vars.map(({ key }) => key));
 };
 
 /**
@@ -111,6 +125,68 @@ const setEnvVarsImpl = async (appId: string, secrets: [string, string][]) => {
   return okResult(undefined);
 };
 
+const REVISION_POLL_BACKOFF_MS = Array<number>(20).fill(1_000);
+
+class RevisionPendingError extends Error {}
+
+const revisionOutcome = (revision: DenoRevision): Result<void> | null => {
+  if (revision.status === "succeeded") return okResult(undefined);
+  if (revision.status === "queued" || revision.status === "building") {
+    return null;
+  }
+  return errorResult(
+    `Deno revision ${revision.id} ${revision.status}: ${
+      revision.failure_reason ?? revision.status
+    }`,
+  );
+};
+
+const fetchRevision = async (
+  revisionId: string,
+): Promise<Result<DenoRevision>> =>
+  getDenoApi(
+    `revisions/${encodeURIComponent(revisionId)}`,
+    "Get revision",
+    parseDenoRevision,
+  );
+
+const waitForRevision = async (
+  initial: DenoRevision,
+): Promise<Result<void>> => {
+  let current = initial;
+  let fetchNext = false;
+  try {
+    return await retryWithBackoff(
+      async () => {
+        if (fetchNext) {
+          const fetched = await fetchRevision(initial.id);
+          if (!fetched.ok) throw new Error(fetched.error);
+          current = fetched.value;
+        }
+        const outcome = revisionOutcome(current);
+        if (outcome) return outcome;
+        fetchNext = true;
+        throw new RevisionPendingError();
+      },
+      REVISION_POLL_BACKOFF_MS,
+      (error, { willRetry }) => {
+        if (willRetry) return;
+        if (error instanceof RevisionPendingError) {
+          throw new Error(
+            `Deno revision ${initial.id} did not finish within ${REVISION_POLL_BACKOFF_MS.length} seconds`,
+          );
+        }
+        throw new Error(
+          `Deno revision ${initial.id} could not be read: ${errorMessage(error)}`,
+          { cause: error },
+        );
+      },
+    );
+  } catch (error) {
+    return errorResult(errorMessage(error));
+  }
+};
+
 /**
  * Deploy code to a Deno Deploy app (production deployment).
  * Returns the primary hostname for the deployment.
@@ -135,18 +211,15 @@ const deployCodeImpl: HostingProviderApi["publishSite"] = async (
   );
 
   if (!res.ok) return parseApiError(res, "Deploy code");
-  return okResult(undefined);
+  return waitForRevision(parseDenoRevision(res.text));
 };
 
 /**
  * Get the names of environment variables currently set on a Deno Deploy app.
  * Used by the secrets backfill UI to diff against the expected set.
  */
-const getEnvVarNamesImpl = async (appId: string): Promise<Result<string[]>> => {
-  const appResult = await fetchAppEnvVars(appId);
-  if (!appResult.ok) return appResult;
-  return okResult(Object.keys(appResult.value));
-};
+const getEnvVarNamesImpl = async (appId: string): Promise<Result<string[]>> =>
+  fetchAppEnvVarNames(appId);
 
 /** Stubbable API for testing */
 export const denoDeployApi = {
@@ -167,7 +240,7 @@ const prepareDenoSiteImpl: PrepareSiteFn = async (name, _code, secrets) => {
     return errorResult(`Failed to set secrets: ${setResult.error}`);
   }
   return okResult({
-    defaultHostname: `https://${createResult.value.slug}.deno.dev`,
+    defaultHostname: `https://${createResult.value.slug}.${getDenoDeployOrgSlug()}.deno.net`,
     hostingId: createResult.value.appId,
   });
 };

src/shared/deno-deploy-schema.ts

@@ -0,0 +1,31 @@
+import * as v from "valibot";
+
+export const DenoAppIdentitySchema = v.object({
+  id: v.string(),
+  slug: v.string(),
+});
+
+export const DenoAppEnvVarsSchema = v.object({
+  env_vars: v.array(v.object({ key: v.string() })),
+});
+
+export const DenoRevisionStatusSchema = v.picklist([
+  "skipped",
+  "queued",
+  "building",
+  "succeeded",
+  "failed",
+]);
+
+export type DenoRevisionStatus = v.InferOutput<typeof DenoRevisionStatusSchema>;
+
+export const DenoRevisionSchema = v.object({
+  failure_reason: v.optional(
+    v.nullable(v.picklist(["error", "cancelled", "timed_out", "skipped"])),
+    null,
+  ),
+  id: v.string(),
+  status: DenoRevisionStatusSchema,
+});
+
+export type DenoRevision = v.InferOutput<typeof DenoRevisionSchema>;

src/shared/maintenance/definition.ts

@@ -20,19 +20,43 @@ export type MaintenanceTaskContext = {
   setCheckpoint: (checkpoint: string | null) => void;
 };
 
+export interface MaintenanceTaskCheck {
+  enabled: () => boolean | Promise<boolean>;
+  maxDatabaseCalls: number;
+  maxExternalCalls: number;
+  settingsKeys: readonly string[];
+}
+
 export interface MaintenanceTaskDeclaration {
+  check: MaintenanceTaskCheck;
   deadlineMs: number;
-  enabled: () => boolean | Promise<boolean>;
   failureRetryIntervalMs: number;
   intervalMs: number;
   maxDatabaseCalls: number;
   maxExternalCalls: number;
   name: string;
   run: (context: MaintenanceTaskContext) => void | Promise<void>;
-  settingsKeys: readonly string[];
   wakePolicy: MaintenanceWakePolicy;
 }
 
+export const maintenanceStartupCalls = (
+  tasks: readonly MaintenanceTaskDeclaration[],
+): { database: number; external: number; total: number } => {
+  if (tasks.length === 0) return { database: 0, external: 0, total: 0 };
+  const settingsRead = tasks.some((task) => task.check.settingsKeys.length > 0)
+    ? 1
+    : 0;
+  const database =
+    1 +
+    settingsRead +
+    tasks.reduce((sum, task) => sum + task.check.maxDatabaseCalls, 0);
+  const external = tasks.reduce(
+    (sum, task) => sum + task.check.maxExternalCalls,
+    0,
+  );
+  return { database, external, total: database + external };
+};
+
 const assertNonNegativeInteger = (
   task: MaintenanceTaskDeclaration,
   value: number,
@@ -72,6 +96,16 @@ const validateTask = (task: MaintenanceTaskDeclaration): void => {
   }
   assertNonNegativeInteger(task, task.maxDatabaseCalls, "database");
   assertNonNegativeInteger(task, task.maxExternalCalls, "external");
+  assertNonNegativeInteger(
+    task,
+    task.check.maxDatabaseCalls,
+    "enabled-check database",
+  );
+  assertNonNegativeInteger(
+    task,
+    task.check.maxExternalCalls,
+    "enabled-check external",
+  );
   const calls = task.maxDatabaseCalls + task.maxExternalCalls;
   if (calls > MAINTENANCE_TASK_CALL_LIMIT) {
     throw new Error(
@@ -93,6 +127,12 @@ export const defineMaintenanceTasks = <
     names.add(task.name);
     validateTask(task);
   }
+  const startup = maintenanceStartupCalls(tasks);
+  if (startup.total > MAINTENANCE_REQUEST_CALL_LIMIT) {
+    throw new Error(
+      `Maintenance checks declare ${startup.total} startup calls; maximum is ${MAINTENANCE_REQUEST_CALL_LIMIT}`,
+    );
+  }
   return tasks;
 };