fix: self-heal claims/earnings indexes (#886) + accepted signal filter (#873)

src/__tests__/schema-migration.test.ts

@@ -1,10 +1,15 @@
 import { describe, it, expect } from "vitest";
 import { SELF } from "cloudflare:test";
-import { EXPECTED_SIGNALS_INDEXES } from "../objects/schema";
+import {
+  EXPECTED_SIGNALS_INDEXES,
+  EXPECTED_POST_MIGRATION_INDEXES,
+  SELF_HEAL_POST_MIGRATION_INDEXES_SQL,
+} from "../objects/schema";
 
 interface SchemaHealthBody {
   healthy: boolean;
   missing_signals_indexes: string[];
+  missing_post_migration_indexes: string[];
   signals_row_count: number | null;
   live_index_count: number;
   live_indexes: string[];
@@ -120,3 +125,55 @@ describe("GET /api/config/schema-health", () => {
     expect(typeof withCount.signals_row_count).toBe("number");
   });
 });
+
+/**
+ * #886 — the claims/earnings hot-path indexes cannot live in SCHEMA_SQL
+ * (beat_claims is created by migration 8, earnings.voided_at added by
+ * migration 9, and SCHEMA_SQL runs before both). They are re-applied after the
+ * migration block on every cold start so a silently-failed migration cannot
+ * leave them permanently missing — the failure mode behind the April
+ * full-table-scan billing incident.
+ */
+describe("post-migration index self-heal (#886)", () => {
+  it("creates every expected claims/earnings index on a fresh DB", async () => {
+    const res = await SELF.fetch("http://example.com/api/config/schema-health");
+    expect(res.status).toBe(200);
+    const body = await res.json<SchemaHealthBody>();
+    expect(body.missing_post_migration_indexes).toEqual([]);
+    for (const idx of EXPECTED_POST_MIGRATION_INDEXES) {
+      expect(body.live_indexes).toContain(idx);
+    }
+  });
+
+  it("folds the post-migration set into the overall healthy flag", async () => {
+    const body = await (
+      await SELF.fetch("http://example.com/api/config/schema-health")
+    ).json<SchemaHealthBody>();
+    expect(body.healthy).toBe(
+      body.missing_signals_indexes.length === 0 &&
+        body.missing_post_migration_indexes.length === 0
+    );
+    expect(body.healthy).toBe(true);
+  });
+
+  it("keeps the EXPECTED list in sync with the SQL that creates the indexes", async () => {
+    // Two hand-maintained lists: the CREATE statements and the names
+    // /schema-health diffs against. If they drift, a dropped index stops being
+    // reported and the self-heal becomes invisible — so pin them together.
+    const createdNames = SELF_HEAL_POST_MIGRATION_INDEXES_SQL.map((stmt) => {
+      const match = /CREATE INDEX IF NOT EXISTS (\S+)/.exec(stmt);
+      expect(match, `not an idempotent CREATE INDEX: ${stmt}`).not.toBeNull();
+      return match![1];
+    });
+    expect([...createdNames].sort()).toEqual(
+      [...EXPECTED_POST_MIGRATION_INDEXES].sort()
+    );
+  });
+
+  it("uses only idempotent statements — the pass re-runs on every cold start", async () => {
+    // Not version-gated, so every statement must tolerate being re-applied.
+    for (const stmt of SELF_HEAL_POST_MIGRATION_INDEXES_SQL) {
+      expect(stmt).toContain("IF NOT EXISTS");
+    }
+  });
+});

src/__tests__/signals.test.ts

@@ -295,3 +295,88 @@ describe("POST /api/signals — validation errors", () => {
     expect(body.error).toContain("not found");
   });
 });
+
+const ACCEPTED_TAG = "accepted-virtual-filter-873";
+
+/**
+ * #873 — an accepted signal moves approved → brief_included when a brief
+ * compiles, so neither exact status alone yields the durable accepted set.
+ * The virtual `status=accepted` filter spans both without disturbing the
+ * existing exact-lifecycle filters.
+ */
+describe("GET /api/signals?status=accepted — virtual accepted filter", () => {
+  const fetchIds = async (query: string): Promise<string[]> => {
+    const res = await SELF.fetch(`http://example.com/api/signals?${query}`);
+    expect(res.status).toBe(200);
+    const body = await res.json<{ signals: Array<{ id: string }> }>();
+    return body.signals.map((s) => s.id).sort();
+  };
+
+  it("spans approved + brief_included while leaving exact filters unchanged", async () => {
+    await seed({
+      signals: [
+        {
+          id: "accepted-873-approved",
+          beat_slug: "agent-social",
+          btc_address: "bc1qar0srrr7xfkvy5l643lydnw9re59gtzzwf5mdq",
+          headline: "Accepted filter — approved",
+          sources: "[]",
+          created_at: "2026-07-17T12:03:00.000Z",
+          status: "approved",
+          reviewed_at: "2026-07-17T12:04:00.000Z",
+        },
+        {
+          id: "accepted-873-compiled",
+          beat_slug: "agent-social",
+          btc_address: "bc1qar0srrr7xfkvy5l643lydnw9re59gtzzwf5mdq",
+          headline: "Accepted filter — brief_included",
+          sources: "[]",
+          created_at: "2026-07-17T12:02:00.000Z",
+          status: "brief_included",
+          reviewed_at: "2026-07-17T12:04:00.000Z",
+        },
+        {
+          id: "accepted-873-rejected",
+          beat_slug: "agent-social",
+          btc_address: "bc1qar0srrr7xfkvy5l643lydnw9re59gtzzwf5mdq",
+          headline: "Accepted filter — rejected",
+          sources: "[]",
+          created_at: "2026-07-17T12:01:00.000Z",
+          status: "rejected",
+          reviewed_at: "2026-07-17T12:04:00.000Z",
+        },
+      ],
+      signal_tags: [
+        { signal_id: "accepted-873-approved", tag: ACCEPTED_TAG },
+        { signal_id: "accepted-873-compiled", tag: ACCEPTED_TAG },
+        { signal_id: "accepted-873-rejected", tag: ACCEPTED_TAG },
+      ],
+    });
+
+    // 1. `accepted` returns both accepted states and excludes the rejected one.
+    expect(await fetchIds(`tag=${ACCEPTED_TAG}&status=accepted`)).toEqual([
+      "accepted-873-approved",
+      "accepted-873-compiled",
+    ]);
+
+    // 2 + 3. Exact lifecycle filters keep exact-state behaviour.
+    expect(await fetchIds(`tag=${ACCEPTED_TAG}&status=approved`)).toEqual([
+      "accepted-873-approved",
+    ]);
+    expect(await fetchIds(`tag=${ACCEPTED_TAG}&status=brief_included`)).toEqual([
+      "accepted-873-compiled",
+    ]);
+    expect(await fetchIds(`tag=${ACCEPTED_TAG}&status=rejected`)).toEqual([
+      "accepted-873-rejected",
+    ]);
+  });
+
+  it("rejects an unknown status but advertises `accepted` as valid", async () => {
+    const res = await SELF.fetch(
+      "http://example.com/api/signals?status=not-a-status"
+    );
+    expect(res.status).toBe(400);
+    const body = await res.json<{ error: string }>();
+    expect(body.error).toContain("accepted");
+  });
+});

src/lib/constants.ts

@@ -128,6 +128,33 @@ export const REVIEWABLE_SIGNAL_STATUSES = [
   "rejected",
 ] as const;
 
+/**
+ * Virtual list filter spanning both terminal "editorially accepted" states.
+ *
+ * An accepted signal moves `approved` → `brief_included` when a brief compiles,
+ * so neither exact status alone yields the durable accepted set: the same item
+ * disappears from a `status=approved` query once it is compiled. This filter
+ * makes retrospective "what did this correspondent get accepted" queries stable
+ * across compilation (#873).
+ */
+export const ACCEPTED_STATUS_FILTER = "accepted" as const;
+
+/** Concrete statuses ACCEPTED_STATUS_FILTER expands to. */
+export const ACCEPTED_SIGNAL_STATUSES = ["approved", "brief_included"] as const;
+
+/**
+ * Statuses accepted by `GET /api/signals?status=`. Superset of SIGNAL_STATUSES
+ * that adds the virtual `accepted` filter.
+ *
+ * Deliberately a separate constant: SIGNAL_STATUSES remains the exact on-disk
+ * state set that backs the SignalStatus type and the review-transition write
+ * path, and `accepted` is not a state a signal can ever be stored in.
+ */
+export const SIGNAL_LIST_STATUS_FILTERS = [
+  ...SIGNAL_STATUSES,
+  ACCEPTED_STATUS_FILTER,
+] as const;
+
 // ── Rate limits ──
 // Route-level constants are retained as call-site labels. Actual enforcement is
 // now handled by Cloudflare `ratelimits` bindings in wrangler:

src/objects/news-do.ts

@@ -4,8 +4,8 @@ import type { Context } from "hono";
 import type { Env, Beat, Signal, SignalStatus, Streak, Brief, Classified, ClassifiedStatus, Earning, Correction, ReferralCredit, BriefSignal, CompiledBriefData, DOResult, ApprovalCapInfo, IncludedSignalMetadata, CompiledSignalRow, PaymentStageKind, PaymentStageLifecycle, PaymentStageMaterialized, PaymentStagePayload, PaymentStageRecord, PaymentTerminalReason, PaymentTrackedState } from "../lib/types";
 import { validateSlug, validateHexColor, sanitizeString, validateDateFormat } from "../lib/validators";
 import { generateId, getUTCDate, getUTCYesterday, getUTCDayStart, getUTCDayEnd, getNextDate, signalContentFingerprint } from "../lib/helpers";
-import { CLASSIFIED_DURATION_DAYS, CLASSIFIED_BRIEF_SLOTS, CLASSIFIED_BRIEF_MAX_CHARS, CLASSIFIED_STATUSES, SIGNAL_COOLDOWN_HOURS, SIGNAL_DEDUP_REJECT_WINDOW_HOURS, BEAT_EXPIRY_DAYS, MAX_SIGNALS_PER_DAY, MAX_INCLUDED_SIGNALS_PER_BRIEF, MAX_APPROVED_SIGNALS_PER_DAY, SIGNAL_STATUSES, REVIEWABLE_SIGNAL_STATUSES, CONFIG_PUBLISHER_ADDRESS, BRIEF_INCLUSION_PAYOUT_SATS, SCORING_WEIGHTS, PAYMENT_STAGE_TTL_MS, PENDING_PAYMENT_STATUS } from "../lib/constants";
-import { SCHEMA_SQL, MIGRATION_PHASE0_SQL, MIGRATION_PAYMENTS_SQL, MIGRATION_BEAT_RESTRUCTURE_SQL, MIGRATION_SBTC_TRACKING_SQL, MIGRATION_CLASSIFIEDS_CLEANUP_SQL, MIGRATION_CLASSIFIEDS_REVIEW_SQL, MIGRATION_SNAPSHOTS_SQL, MIGRATION_BEAT_CLAIMS_SQL, MIGRATION_RETRACTION_SQL, MIGRATION_BEAT_NETWORK_FOCUS_SQL, MIGRATION_BITCOIN_MACRO_SQL, MIGRATION_QUANTUM_BEAT_SQL, MIGRATION_PAYMENT_STAGING_SQL, MIGRATION_APPROVAL_CAP_INDEX_SQL, MIGRATION_BEAT_EDITORS_SQL, MIGRATION_EDITORIAL_REVIEWS_SQL, MIGRATION_EDITOR_REVIEW_RATE_SQL, MIGRATION_CURATION_CLEANUP_SQL, MIGRATION_LEADERBOARD_INDEXES_SQL, MIGRATION_BEAT_CONSOLIDATION_SQL, MIGRATION_SIGNAL_SCORING_SQL, MIGRATION_APR7_EARNINGS_SQL, MIGRATION_CLASSIFIEDS_TXID_UNIQUE_SQL, MIGRATION_SIGNAL_HOT_PATH_INDEXES_SQL, MIGRATION_CORRESPONDENTS_BUNDLE_INDEXES_SQL, MIGRATION_CORRESPONDENT_STATS_SQL, MIGRATION_SIGNAL_PAYMENT_SQL, EXPECTED_SIGNALS_INDEXES } from "./schema";
+import { CLASSIFIED_DURATION_DAYS, CLASSIFIED_BRIEF_SLOTS, CLASSIFIED_BRIEF_MAX_CHARS, CLASSIFIED_STATUSES, SIGNAL_COOLDOWN_HOURS, SIGNAL_DEDUP_REJECT_WINDOW_HOURS, BEAT_EXPIRY_DAYS, MAX_SIGNALS_PER_DAY, MAX_INCLUDED_SIGNALS_PER_BRIEF, MAX_APPROVED_SIGNALS_PER_DAY, SIGNAL_STATUSES, REVIEWABLE_SIGNAL_STATUSES, CONFIG_PUBLISHER_ADDRESS, BRIEF_INCLUSION_PAYOUT_SATS, SCORING_WEIGHTS, PAYMENT_STAGE_TTL_MS, PENDING_PAYMENT_STATUS, ACCEPTED_STATUS_FILTER, ACCEPTED_SIGNAL_STATUSES } from "../lib/constants";
+import { SCHEMA_SQL, MIGRATION_PHASE0_SQL, MIGRATION_PAYMENTS_SQL, MIGRATION_BEAT_RESTRUCTURE_SQL, MIGRATION_SBTC_TRACKING_SQL, MIGRATION_CLASSIFIEDS_CLEANUP_SQL, MIGRATION_CLASSIFIEDS_REVIEW_SQL, MIGRATION_SNAPSHOTS_SQL, MIGRATION_BEAT_CLAIMS_SQL, MIGRATION_RETRACTION_SQL, MIGRATION_BEAT_NETWORK_FOCUS_SQL, MIGRATION_BITCOIN_MACRO_SQL, MIGRATION_QUANTUM_BEAT_SQL, MIGRATION_PAYMENT_STAGING_SQL, MIGRATION_APPROVAL_CAP_INDEX_SQL, MIGRATION_BEAT_EDITORS_SQL, MIGRATION_EDITORIAL_REVIEWS_SQL, MIGRATION_EDITOR_REVIEW_RATE_SQL, MIGRATION_CURATION_CLEANUP_SQL, MIGRATION_LEADERBOARD_INDEXES_SQL, MIGRATION_BEAT_CONSOLIDATION_SQL, MIGRATION_SIGNAL_SCORING_SQL, MIGRATION_APR7_EARNINGS_SQL, MIGRATION_CLASSIFIEDS_TXID_UNIQUE_SQL, MIGRATION_SIGNAL_HOT_PATH_INDEXES_SQL, MIGRATION_CORRESPONDENTS_BUNDLE_INDEXES_SQL, MIGRATION_CORRESPONDENT_STATS_SQL, MIGRATION_SIGNAL_PAYMENT_SQL, EXPECTED_SIGNALS_INDEXES, SELF_HEAL_POST_MIGRATION_INDEXES_SQL, EXPECTED_POST_MIGRATION_INDEXES } from "./schema";
 import { scoreSignal, withScoreOverride } from "../lib/signal-scorer";
 
 // ── State machine transition maps ──
@@ -128,7 +128,15 @@ function buildSignalListWhere(filters: SignalListFilters): { whereSql: string; p
     clauses.push("s.id IN (SELECT signal_id FROM signal_tags WHERE tag = ?)");
     params.push(filters.tag);
   }
-  if (filters.status) {
+  if (filters.status === ACCEPTED_STATUS_FILTER) {
+    // Virtual filter — expands to the two terminal accepted states so the set
+    // stays stable when a brief compile flips approved → brief_included (#873).
+    // Explicit IN list (not an inequality) so SQLite can still use
+    // idx_signals_status_created rather than falling back to a scan.
+    const placeholders = ACCEPTED_SIGNAL_STATUSES.map(() => "?").join(", ");
+    clauses.push(`s.status IN (${placeholders})`);
+    params.push(...ACCEPTED_SIGNAL_STATUSES);
+  } else if (filters.status) {
     clauses.push("s.status = ?");
     params.push(filters.status);
   } else if (!filters.includePending) {
@@ -1392,6 +1400,25 @@ export class NewsDO extends DurableObject<Env> {
       );
     }
 
+    // Self-heal the claims/earnings hot-path indexes on every cold start — NOT
+    // gated on appliedVersion, which is the whole point: a version-gated
+    // migration that fails once is never retried (the counter advances anyway),
+    // so its index can go permanently missing while the code believes the
+    // schema is complete. That is what turned leaderboard/correspondents reads
+    // into full-table scans during the April billing incident (#886).
+    //
+    // These run here rather than in SCHEMA_SQL because beat_claims (migration 8)
+    // and earnings.voided_at (migration 9) do not exist until the block above
+    // has run. Each statement is individually guarded so a genuinely absent
+    // table can never brick DO construction.
+    for (const stmt of SELF_HEAL_POST_MIGRATION_INDEXES_SQL) {
+      try {
+        this.ctx.storage.sql.exec(stmt);
+      } catch (e) {
+        console.error("Post-migration index self-heal failed:", stmt, e);
+      }
+    }
+
     // Schedule a keep-alive alarm if none exists. The alarm fires every 50 seconds,
     // preventing the DO from being evicted after 70-140 seconds of inactivity.
     // This eliminates cold start overhead for the singleton DO that serves all traffic.
@@ -3924,6 +3951,12 @@ export class NewsDO extends DurableObject<Env> {
       const missingSignalsIndexes = EXPECTED_SIGNALS_INDEXES.filter(
         (name) => !liveSet.has(name)
       );
+      // Claims/earnings indexes self-healed after the migration block (#886).
+      // Reported separately from the signals set so a drift report still says
+      // which layer drifted — base schema or post-migration pass.
+      const missingPostMigrationIndexes = EXPECTED_POST_MIGRATION_INDEXES.filter(
+        (name) => !liveSet.has(name)
+      );
 
       // `signals_row_count` is opt-in: COUNT(*) full-scans the signals table,
       // and this endpoint is public + unthrottled. Default off so routine /
@@ -3940,8 +3973,11 @@ export class NewsDO extends DurableObject<Env> {
       return c.json({
         ok: true,
         data: {
-          healthy: missingSignalsIndexes.length === 0,
+          healthy:
+            missingSignalsIndexes.length === 0 &&
+            missingPostMigrationIndexes.length === 0,
           missing_signals_indexes: missingSignalsIndexes,
+          missing_post_migration_indexes: missingPostMigrationIndexes,
           signals_row_count: signalsRowCount,
           live_index_count: liveIndexes.length,
           live_indexes: liveIndexes,

src/objects/schema.ts

@@ -154,8 +154,13 @@ CREATE INDEX IF NOT EXISTS idx_signals_correction_of    ON signals(correction_of
 -- base-schema index on it would throw a no-such-column error and brick DO
 -- construction before any migration could repair it. Its index stays in
 -- migration #24, where the column is guaranteed to exist.
--- TODO: extend this self-heal + EXPECTED set to claims/earnings indexes once
--- /schema-health confirms their version-gated indexes are also at risk.
+--
+-- The claims/earnings hot-path indexes get the same self-heal guarantee, but
+-- they cannot live here for exactly the quality_score reason above:
+-- beat_claims is created by migration #8 and earnings.voided_at is added by
+-- migration #9, so a base-schema index on either would throw on a fresh DB.
+-- They are re-applied after the migration block instead — see
+-- SELF_HEAL_POST_MIGRATION_INDEXES_SQL below.
 CREATE INDEX IF NOT EXISTS idx_signals_status_reviewed          ON signals(status, reviewed_at);
 CREATE INDEX IF NOT EXISTS idx_signals_correction_created       ON signals(correction_of, created_at);
 CREATE INDEX IF NOT EXISTS idx_signals_correction_btc_created   ON signals(correction_of, btc_address, created_at DESC);
@@ -212,6 +217,41 @@ export const EXPECTED_SIGNALS_INDEXES: readonly string[] = [
   // ALTER (see the SCHEMA_SQL comment above).
 ];
 
+/**
+ * Hot-path indexes on tables/columns that only exist *after* the versioned
+ * migrations have run, re-applied on every cold start so they self-heal the
+ * same way the signals composites in SCHEMA_SQL do (#886).
+ *
+ * Why these are not in SCHEMA_SQL: that block executes before any migration,
+ * and neither target exists yet on a fresh DB —
+ *   - `beat_claims` is created by migration #8 (MIGRATION_BEAT_CLAIMS_SQL)
+ *   - `earnings.voided_at` is added by migration #9 (MIGRATION_RETRACTION_SQL)
+ * so indexing either from SCHEMA_SQL would throw "no such table" / "no such
+ * column" and brick DO construction before a migration could repair it. That
+ * is the same trap documented for idx_signals_quality_score.
+ *
+ * Running them after the migration block gives the self-heal property without
+ * the bricking risk: a version-gated migration that failed once is never
+ * retried (the version counter advances regardless), which is how the April
+ * billing incident silently dropped the signals composites and turned hot
+ * reads into full-table scans.
+ */
+export const SELF_HEAL_POST_MIGRATION_INDEXES_SQL: readonly string[] = [
+  // Beat-membership gate — read on every signal filing and roster check.
+  "CREATE INDEX IF NOT EXISTS idx_beat_claims_status_beat_address ON beat_claims(status, beat_slug, btc_address)",
+  // Unpaid-earnings rollup behind the leaderboard / correspondents bundle.
+  "CREATE INDEX IF NOT EXISTS idx_earnings_unpaid_leaderboard ON earnings(voided_at, payout_txid, btc_address)",
+];
+
+/**
+ * Index names created by SELF_HEAL_POST_MIGRATION_INDEXES_SQL, diffed against
+ * live sqlite_master by GET /api/config/schema-health. Keep in sync.
+ */
+export const EXPECTED_POST_MIGRATION_INDEXES: readonly string[] = [
+  "idx_beat_claims_status_beat_address",
+  "idx_earnings_unpaid_leaderboard",
+];
+
 /**
  * Migration SQL for existing databases that lack Phase 0 columns.
  * Each statement is wrapped in a try/catch-friendly pattern (columns may already exist).

src/routes/signals.ts

@@ -1,7 +1,7 @@
 import { Hono } from "hono";
 import type { Env, AppVariables } from "../lib/types";
 import { checkRateLimit, createRateLimitMiddleware } from "../middleware/rate-limit";
-import { SIGNAL_RATE_LIMIT, SIGNAL_READ_RATE_LIMIT, SIGNAL_STATUSES, SIGNAL_PRICE_SATS, CONFIG_PUBLISHER_ADDRESS } from "../lib/constants";
+import { SIGNAL_RATE_LIMIT, SIGNAL_READ_RATE_LIMIT, SIGNAL_LIST_STATUS_FILTERS, SIGNAL_PRICE_SATS, CONFIG_PUBLISHER_ADDRESS } from "../lib/constants";
 import {
   validateBtcAddress,
   validateSlug,
@@ -85,8 +85,13 @@ signalsRouter.get("/api/signals", async (c) => {
   const status = c.req.query("status");
   const includePending = c.req.query("include_pending") === "true";
 
-  if (status && !(SIGNAL_STATUSES as readonly string[]).includes(status)) {
-    return c.json({ error: `Invalid status. Must be one of: ${SIGNAL_STATUSES.join(", ")}` }, 400);
+  // Validated against the list-filter set, which adds the virtual `accepted`
+  // filter (approved + brief_included) on top of the concrete statuses (#873).
+  if (status && !(SIGNAL_LIST_STATUS_FILTERS as readonly string[]).includes(status)) {
+    return c.json(
+      { error: `Invalid status. Must be one of: ${SIGNAL_LIST_STATUS_FILTERS.join(", ")}` },
+      400
+    );
   }
 
   // Pending visibility is author-only. Require an `agent` filter that