Skip to content
Merged
Show file tree
Hide file tree
Changes from 3 commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 2 additions & 2 deletions scripts/mutation/equivalent-mutants.txt
Original file line number Diff line number Diff line change
Expand Up @@ -1040,8 +1040,8 @@ src/shared/cache-registry.ts:126:47 ?? → || # get(): Set|undefined — a Se

# Stripe request null checks: both values are typed as string|null, never undefined,
# so strict and loose null equality agree for every possible input.
src/shared/stripe/request.ts:89:17 === → == # retryAfter is string|null; both comparisons are true only for null
src/shared/stripe/request.ts:151:15 === → == # Headers.get returns string|null; both comparisons are true only for null
src/shared/stripe/request.ts:90:17 === → == # retryAfter is string|null; both comparisons are true only for null
src/shared/stripe/request.ts:152:15 === → == # Headers.get returns string|null; both comparisons are true only for null

# A header part without "=" has no value. Its key cannot affect timestamp or
# signature state: a timestamp assignment remains undefined and a v1 signature
Expand Down
23 changes: 8 additions & 15 deletions src/shared/db/activity-log-backfill.ts
Original file line number Diff line number Diff line change
Expand Up @@ -7,21 +7,16 @@
* with the env key, re-encrypt under the owner's public key, write it back.
* Only the public key is needed (no password), so it runs unattended.
*
* It is resumable without a cursor: a converted row no longer matches the
* `enc:` prefix, so each batch shrinks the remaining set. A batch that finds
* nothing leaves the task disabled until more legacy data exists.
* It is resumable without a row cursor: a converted row no longer matches the
* `enc:` prefix, so each batch shrinks the remaining set. The maintenance task
* stores a completion checkpoint after the final batch, avoiding future scans.
*/

/* jscpd:ignore-start */
import { decrypt, ENCRYPTION_PREFIX } from "#shared/crypto/encryption.ts";
import { encryptWithOwnerKey } from "#shared/crypto/keys.ts";
import type { EnvKeyEncrypted } from "#shared/crypto/sealed.ts";
import {
executeBatch,
queryAll,
rowExists,
update,
} from "#shared/db/client.ts";
import { executeBatch, queryAll, update } from "#shared/db/client.ts";
import { ACTIVITY_LOG_BACKFILL_BATCH } from "#shared/limits.ts";
import { logDebug } from "#shared/logger.ts";

Expand All @@ -32,6 +27,8 @@ import { logDebug } from "#shared/logger.ts";
// legacy env-key ciphertext.
type LegacyRow = { id: number; message: EnvKeyEncrypted };

export const ACTIVITY_LOG_BACKFILL_COMPLETE = "complete";

/**
* Re-encrypt one batch of legacy env-key rows to the owner key. Returns the
* number of rows converted (0 when none remain). All the rewrites land in a
Expand Down Expand Up @@ -64,16 +61,12 @@ export const backfillActivityLogBatch = async (
return rows.length;
};

export const hasLegacyActivityLog = (): Promise<boolean> =>
rowExists("SELECT 1 FROM activity_log WHERE message LIKE ? LIMIT 1", [
`${ENCRYPTION_PREFIX}%`,
]);

export const runActivityLogBackfill = async (
publicKey: string,
): Promise<void> => {
): Promise<number> => {
const converted = await backfillActivityLogBatch(publicKey);
if (converted > 0) {
logDebug("Backfill", `activity_log: re-encrypted ${converted} rows`);
}
return converted;
};
15 changes: 15 additions & 0 deletions src/shared/db/migrations/2026-07-21_activity_backfill_complete.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,15 @@
import { ACTIVITY_LOG_BACKFILL_COMPLETE } from "#shared/db/activity-log-backfill.ts";
import { bareSchemaMigration } from "./define.ts";

export default bareSchemaMigration(
"2026-07-21_activity_backfill_complete",
"Keep completed activity log backfills complete after moving to checkpoints.",
async ({ getDb }) => {
await getDb().execute({
args: ["activity_log_backfill", ACTIVITY_LOG_BACKFILL_COMPLETE],
sql: `INSERT OR IGNORE INTO maintenance_tasks
(name, next_run_at, checkpoint)
VALUES (?, 0, ?)`,
});
},
);
5 changes: 5 additions & 0 deletions src/shared/db/migrations/registry.ts
Original file line number Diff line number Diff line change
Expand Up @@ -372,6 +372,11 @@ export const MIGRATION_REGISTRY: MigrationRegistryEntry[] = [
"2026-07-19_maintenance_checkpoint",
() => import("./2026-07-19_maintenance_checkpoint.ts"),
),
// Preserve the old migration's missing-row signal as a completed checkpoint.
entry(
"2026-07-21_activity_backfill_complete",
Comment thread
stefan-burke marked this conversation as resolved.
() => import("./2026-07-21_activity_backfill_complete.ts"),
),
];
/* jscpd:ignore-end */

Expand Down
48 changes: 28 additions & 20 deletions src/shared/maintenance/registry.ts
Original file line number Diff line number Diff line change
@@ -1,25 +1,33 @@
import { hasLegacyActivityLog } from "#shared/db/activity-log-backfill.ts";
import { ACTIVITY_LOG_BACKFILL_COMPLETE } from "#shared/db/activity-log-backfill.ts";
import { settings } from "#shared/db/settings.ts";
import {
ACTIVITY_LOG_BACKFILL_BATCH,
ACTIVITY_LOG_BACKFILL_INTERVAL_MS,
PRUNE_INTERVAL_MS,
} from "#shared/limits.ts";
import { defineMaintenanceTasks } from "#shared/maintenance/definition.ts";
import {
defineMaintenanceTasks,
type MaintenanceTaskCheck,
} from "#shared/maintenance/definition.ts";
import { CONFIG_KEYS } from "#shared/settings/keys.ts";

const FAILURE_RETRY_MS = 5 * 60 * 1000;

const alwaysEnabled = (
settingsKeys: readonly string[],
): MaintenanceTaskCheck => ({
enabled: () => true,
maxDatabaseCalls: 0,
maxExternalCalls: 0,
settingsKeys,
});

export const MAINTENANCE_TASKS = defineMaintenanceTasks([
{
check: {
enabled: () => true,
maxDatabaseCalls: 0,
maxExternalCalls: 0,
settingsKeys: [
CONFIG_KEYS.AUTO_PURGE_ORPHANS,
CONFIG_KEYS.ORPHAN_PURGE_RETENTION,
],
},
check: alwaysEnabled([
CONFIG_KEYS.AUTO_PURGE_ORPHANS,
CONFIG_KEYS.ORPHAN_PURGE_RETENTION,
]),
deadlineMs: 15_000,
failureRetryIntervalMs: FAILURE_RETRY_MS,
intervalMs: PRUNE_INTERVAL_MS,
Expand All @@ -35,24 +43,24 @@ export const MAINTENANCE_TASKS = defineMaintenanceTasks([
wakePolicy: "organic_safe",
},
{
check: {
enabled: async () =>
Boolean(settings.publicKey) && hasLegacyActivityLog(),
maxDatabaseCalls: 1,
maxExternalCalls: 0,
settingsKeys: [CONFIG_KEYS.PUBLIC_KEY],
},
check: alwaysEnabled([CONFIG_KEYS.PUBLIC_KEY]),
Comment thread
stefan-burke marked this conversation as resolved.
deadlineMs: 10_000,
failureRetryIntervalMs: ACTIVITY_LOG_BACKFILL_INTERVAL_MS,
intervalMs: ACTIVITY_LOG_BACKFILL_INTERVAL_MS,
maxDatabaseCalls: 2,
maxExternalCalls: 0,
name: "activity_log_backfill",
run: async () => {
run: async ({ checkpoint, requestFollowUp, setCheckpoint }) => {
if (checkpoint === ACTIVITY_LOG_BACKFILL_COMPLETE) return;
const { runActivityLogBackfill } = await import(
"#shared/db/activity-log-backfill.ts"
);
await runActivityLogBackfill(settings.publicKey);
const converted = await runActivityLogBackfill(settings.publicKey);
if (converted < ACTIVITY_LOG_BACKFILL_BATCH) {
setCheckpoint(ACTIVITY_LOG_BACKFILL_COMPLETE);
} else {
requestFollowUp();
}
},
wakePolicy: "organic_safe",
},
Expand Down
2 changes: 2 additions & 0 deletions src/shared/stripe/request.ts
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
import * as v from "valibot";
import { namedError } from "#shared/named-error.ts";
import { delay } from "#shared/now.ts";
import { countExternalSubrequest } from "#shared/subrequest-budget.ts";
import { encodeStripeForm, type StripeFormValue } from "./form.ts";
import { parseStripeErrorBody } from "./schemas.ts";

Expand Down Expand Up @@ -288,6 +289,7 @@ export const createStripeRequest = (

async function attempt(retry: number): Promise<T> {
let response: Response;
countExternalSubrequest("Stripe API request");
try {
response = await config.fetch(url, {
...(method === "POST" ? { body: encoded } : {}),
Expand Down
5 changes: 3 additions & 2 deletions test/integration/migration-restore-verify.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -67,8 +67,9 @@ describeWithEnv(
// covered by its own data test), and removal-only migrations whose
// absent-table checks cannot be rebuilt by a restore case. enabled_features
// now owns the triggers that keep saved feature data and visibility in step.
// The built-site marker drop is also removal-only.
expect(additiveMigrations.length).toBe(MIGRATIONS.length - 21);
// The built-site marker drop is also removal-only. The activity backfill
// completion migration is data-only and covered by its direct tests.
expect(additiveMigrations.length).toBe(MIGRATIONS.length - 22);
});

test("restores triggers attached to a dropped table", async () => {
Expand Down
40 changes: 32 additions & 8 deletions test/shared/db/activity-log-backfill.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -5,11 +5,10 @@ import { ENCRYPTION_PREFIX } from "#shared/crypto/encryption.ts";
import { HYBRID_PREFIX } from "#shared/crypto/keys.ts";
import {
backfillActivityLogBatch,
hasLegacyActivityLog,
runActivityLogBackfill,
} from "#shared/db/activity-log-backfill.ts";
import { getAllActivityLog, logActivity } from "#shared/db/activityLog.ts";
import { execute } from "#shared/db/client.ts";
import { execute, queryOne } from "#shared/db/client.ts";
import { settings } from "#shared/db/settings.ts";
import { setSuppressDebugLogs } from "#shared/log-settings.ts";
import { MAINTENANCE_TASKS } from "#shared/maintenance/registry.ts";
Expand All @@ -20,10 +19,11 @@ import {
rawActivityMessage,
} from "#test-utils/activity-log.ts";
import { describeWithEnv } from "#test-utils/db.ts";
import { recordQueries } from "#test-utils/record-queries.ts";
import { withTestSession } from "#test-utils/session.ts";

const captureBackfillLogs = async (
operation: () => Promise<void>,
operation: () => Promise<unknown>,
): Promise<string[]> => {
setSuppressDebugLogs(false);
const debugStub = stub(console, "debug");
Expand Down Expand Up @@ -116,12 +116,11 @@ describeWithEnv("db > activity log backfill", { db: true }, () => {
expect(logs).toEqual([]);
});

test("reports whether legacy work remains", async () => {
expect(await hasLegacyActivityLog()).toBe(false);
test("returns the size of each remaining batch", async () => {
expect(await runActivityLogBackfill(settings.publicKey)).toBe(0);
await insertLegacyActivity("legacy");
expect(await hasLegacyActivityLog()).toBe(true);
await runActivityLogBackfill(settings.publicKey);
expect(await hasLegacyActivityLog()).toBe(false);
expect(await runActivityLogBackfill(settings.publicKey)).toBe(1);
expect(await runActivityLogBackfill(settings.publicKey)).toBe(0);
});

test("runs the legacy backfill through the maintenance registry", async () => {
Expand All @@ -132,6 +131,31 @@ describeWithEnv("db > activity log backfill", { db: true }, () => {
expect((await rawActivityMessage(id)).startsWith(HYBRID_PREFIX)).toBe(true);
});

test("a completed task does not scan the activity log again", async () => {
await maintenance.run(MAINTENANCE_TASKS);
expect(
await queryOne<{ checkpoint: string }>(
"SELECT checkpoint FROM maintenance_tasks WHERE name = 'activity_log_backfill'",
),
).toEqual({ checkpoint: "complete" });
await execute(
"UPDATE maintenance_tasks SET next_run_at = 0 WHERE name = 'activity_log_backfill'",
);
const queries: string[] = [];
const restore = recordQueries(queries);
try {
await maintenance.run(MAINTENANCE_TASKS);
} finally {
restore();
}

expect(
queries.filter((sql) =>
sql.includes("FROM activity_log WHERE message LIKE"),
),
).toEqual([]);
});

test("surfaces a corrupt legacy message to the task runner", async () => {
await execute(
"INSERT INTO activity_log (message, created, listing_id, attendee_id) VALUES (?, ?, NULL, NULL)",
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,64 @@
import { expect } from "@std/expect";
import { it as test } from "@std/testing/bdd";
import { ACTIVITY_LOG_BACKFILL_COMPLETE } from "#shared/db/activity-log-backfill.ts";
import { getDb } from "#shared/db/client.ts";
import activityBackfillCompleteMigration from "#shared/db/migrations/2026-07-21_activity_backfill_complete.ts";
import { describeWithEnv } from "#test-utils/db.ts";
import { buildMigrationContext } from "#test-utils/migrations.ts";

const context = buildMigrationContext({});
const runMigration = () => activityBackfillCompleteMigration(context).up();
const activityTask = () =>
getDb().execute(
"SELECT checkpoint, next_run_at FROM maintenance_tasks WHERE name = 'activity_log_backfill'",
);

describeWithEnv(
"db > migrations > completed activity backfill",
{ db: true },
() => {
test("declares its identity", () => {
const migration = activityBackfillCompleteMigration(context);
expect({
description: migration.description,
id: migration.id,
requires: migration.requires,
}).toEqual({
description:
"Keep completed activity log backfills complete after moving to checkpoints.",
id: "2026-07-21_activity_backfill_complete",
requires: {},
});
});

test("restores the completed checkpoint for an absent old task", async () => {
await getDb().execute(
"DELETE FROM maintenance_tasks WHERE name = 'activity_log_backfill'",
);

await runMigration();

expect((await activityTask()).rows).toEqual([
{
checkpoint: ACTIVITY_LOG_BACKFILL_COMPLETE,
next_run_at: 0,
},
]);
});

test("keeps an existing unfinished task", async () => {
await getDb().execute({
args: ["activity_log_backfill", 123],
sql: `INSERT OR REPLACE INTO maintenance_tasks
(name, next_run_at, checkpoint)
VALUES (?, ?, NULL)`,
});

await runMigration();

expect((await activityTask()).rows).toEqual([
{ checkpoint: null, next_run_at: 123 },
]);
});
Comment thread
coderabbitai[bot] marked this conversation as resolved.
},
);
3 changes: 2 additions & 1 deletion test/shared/db/migrations/registry.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -31,10 +31,11 @@ describe("db > migration registry", () => {
});

test("orders every scheduled-maintenance schema change", () => {
expect(MIGRATION_IDS.slice(-3)).toEqual([
expect(MIGRATION_IDS.slice(-4)).toEqual([
"2026-07-18_maintenance_tasks",
"2026-07-18_drop_built_sites_last_pruned",
"2026-07-19_maintenance_checkpoint",
"2026-07-21_activity_backfill_complete",
]);
});
});
1 change: 1 addition & 0 deletions test/shared/db/migrations/schema/version/guard.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -98,6 +98,7 @@ describe("db > migrations > schema change guard", () => {
"2026-07-18_maintenance_tasks",
"2026-07-18_drop_built_sites_last_pruned",
"2026-07-19_maintenance_checkpoint",
"2026-07-21_activity_backfill_complete",
],
schemaHash: "dhpq62",
});
Expand Down
Loading
Loading