From 097eff9912023db5eb875729dc38dd7388df705e Mon Sep 17 00:00:00 2001 From: EricSanchezok <115814526+EricSanchezok@users.noreply.github.com> Date: Mon, 14 Sep 2026 21:25:21 +0800 Subject: [PATCH 01/14] feat(storage): make agent authority transactional Replace JSON authority with Runtime-owned SQLite and PostgreSQL storage, atomic domain mutations, durable receipts and notifications, and recoverable legacy upgrades and logical data transfers. Preserve artifact ownership, unknown domain data, rollback evidence and installed runtime portability. Co-authored-by: synergy-agent <299070056+synergy-agent@users.noreply.github.com> --- .github/workflows/build-helpers.yml | 21 + .github/workflows/ci.yml | 34 + .github/workflows/release.yml | 6 + .synergy/skill/change-persistence/SKILL.md | 16 +- README.md | 2 + docs/README.md | 1 + docs/architecture/agent-storage.md | 53 ++ docs/architecture/runtime-and-scope.md | 2 + ...026-09-14-transactional-agent-authority.md | 33 + .../migrations/transactional-agent-storage.md | 27 + docs/reference/cli.md | 8 + docs/reference/configuration.md | 9 + docs/reference/storage-and-paths.md | 261 ++---- packages/agent-integrations/src/acp/README.md | 2 +- .../agent-integrations/src/acp/cli/acp.ts | 112 +-- .../test/synergy-link/target-store.test.ts | 8 +- packages/browser-runtime/src/migration.ts | 139 +-- packages/browser-runtime/src/storage.ts | 53 +- .../browser-runtime/test/migration.test.ts | 101 +-- .../test/session-lazy-restore.test.ts | 3 +- .../test/storage-security.test.ts | 37 +- packages/cli/src/cli/cmd/data/index.ts | 2 + packages/cli/src/cli/cmd/data/shared.ts | 21 +- packages/cli/src/cli/cmd/data/snapshots.ts | 7 +- packages/cli/src/cli/cmd/data/storage.ts | 92 ++ packages/cli/src/cli/commands.ts | 10 + packages/cli/src/cli/network.ts | 28 +- packages/cli/src/daemon/spec.ts | 9 +- packages/cli/src/index.ts | 12 +- packages/cli/src/main.ts | 14 + .../connections/src/channel/diagnostics.ts | 4 +- .../src/channel/managed-project-ownership.ts | 130 +-- packages/harness/AGENTS.md | 2 + packages/harness/package.json | 11 +- packages/harness/script/benchmark-storage.ts | 63 ++ packages/harness/script/build-sqlite.ts | 70 ++ packages/harness/src/bus/index.ts | 11 +- packages/harness/src/config/config.ts | 14 +- packages/harness/src/config/domain.ts | 10 + packages/harness/src/config/schema.ts | 4 + packages/harness/src/global/index.ts | 2 + packages/harness/src/lifecycle/runtime.ts | 27 +- packages/harness/src/migration/index.ts | 79 +- .../harness/src/observability/migration.ts | 2 + packages/harness/src/observability/store.ts | 3 + .../src/observability/telemetry-worker.ts | 2 + packages/harness/src/public/persistence.ts | 3 + packages/harness/src/scope/index.ts | 135 +-- packages/harness/src/scope/migration.ts | 96 +-- packages/harness/src/session/dag.ts | 10 +- packages/harness/src/session/history.ts | 188 ++-- packages/harness/src/session/inbox.ts | 273 +++--- packages/harness/src/session/index.ts | 814 ++++++++++-------- packages/harness/src/session/input.ts | 18 +- packages/harness/src/session/invoke.ts | 4 +- packages/harness/src/session/manager.ts | 60 +- packages/harness/src/session/message-cache.ts | 171 ++-- packages/harness/src/session/message-v2.ts | 183 ++-- packages/harness/src/session/migration.ts | 208 ++--- packages/harness/src/session/nav.ts | 68 +- .../harness/src/session/part-write-buffer.ts | 111 ++- .../harness/src/session/rollout/archive.ts | 36 +- .../harness/src/session/rollout/artifact.ts | 17 +- .../session/rollout/continuation-migration.ts | 11 +- .../session/rollout/continuation-recovery.ts | 12 +- .../harness/src/session/rollout/journal.ts | 47 +- .../harness/src/session/rollout/ledger.ts | 10 +- .../harness/src/session/rollout/migration.ts | 23 +- .../harness/src/session/rollout/pending.ts | 43 +- .../harness/src/session/rollout/recovery.ts | 4 +- packages/harness/src/session/search-index.ts | 89 +- .../harness/src/session/session-import.ts | 172 ++-- .../harness/src/session/snapshot-archive.ts | 35 +- .../harness/src/session/snapshot-lease.ts | 7 +- .../harness/src/session/snapshot-lifecycle.ts | 30 +- .../src/session/snapshot-maintenance.ts | 81 +- .../harness/src/session/snapshot-records.ts | 17 +- .../harness/src/session/snapshot-store.ts | 26 +- .../harness/src/session/snapshot-transfer.ts | 3 + packages/harness/src/session/staging.ts | 63 ++ packages/harness/src/session/todo.ts | 8 +- .../session/user-message-materialization.ts | 40 +- packages/harness/src/storage/atomic-file.ts | 81 ++ packages/harness/src/storage/bootstrap.ts | 282 ++++++ packages/harness/src/storage/config.ts | 66 ++ packages/harness/src/storage/errors.ts | 43 + packages/harness/src/storage/legacy-import.ts | 444 ++++++++++ packages/harness/src/storage/maintenance.ts | 59 ++ packages/harness/src/storage/portable.ts | 155 ++++ .../harness/src/storage/postgres-driver.ts | 154 ++++ packages/harness/src/storage/queue.ts | 30 + packages/harness/src/storage/recovery.ts | 75 ++ packages/harness/src/storage/sql-contract.ts | 38 + packages/harness/src/storage/sqlite-driver.ts | 176 ++++ packages/harness/src/storage/sqlite-engine.ts | 36 + packages/harness/src/storage/sqlite-worker.ts | 55 ++ packages/harness/src/storage/storage.ts | 433 ++++------ .../src/storage/transactional-store.ts | 683 +++++++++++++++ .../cortex/cancel-queued-followups.test.ts | 4 +- .../harness/test/lifecycle/runtime.test.ts | 6 +- .../test/migration/concurrent-write.test.ts | 52 +- .../harness/test/migration/context.test.ts | 12 +- .../harness/test/migration/display.test.ts | 16 +- .../harness/test/migration/dry-run.test.ts | 30 +- .../test/migration/owner-ledger.test.ts | 2 + .../test/migration/registry-lock.test.ts | 7 +- packages/harness/test/migration/retry.test.ts | 16 +- .../harness/test/migration/rollback.test.ts | 26 +- .../test/migration/tracking-migration.test.ts | 84 +- .../continuation-kernel-empty-worker.ts | 18 +- .../fixtures/restart-while-queued-worker.ts | 4 + .../session/mutation-serialization.test.ts | 12 +- .../test/session/part-write-buffer.test.ts | 28 + .../harness/test/session/rollback.test.ts | 4 +- .../test/session/rollout-archive.test.ts | 2 + .../test/session/rollout-artifact.test.ts | 10 +- .../test/session/rollout-continuation.test.ts | 2 +- .../test/session/rollout-journal.test.ts | 23 +- .../rollout-migration-attachments.test.ts | 2 +- .../test/session/schema-registry.test.ts | 2 + packages/harness/test/session/staging.test.ts | 19 + .../session/transaction-atomicity.test.ts | 50 ++ .../harness/test/session/wake-retry.test.ts | 9 +- packages/harness/test/snapshot/lease.test.ts | 2 +- .../harness/test/storage/bootstrap.test.ts | 136 +++ packages/harness/test/storage/context.test.ts | 74 ++ .../test/storage/crash-recovery.test.ts | 95 ++ .../harness/test/storage/fixtures/README.md | 5 + .../test/storage/fixtures/v1.2.33.json | 90 ++ .../harness/test/storage/fixtures/v2.4.4.json | 99 +++ .../test/storage/fixtures/v3.0.22.json | 103 +++ .../test/storage/legacy-import.test.ts | 160 ++++ .../harness/test/storage/portable.test.ts | 63 ++ .../test/storage/postgres-ownership.test.ts | 42 + .../test/storage/released-upgrade.test.ts | 47 + .../test/storage/storage-retry.test.ts | 27 +- packages/harness/test/storage/storage.test.ts | 30 +- .../test/storage/transactional-store.test.ts | 163 ++++ .../harness/test/storage/verification.test.ts | 41 + packages/harness/test/support/preload.ts | 14 + .../test/tool/registry-observability.test.ts | 2 + packages/library/src/database.ts | 30 +- packages/note/src/store.ts | 309 ++++--- packages/note/test/storage-atomicity.test.ts | 30 + packages/plugin-host/src/plugin/audit.ts | 55 +- .../plugin-host/src/plugin/cli-metadata.ts | 12 + .../src/plugin/consent/approval-service.ts | 7 +- .../src/plugin/consent/approval-store.ts | 53 +- packages/plugin-host/src/plugin/doctor.ts | 21 +- .../src/plugin/incompatible-store.ts | 22 +- .../src/plugin/installation-recovery.ts | 196 +++++ .../src/plugin/installation-transaction.ts | 162 +--- .../src/plugin/local-registry-store.ts | 21 +- packages/plugin-host/src/plugin/lockfile.ts | 53 +- packages/plugin-host/src/plugin/migration.ts | 58 +- .../plugin/routes/plugin-registry-routes.ts | 35 +- packages/plugin-host/src/plugin/startup.ts | 7 + packages/plugin-host/src/plugin/trust.ts | 12 +- .../test/plugin/approval-store.test.ts | 4 +- .../plugin-host/test/plugin/doctor.test.ts | 15 +- .../test/plugin/incompatible-store.test.ts | 21 +- .../test/plugin/installation-recovery.test.ts | 48 ++ .../plugin/installation-transaction.test.ts | 2 +- .../plugin-host/test/plugin/migration.test.ts | 61 +- .../test/plugin/tool-invocation.test.ts | 2 +- .../product-runtime/schema/config.schema.json | 26 + packages/product-runtime/src/cli-commands.ts | 13 +- .../product-runtime/src/cli/data/merge.ts | 35 +- packages/product-runtime/src/cli/data/move.ts | 35 +- packages/product-runtime/src/cli/data/pack.ts | 11 +- .../product-runtime/src/cli/data/transfer.ts | 134 +++ packages/product-runtime/src/cli/server.ts | 14 +- packages/product-runtime/src/daemon-entry.ts | 6 +- packages/product-runtime/src/index.ts | 11 + .../product-runtime/src/server/runtime.ts | 27 +- .../test/cli/data-transfer.test.ts | 94 ++ .../test/config/domain.test.ts | 1 + .../test/daemon/config-migration.test.ts | 2 + .../fixtures/models-runtime-offline.ts | 18 +- .../server/plugin-approval-routes.test.ts | 37 +- .../test/session/invoke.test.ts | 2 +- .../test/session/migration.test.ts | 4 +- .../test/session/search-index.test.ts | 20 +- .../test/tool/bash-github-token.test.ts | 3 +- packages/sdk/js/src/gen/sdk.gen.ts | 5 + packages/sdk/js/src/gen/types.gen.ts | 26 + packages/sdk/openapi.json | 59 ++ packages/testing/src/preload.ts | 17 +- packages/workbench/src/stats/engine.ts | 8 +- .../workbench/test/cortex/manager.test.ts | 2 +- packages/workflows/src/agenda/store.ts | 260 +++--- .../workflows/src/blueprint/loop-store.ts | 292 +++---- packages/workflows/src/lattice/store.ts | 301 ++++--- packages/workflows/src/superplan/store.ts | 150 ++-- .../test/blueprint/storage-atomicity.test.ts | 27 + .../test/lattice/run-service.test.ts | 12 +- .../test/lattice/storage-atomicity.test.ts | 31 + packages/workflows/test/lattice/store.test.ts | 10 +- .../test/migration/lattice-v2-reset.test.ts | 4 +- script/dev.ts | 1 + script/release/shared/runtime-assets.ts | 2 + script/release/shared/runtime-contract.ts | 1 + 202 files changed, 8306 insertions(+), 3747 deletions(-) create mode 100644 docs/architecture/agent-storage.md create mode 100644 docs/decisions/implemented/architecture/2026-09-14-transactional-agent-authority.md create mode 100644 docs/migrations/transactional-agent-storage.md create mode 100644 packages/cli/src/cli/cmd/data/storage.ts create mode 100644 packages/harness/script/benchmark-storage.ts create mode 100644 packages/harness/script/build-sqlite.ts create mode 100644 packages/harness/src/session/staging.ts create mode 100644 packages/harness/src/storage/atomic-file.ts create mode 100644 packages/harness/src/storage/bootstrap.ts create mode 100644 packages/harness/src/storage/config.ts create mode 100644 packages/harness/src/storage/errors.ts create mode 100644 packages/harness/src/storage/legacy-import.ts create mode 100644 packages/harness/src/storage/maintenance.ts create mode 100644 packages/harness/src/storage/portable.ts create mode 100644 packages/harness/src/storage/postgres-driver.ts create mode 100644 packages/harness/src/storage/queue.ts create mode 100644 packages/harness/src/storage/recovery.ts create mode 100644 packages/harness/src/storage/sql-contract.ts create mode 100644 packages/harness/src/storage/sqlite-driver.ts create mode 100644 packages/harness/src/storage/sqlite-engine.ts create mode 100644 packages/harness/src/storage/sqlite-worker.ts create mode 100644 packages/harness/src/storage/transactional-store.ts create mode 100644 packages/harness/test/session/staging.test.ts create mode 100644 packages/harness/test/session/transaction-atomicity.test.ts create mode 100644 packages/harness/test/storage/bootstrap.test.ts create mode 100644 packages/harness/test/storage/context.test.ts create mode 100644 packages/harness/test/storage/crash-recovery.test.ts create mode 100644 packages/harness/test/storage/fixtures/README.md create mode 100644 packages/harness/test/storage/fixtures/v1.2.33.json create mode 100644 packages/harness/test/storage/fixtures/v2.4.4.json create mode 100644 packages/harness/test/storage/fixtures/v3.0.22.json create mode 100644 packages/harness/test/storage/legacy-import.test.ts create mode 100644 packages/harness/test/storage/portable.test.ts create mode 100644 packages/harness/test/storage/postgres-ownership.test.ts create mode 100644 packages/harness/test/storage/released-upgrade.test.ts create mode 100644 packages/harness/test/storage/transactional-store.test.ts create mode 100644 packages/harness/test/storage/verification.test.ts create mode 100644 packages/note/test/storage-atomicity.test.ts create mode 100644 packages/plugin-host/src/plugin/installation-recovery.ts create mode 100644 packages/plugin-host/test/plugin/installation-recovery.test.ts create mode 100644 packages/product-runtime/src/cli/data/transfer.ts create mode 100644 packages/product-runtime/test/cli/data-transfer.test.ts create mode 100644 packages/workflows/test/blueprint/storage-atomicity.test.ts create mode 100644 packages/workflows/test/lattice/storage-atomicity.test.ts diff --git a/.github/workflows/build-helpers.yml b/.github/workflows/build-helpers.yml index 6caa022bd..fb15455cf 100644 --- a/.github/workflows/build-helpers.yml +++ b/.github/workflows/build-helpers.yml @@ -14,6 +14,8 @@ on: - "packages/runtime-local/script/watcher/**" - "packages/runtime-local/test/file/fixtures/watcher-*" - "test/script/watcher-native.test.ts" + - "packages/harness/script/build-sqlite.ts" + - "packages/harness/src/storage/sqlite-engine.ts" push: branches: [dev, main] paths: @@ -25,6 +27,8 @@ on: - "packages/runtime-local/script/watcher/**" - "packages/runtime-local/test/file/fixtures/watcher-*" - "test/script/watcher-native.test.ts" + - "packages/harness/script/build-sqlite.ts" + - "packages/harness/src/storage/sqlite-engine.ts" permissions: contents: read @@ -133,3 +137,20 @@ jobs: name: sandbox-assets-windows-x64 path: packages/runtime-local/sandbox-assets if-no-files-found: error + + sqlite: + name: Patched macOS SQLite engine + runs-on: macos-latest + steps: + - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4 + with: + persist-credentials: false + - uses: oven-sh/setup-bun@0c5077e51419868618aeaa5fe8019c62421857d6 # v2 + with: + bun-version: "1.3.14" + - run: bun packages/harness/script/build-sqlite.ts + - uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4 + with: + name: sqlite-assets-darwin + path: packages/harness/.artifacts/sqlite/ + if-no-files-found: error diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index e4acd6d4a..ecac89475 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -72,6 +72,40 @@ jobs: bun script/pack-workspace.ts packages/product-runtime "${{ runner.temp }}/runtime-packages" bun script/runtime-composition-check.ts "${{ runner.temp }}/runtime-packages" + agent-storage: + name: Agent Storage (PostgreSQL ${{ matrix.postgres }}) + runs-on: ubuntu-latest + timeout-minutes: 15 + strategy: + fail-fast: false + matrix: + postgres: [16, 17, 18] + services: + postgres: + image: postgres:${{ matrix.postgres }} + env: + POSTGRES_PASSWORD: storage-ci-only + POSTGRES_DB: synergy_storage_test + ports: + - 5432:5432 + options: >- + --health-cmd "pg_isready -U postgres" + --health-interval 5s --health-timeout 5s --health-retries 12 + steps: + - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4 + with: + persist-credentials: false + - uses: oven-sh/setup-bun@0c5077e51419868618aeaa5fe8019c62421857d6 # v2 + with: + bun-version: "1.3.14" + - run: bun install --frozen-lockfile + - name: Verify both database implementations against the same contract + working-directory: packages/harness + run: bun test --timeout 30000 test/storage + env: + SYNERGY_REQUIRE_POSTGRES_TESTS: "1" + SYNERGY_TEST_POSTGRES_URL: postgres://postgres:storage-ci-only@127.0.0.1:5432/synergy_storage_test + oryn-validation: name: Oryn Configuration runs-on: ubuntu-latest diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index a71b074eb..69aa81b73 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -68,6 +68,12 @@ jobs: pattern: sandbox-assets-* merge-multiple: true + - name: Download patched SQLite engine + uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4 + with: + name: sqlite-assets-darwin + path: packages/harness/.artifacts/sqlite + - name: Validate product release environment run: bun run ./script/release/validate-product-environment.ts env: diff --git a/.synergy/skill/change-persistence/SKILL.md b/.synergy/skill/change-persistence/SKILL.md index 806ba08e4..e9926a516 100644 --- a/.synergy/skill/change-persistence/SKILL.md +++ b/.synergy/skill/change-persistence/SKILL.md @@ -13,13 +13,15 @@ description: Add or modify Synergy durable state, JSON storage keys, SQLite tabl ## Implement the Current Model -### File-backed JSON - -1. Build logical keys through `StoragePath`; use `Storage` for locks, atomic writes, reads, scans, and removal. -2. Keep independently updated or streamed records independently addressable. Do not rewrite a whole session or collection for one leaf update. -3. Update derived indexes and events in the same owner transaction/lifecycle as the canonical write. -4. Preserve the atomic-write transient-retry contract: `Storage` write+rename retries `EPERM`/`EACCES`/`EBUSY` (classified by `isRetryableIOError`) so Windows sharing violations do not fail persistence, permanent errors fail fast, and temp files are removed (with the same transient retry) on the failure path. Do not bypass `Storage` with a bare rename; extend `test/storage/storage-retry.test.ts` when changing write-path failure behavior. -5. Authoritative rollout evidence uses private, durable Storage writes and the bounded `RolloutArtifact` stream store. Keep progress independently committed, verify content hashes, and preserve partial observations. Do not replace its persistence failures with diagnostic warnings, empty data, or successful completion; propagate `RolloutRecordingError` so execution admission can stop. +### Authoritative Agent records + +1. Build logical keys through `StoragePath`; use an explicit `Storage.Handle`. Normal Agent record code must never read or write legacy JSON files. +2. Keep independently updated or streamed records independently addressable. Wrap the complete business mutation, indexes, receipts and outbox notifications in `Storage.transaction()`. +3. Nested writes join the caller's transaction. Defer cache and event effects until commit. Never run tools, network calls, plugin reloads, filesystem writes or buffer drains inside a retryable SQL transaction. +4. Treat commit uncertainty as an unresolved result; reconcile the operation receipt before retrying. Preserve storage, ownership and integrity errors instead of treating them as missing records. +5. Flush artifact bytes before publishing references. Stage unpublished large imports and register resumable post-deletion cleanup. Rollout evidence retains its separate allocation/evidence and projection/head transactions. +6. Physical writes retain the atomic-file transient-retry contract for Windows sharing violations. Extend the real-file retry tests when changing that helper. +7. Run the shared SQLite/PostgreSQL contract tests for engine changes; CI requires real PostgreSQL 16–18. macOS source development needs the verified SQLite engine from `bun packages/harness/script/build-sqlite.ts`. ### SQLite and other domain stores diff --git a/README.md b/README.md index 358727aaf..5f51ff3ab 100644 --- a/README.md +++ b/README.md @@ -259,3 +259,5 @@ Coding agents and LLM tools should begin with [llms.txt](llms.txt). Read [AGENTS Contributions, bug reports, and feature ideas are welcome. Read [CONTRIBUTING.md](CONTRIBUTING.md), follow the [Code of Conduct](CODE_OF_CONDUCT.md), and use the repository's [security reporting process](.github/SECURITY.md) for vulnerabilities rather than opening a public issue. Synergy is open source under the [MIT License](LICENSE). + +Agent records use transactional SQLite by default, with an explicit PostgreSQL option. Existing Home data upgrades through a resumable, backed-up migration. See [Agent storage](docs/architecture/agent-storage.md) and [storage operations](docs/reference/storage-and-paths.md). diff --git a/docs/README.md b/docs/README.md index 2f8072a1d..0953fbfec 100644 --- a/docs/README.md +++ b/docs/README.md @@ -35,6 +35,7 @@ Architecture documents define current invariants, ownership boundaries, and the - [Architecture overview](architecture/README.md) - [Runtime and Scope](architecture/runtime-and-scope.md) +- [Agent storage](architecture/agent-storage.md) - [Workspace and files](architecture/workspace-and-files.md) - [Sessions and messages](architecture/session-and-messages.md) - [LLM loop and compaction](architecture/llm-loop.md) diff --git a/docs/architecture/agent-storage.md b/docs/architecture/agent-storage.md new file mode 100644 index 000000000..f99025699 --- /dev/null +++ b/docs/architecture/agent-storage.md @@ -0,0 +1,53 @@ +# Agent Storage + +## Authority and ownership + +`Storage.Handle` binds one `TransactionalStore` namespace and one artifact directory. Logical keys do not resolve through the current project directory. The Runtime owns the Handle, runs migrations and recovery before admission, drains outstanding writes during shutdown, and closes only Handles it opened. An embedding caller can supply a Handle and retain responsibility for its lifetime. Scope and Session identify logical ownership; Workspace files and execution environments do not own Agent records. + +SQLite is the default backend. PostgreSQL is an explicit deployment choice using the same transaction, revision, pagination, receipt and outbox contract. One Runtime owns a namespace. PostgreSQL advisory ownership and a namespace owner identity fence stale writers; ordinary transactions are serialized inside that owner. Multiple sessions can run concurrently, but automatic Runtime failover and simultaneous replicas writing one namespace are not supported. + +## Records and transactions + +The SQL schema stores independently addressable records with keys, revisions and indexed kind, Scope, Session, Message and ordering columns. Session metadata, message info and individual parts remain separate records. Relational identity checks and domain schemas complement the generic storage contract. Unknown fields owned by unloaded packages survive persistence, upgrades and portable exports. + +`Storage.transaction()` is the business boundary. Nested domain writes join the caller's transaction; readers inside it see their own writes, while external readers see a committed snapshot. Derived indexes and durable notifications commit with their canonical mutation. A failed SQL statement poisons the transaction even if a caller catches its exception. Compare-and-swap uses expected revisions, and deletion leaves a revision tombstone so a delayed writer cannot treat an old revision as a new record. + +Commands that may be retried can supply an operation ID and request hash. The receipt and mutation commit together. Reusing the ID with identical input returns the committed result; different input conflicts. A lost commit response is explicitly uncertain and must be reconciled through the receipt. Database serialization failures can retry a transaction whose callback contains only database operations and deferred effects. Network requests, tool execution, plugin reloads and filesystem mutations stay outside retryable callbacks. + +Inbox message publication, delivery receipts and queue removal commit together for task, steer and context inputs. Invalid attachments retain a failed queue item for repair instead of deleting its payload. BlueprintLoop transitions commit their Note lifecycle projection and Session bindings together; user-edit plugin hooks remain outside that transaction. + +## Files and evidence + +Large content remains in artifact storage. Writers flush bytes before publishing a database reference; artifact references include byte bounds, completeness and hashes. A database transaction cannot atomically commit a filesystem rename or an external provider request. Session imports and forks therefore stage unpublished identities, retain their evidence, and publish all Session records and indexes in the final transaction. Startup removes interrupted unpublished jobs. Deletion commits canonical removal and a cleanup record before releasing Git references and physical files. + +Rollout's application journal is distinct from the database WAL. The first transaction allocates a journal sequence and records evidence; the second applies the projection and advances the committed head. Recovery projects committed evidence without repeating the tool or provider request. Historical missing sequences remain explicit gaps. Database rollback does not erase previously committed observations. + +Plugin installation has a durable recovery intent and a private snapshot of the affected registration, approval, configuration and directory promotion. SQL metadata changes commit together; reload runs afterward. Interrupted installations reconcile before plugin startup. Completed installation cleanup can resume without replaying the installation or its hooks. + +## Streaming and notifications + +Streaming part writes coalesce for 500 ms. Terminal writes wait for prior in-flight writes; drain boundaries include timer-triggered writes and propagate persistence failures. Buffered values and caches belong to the Storage Handle. Part persistence verifies the owning Session and Message and rejects a deleted part, preventing late writes from resurrecting removed data. + +State notifications enter a durable SQL outbox in the business transaction. Publication and cache changes happen after commit. An observer failure cannot roll back an already committed mutation. A new Runtime changes the frontend event epoch and reconciles outstanding notifications by requiring a fresh snapshot; it does not replay arbitrary subscribers that might perform external actions. Stream deltas remain provisional until their persistence boundary completes. + +## Engines and limits + +SQLite runs in a dedicated Bun subprocess, keeping synchronous SQL off the Control Plane event loop. It uses WAL, `synchronous=FULL`, separate read and write connections, bounded admission, IPC byte limits, operation deadlines and explicit child drainage. macOS packages include a checksum-verified SQLite engine; initialization rejects versions without the WAL reset fix. See the [SQLite WAL documentation](https://www.sqlite.org/wal.html) and [synchronous pragma](https://www.sqlite.org/pragma.html#pragma_synchronous). + +PostgreSQL keeps its advisory ownership connection in a separate single-connection pool; loss of that connection permanently fences the Handle. SQLSTATE-based serialization retries are bounded to three attempts. PostgreSQL uses Bun's native SQL driver, `SERIALIZABLE` writes, `REPEATABLE READ READ ONLY` snapshots, synchronous commit, connection limits and statement/lock deadlines. PostgreSQL 16, 17 and 18 run the same contract suite in CI. Isolation does not make external side effects transactional; see [transaction isolation](https://www.postgresql.org/docs/current/transaction-iso.html) and [advisory locks](https://www.postgresql.org/docs/current/explicit-locking.html). + +## Upgrade and movement + +Home ownership excludes the running legacy writer. Bootstrap backs up original bytes, hashes an independently readable inventory, imports records with resumable checkpoints, runs registered owner migrations, validates relationships and activates the new authority. Malformed historical Session data is preserved and quarantined with a persistent execution block. Permission, I/O, identity or backup-integrity failures stop activation. Once active, normal code has one SQL record path; reappearing legacy authority files cause startup to fail rather than silently selecting a dataset. + +Portable data contains records, revisions, command receipts and pending events with a checksum footer. Pack, merge and move use this logical representation and separately preserve artifact bytes and Git objects; they do not copy a live SQLite database or assume PostgreSQL data resides in the Home. A conflicting Session ID keeps the target aggregate intact, including its artifacts and snapshot references. Skipped source data and a transfer report remain available even when move removes the original Home. Session indexes are rebuilt in the import transaction. + +A target switch first saves a verified portable archive and a durable switch intent. Import is idempotent, verification precedes activation, and the intent blocks normal startup until both configuration and dataset identity agree. `data storage resume` completes an interrupted switch. Storage configuration cannot be hot-reloaded. Downgrade uses the immutable pre-upgrade backup in a separate Home; there is no reverse writer or live JSON mirror. + +See [storage and paths](../reference/storage-and-paths.md) for locations and [transactional storage migration](../migrations/transactional-agent-storage.md) for operational recovery. + +## Reproducible validation + +`bun packages/harness/script/benchmark-storage.ts` measures 1,000 transactions with two records each, a 1 KiB payload, 32 concurrent callers and a 100-record page read. A local macOS run with Bun 1.3.14 and PostgreSQL 16 in Docker measured SQLite at 1,186 transactions/s (queued p95 31.76 ms, page read 1.50 ms) and PostgreSQL at 134 transactions/s (queued p95 236.98 ms, page read 1.91 ms). The run shared the host with build/test processes. These are reproducible development measurements, not production capacity guarantees or a comparison against the old JSON writer. PostgreSQL throughput currently includes namespace serialization and ownership checks. + +Historical upgrade fixtures reconstruct the published v1.2.33, v2.4.4 and v3.0.22 writer formats with their exact source commits. See the [fixture provenance](../../packages/harness/test/storage/fixtures/README.md). Fault tests cover worker crashes, owner loss, rollback, stale revisions, ambiguous commit receipts, malformed legacy records, interrupted target activation, unpublished Session recovery and plugin installation recovery. diff --git a/docs/architecture/runtime-and-scope.md b/docs/architecture/runtime-and-scope.md index e6b92f637..aefc68d8a 100644 --- a/docs/architecture/runtime-and-scope.md +++ b/docs/architecture/runtime-and-scope.md @@ -1,5 +1,7 @@ # Runtime and Scope +The Runtime owns an explicit transactional Agent Storage Handle independently of Scope directories. It completes database upgrade and recovery before admission, then drains and closes owned storage during shutdown. See [Agent storage](agent-storage.md) for authority, ownership, engine and file-commit boundaries. + ## Runtime Model Synergy has one execution runtime and one writing owner per home in a process. Harness can run without an HTTP server; a server composition exposes that runtime to multiple clients and project contexts. The runtime is not bound to the launch directory: scoped operations select a `scopeID` or directory, and each session persists its own Scope and workspace binding. diff --git a/docs/decisions/implemented/architecture/2026-09-14-transactional-agent-authority.md b/docs/decisions/implemented/architecture/2026-09-14-transactional-agent-authority.md new file mode 100644 index 000000000..671936a7a --- /dev/null +++ b/docs/decisions/implemented/architecture/2026-09-14-transactional-agent-authority.md @@ -0,0 +1,33 @@ +# Decision Record: Transactional Agent Authority + +Status: implemented + +## Problem + +Individually atomic JSON file replacement cannot commit a Session mutation with its message, indexes and notifications. Concurrent read-modify-write operations can lose updates, interrupted operations can expose partial aggregates, and directory resolution makes Agent authority depend on a particular execution filesystem. Existing installations also contain historical owner schemas, Git snapshots, artifact bytes and data from optional packages that must survive an upgrade. + +## Decision + +The Runtime owns an explicit Storage Handle containing a transactional logical-record store and an independent artifact location. SQLite is the default implementation; PostgreSQL uses the same contract. One Runtime owns each namespace, with revision checks, idempotent command receipts, consistent read snapshots and a durable notification outbox. Runtime concurrency spans Sessions; this change does not introduce active-active Runtime replicas. + +Business transactions include their projections and notification intents. Cache updates and publication follow commit. Rollout retains its application evidence journal, including its separate evidence/allocation and projection/head commits. Files commit before their database references. Session import/fork stages unpublished identities, deletion records cleanup work, and plugin installation retains a recoverable intent for its database/configuration/directory boundary. + +Bootstrap seals an immutable original-byte backup and independently readable inventory before importing JSON. Import checkpoints, domain migration ledgers, unknown owner fields and quarantine records are retained. Activation leaves one SQL authority. Pack/merge/move export logical records and retain artifacts separately. Conflicting Session IDs preserve the target aggregate and retain the skipped source evidence. Target switching uses a durable intent and verified archive rather than two uncoordinated configuration writes. + +SQLite executes in a subprocess with bounded IPC and explicit shutdown. Bun 1.3.14 worker-thread tests exposed a persistence-await deadlock, so database isolation uses the same supported process mechanism as other runtime workers. macOS packages carry a checksum-pinned SQLite 3.51.3 engine; Linux and Windows validate their embedded SQLite version at initialization. The engine requirement follows the [SQLite WAL reset fix](https://www.sqlite.org/wal.html); durability follows the [synchronous pragma](https://www.sqlite.org/pragma.html#pragma_synchronous). PostgreSQL isolation and ownership follow its [transaction isolation](https://www.postgresql.org/docs/current/transaction-iso.html) and [explicit locking](https://www.postgresql.org/docs/current/explicit-locking.html) contracts. + +## Alternatives considered + +**Keep JSON and add more per-file locks.** This preserves inspectable files but cannot atomically commit cross-record invariants, support engine-independent snapshots or solve commit uncertainty. Locks also tie ownership to physical paths. + +**Require PostgreSQL for every installation.** PostgreSQL is a useful service deployment option, but requiring a separate database service for a desktop or local CLI is unnecessary. A shared contract with SQLite keeps both deployment forms first-class. + +**Mirror every SQL change back to JSON.** A permanent mirror introduces a second authority, synchronization failures and ambiguous downgrade semantics. Original backups and explicit portable exports provide recoverability without another live write path. + +**Wrap all execution in a database transaction.** Filesystem operations, plugin reloads and provider requests cannot be rolled back by SQL. Long transactions also monopolize the SQLite writer and can replay external effects during retries. Durable intents and narrowly defined commit boundaries preserve evidence and recovery instead. + +## Consequences + +The architecture can embed Agent storage without a project directory controlling record resolution. Transactions make canonical updates and their projections reviewable as one operation; SQLite and PostgreSQL share behavior and CI coverage. The cost is explicit Handle ownership, staged file operations, migration inventory, engine packaging and additional recovery paths. + +A PostgreSQL connection is not automatic high availability. Namespace ownership must be recovered deliberately after an unclean owner exit, and ambiguous external actions remain ambiguous. Portable transfer and immutable backups consume extra disk space. Downgrade requires a separate restored Home and does not include changes made after the historical snapshot. These boundaries are documented in [Agent storage](../../../architecture/agent-storage.md) and the [upgrade procedure](../../../migrations/transactional-agent-storage.md). diff --git a/docs/migrations/transactional-agent-storage.md b/docs/migrations/transactional-agent-storage.md new file mode 100644 index 000000000..ea43d8b27 --- /dev/null +++ b/docs/migrations/transactional-agent-storage.md @@ -0,0 +1,27 @@ +# Transactional Agent Storage Upgrade + +## Upgrade + +Stop the instance through its normal user-facing shutdown flow before running offline maintenance. The next Runtime startup, or `synergy data storage resume`, acquires exclusive Home ownership, backs up original data and configuration, imports legacy records, runs the installed domain migrations and activates SQL authority. Never run an old binary against an activated Home. + +The immutable backup is `data/storage/backups//`. Its `manifest.json` records the source identity, file count, bytes and inventory checksum. `inventory.ndjson` records every original path, content hash, size and symbolic-link target. Original data is under `data/`; `data/@home/config/` and `data/@home/plugin.lock` preserve Home-level inputs. Workspace symbolic links are preserved as links and never traversed. Authoritative record links are rejected. + +Before copying, bootstrap estimates space for the immutable backup, records and database overhead; insufficient free space stops the import. Corrupt global Scope or migration-ledger records block activation even on repeated resume. + +Repeated `resume` uses recorded checkpoints and the same sealed backup. It rejects changed source files, missing backup bytes and identity mismatches. A record with invalid historical JSON is preserved in the backup and assigned a persistent recovery issue. Affected Sessions cannot execute until their evidence is repaired and their block is resolved. Do not delete a recovery marker merely to bypass a failed upgrade. + +## Verify and troubleshoot + +`data storage status` reports the active backend, namespace, database/artifact identity, recovery records and outstanding notifications. `data storage verify` reads database integrity and record relationships without modifying data and returns a failing exit code for reported issues. Storage I/O, authorization, ownership and malformed engine/configuration errors remain fatal rather than becoming empty data. + +If an upgrade is interrupted, preserve the entire Home, correct the reported cause and run `resume`. If an old JSON writer reappears after activation, preserve both datasets and stop that writer before reconciling records. Never overwrite the SQL database with an old copy or delete the manifest to force a fresh installation. + +An interrupted target switch retains `data/storage/switch.json` and its checksummed transfer archive. Normal startup is blocked until `resume` verifies the target and completes activation. Restore missing connection credentials through the named environment variable; do not put credentials into a shared diagnostic report. + +## Downgrade + +There is no SQL-to-legacy live writer. Restore the pre-upgrade snapshot into a separate empty Home and use an appropriate old release there. Reconstruct `data/` from the backup, restore `@home/config/` to the Home's `config/`, and restore `@home/plugin.lock` to the Home root. Check inventory hashes before opening the old version. Keep the upgraded Home intact: post-upgrade changes are not part of the historical snapshot. + +## Transfer + +Use `data pack` for a portable logical backup, and `data merge` or `data move` to restore or combine data. Agent records, revisions and operation receipts move independently of the engine; Git objects and artifacts move with their references. A target Session ID wins as a whole aggregate. The retained source snapshot and transfer report let an operator recover skipped content even after `move --remove-original`. diff --git a/docs/reference/cli.md b/docs/reference/cli.md index 8612da463..ce56584b2 100644 --- a/docs/reference/cli.md +++ b/docs/reference/cli.md @@ -681,6 +681,14 @@ stop channels stop plugin runtime +## storage + +inspect, verify, recover, and move authoritative Agent storage + +| Option | Description | +| --- | --- | +| `--target` (string) | JSONC configuration file containing the target storage domain; credentials use an environment reference | + ## symbols search workspace symbols diff --git a/docs/reference/configuration.md b/docs/reference/configuration.md index 1fa56c60b..ad7e7fcc8 100644 --- a/docs/reference/configuration.md +++ b/docs/reference/configuration.md @@ -24,6 +24,7 @@ Generated from `packages/harness/src/config/domain.ts` and the domain-owned conf | `github` | `115-github.jsonc` | merge | | `runtime` | `120-runtime.jsonc` | merge | | `voice` | `125-voice.jsonc` | merge | +| `storage` | `130-storage.jsonc` | replace-domain | ## General @@ -205,3 +206,11 @@ File: `125-voice.jsonc` · Merge: merge | Key | Type | Description | | --- | --- | --- | | `voice` | VoiceConfig | | + +## Storage + +File: `130-storage.jsonc` · Merge: replace-domain + +| Key | Type | Description | +| --- | --- | --- | +| `storage` | StorageConfiguration.optional (optional) | | diff --git a/docs/reference/storage-and-paths.md b/docs/reference/storage-and-paths.md index 06fdbe3ec..e266db4d6 100644 --- a/docs/reference/storage-and-paths.md +++ b/docs/reference/storage-and-paths.md @@ -1,198 +1,81 @@ # Storage and Paths -Synergy keeps installation state under one root: - -```text -/.synergy/ -``` - -`SYNERGY_HOME` changes the parent home, not the `.synergy` suffix. For example, `SYNERGY_HOME=/tmp/example` produces `/tmp/example/.synergy/`. - -## Top-Level Layout - -| Path | Responsibility | -| --------- | ---------------------------------------------------------------------- | -| `bin/` | installed launchers and binaries | -| `config/` | global domain config, global agents/commands/skills, instruction files | -| `data/` | durable product data and auth stores | -| `log/` | normal process logs | -| `state/` | daemon, runtime, trace, and process state | -| `cache/` | disposable model/provider/marketplace and derived caches | -| `schema/` | installed JSON schemas | - -Cache version changes can clear `cache/` on startup. Treat cache as reproducible, not as a backup source. - -## JSON Storage - -Most durable product objects use file-based JSON storage rooted at `data/`. A logical storage key maps to nested directories plus a `.json` suffix. Writes take per-file locks and use a temporary file followed by atomic rename. The write+rename sequence retries transient sharing-violation errors (`EPERM`/`EACCES`/`EBUSY`, classified via `isRetryableIOError`) up to 4 attempts with 50–200 ms backoff, because Windows renames fail when antivirus, sync clients, or cross-process readers briefly hold a handle; permanent errors fail on the first attempt and the temp file is removed (with the same transient retry) before the original error propagates. Cross-process readers of these files read through `readFileWithRetry` for the same reason. Streaming message/part writes can use compact JSON; lower-frequency records remain indented. - -Major collections include: - -```text -data/projects/ -data/session_index/ -data/sessions_page_index/ -data/session_child_index/ -data/session_nav_v2/ -data/session_search_v1/// -data/session_search_dirty_v1/// -data/sessions/// -data/session_message_order_v1/// -data/channel/managed_ownership/ -data/embedding/models/ -data/channel/managed_ownership_reverse/ -data/channel/workspaces//workspace/ -data/channel/diagnostics/ -data/channel/providers/clarus/accounts/ -data/permissions/ -data/channel/response_cards///.json -data/channel/feishu/streaming_cards///.json -data/channel/feishu/thread_bindings///.json -data/permission-rules.json -data/notes// -data/agenda/items// -data/agenda/runs/// -data/blueprint_loops// -data/superplan/runs// -data/superplan/events/// -data/lattice/runs//.json -data/lattice/current//.json -data/lattice/events/// -data/holos/contacts/ -data/holos/mailbox/ -data/synergy_link/targets/ -data/stats/ -data/meta/rollout/ -``` - -Local embedding model assets are cached under `data/embedding/models/`; the location can be redirected with `embedding.local.cacheDir`. - -Channel-managed Project ownership uses a hashed forward record under `managed_ownership/` and a Scope-ID reverse index under `managed_ownership_reverse/`. Raw external account and Project IDs remain record values rather than path components. `workspaces//workspace/` is the deterministic, symlink-rejecting Project directory and an independent Git repository. - -Channel diagnostics store bounded, redacted, independently addressable records below `data/channel/diagnostics/accounts//records/`. Account-level NDJSON downloads first scan the bounded set of at most 10,000 retained record IDs, then read, validate, and encode one record per response pull instead of materializing record payloads. Obsolete pre-release per-account array files directly under `data/channel/diagnostics/` are left untouched and ignored. Clarus provider-private state is isolated below each hashed account root: `assignments/`, `assignment_session_index/`, `dedup/`, `outbox/results/`, and `outbox/extensions/`. Result and extension outboxes are durable-before-send recovery state; pending records recovered after an interrupted process become ambiguous rather than being retried blindly. - -The GitHub Channel keeps provider-private state below each hashed account root at `data/channel/providers/github/accounts//`: per-repository poll cursors and dedup state under `poll-state/`, and thread→checkout records under `workspaces/index/`. Actual repository checkouts live under the configured account `workspaceDir`, one random-hash directory per issue/PR thread; expired checkouts are removed after the account's `workspaceTtlHours` (default 24h) and recreated on the next thread trigger, while session history is preserved. - -Channel response-card registrations live under `data/channel/response_cards/`. Each provider-neutral record is keyed by channel type, account, and response-card tool-part ID. A pending record is written before the provider side effect and becomes active only after the provider returns a sent message ID. Both states retain the original chat, requester, session, card contract, and a 14-day expiry. An active registration additionally binds the provider message ID used to validate callbacks. A surviving pending record blocks resend until its expiry. Expired and malformed registrations are pruned at global runtime startup. - -Active Feishu/Lark streaming cards live under `data/channel/feishu/streaming_cards/`, keyed independently by account, session, and card ID. Each record is written after CardKit creates the card but before the card is exposed in chat, and contains the identifiers and start time needed to terminate an orphan after process restart. The message ID is added after provider delivery succeeds. A successful terminal close removes only that card's record. Account reconnect scans all records for its account, closes each orphan with a terminal recovery mutation, and preserves records whose provider call fails transiently so a later reconnect can retry without a newer card overwriting them. - -Feishu/Lark thread bindings live under `data/channel/feishu/thread_bindings/`, keyed by account, chat, and thread ID. Each record durably maps a Feishu `thread_id` to the endpoint `scopeKey` it belongs to, so `group_thread` sessions resume the same conversation when a later message arrives in the same thread. - -Synergy Link targets live under `data/synergy_link/targets/`, one JSON record per stable target ID. They contain routing identifiers, local visibility policy, authorization state, and last observed host capabilities. Holos account secrets remain in `data/auth/` and are never copied into target records. - -The standalone Synergy Link host keeps its own per-instance state root at `SYNERGY_LINK_HOME` (default `~/.synergy-link/`), containing `state.json`, `migrations.json`, `owner.json`, `control.sock`, and `logs/`. It is a separate root from the Synergy installation and must never be shared between Link instances; the control socket, state writes, and Holos credential handling all assume one live service per root. See [Qizhi Synergy Link operations](../operations/qizhi-synergy-link.md) for the per-instance namespace boundary. - -Inside a session, `info.json`, `summary.json`, `summary_cursor.json`, `todo.json`, `dag.json`, `lightloop_terminal.json`, `inbox/`, `messages/`, and `history/` are separate records. `lightloop_terminal.json` preserves a plugin-owned Light Loop result and its `lightloop.after` delivery acknowledgement after the interactive workflow is cleared. The summary cursor is derived, discardable state used to extend cumulative diff ranges from bounded loop messages; missing cursors rebuild from session history, and rollback or unrollback invalidates them. Message info and each part are independently addressable, which supports streaming writes and narrow reads. - -The session index, paged-session index, child-session index, navigation index, message-order index, and session-search index are derived but operationally important. `session_message_order_v1` contains sortable per-message markers and a readiness/count record for bounded newest-first reads; missing or interrupted state rebuilds from canonical message info. `session_search_v1` caches per-session searchable text excerpts (with `session_search_dirty_v1` dirty markers); both are discardable — deleting them only forces a lazy rebuild on the next `session_search` query. Do not hand-move one session directory without its Scope/session indexes; use export/import, data, migration, or repair workflows. - -Lattice stores every v2 run by immutable run ID. A session's `lattice/current` record selects the run shown as current without overwriting older terminal runs; it is a repairable index over canonical Run records. Per-run event files are idempotent, best-effort audit records, not an event-sourced reconstruction of the Run. Run, Step, Blueprint binding, and BlueprintLoop records remain the recovery facts. - -## Rollout Artifacts - -The rollout artifact store uses `rollout/` beneath its owning session, or `data/operations///rollout/` for sessionless operations. `artifacts//info.json` commits the readable byte/chunk count and completeness state; individually addressed chunk descriptors reference owner-local, SHA-256-addressed binary blobs. Payloads are streamed in bounded chunks and verified on read. Interrupted streams retain their committed prefix. Under the same rollout owner, `runs//info.json` stores run state, `runs//calls/.json` stores logical calls, and `runs//attempts//.json` stores actual provider attempts with ordered indices and body references. Private records use owner-only permissions and durable atomic writes; they are separate from public product assets and telemetry retention. - -Session rollout `continuation-recovery/.json` records the recovery intent created by the unanswered-continuation migration before repairing terminal state. Startup uses it to wake the existing root without adding inbox or transcript records. The intent survives failed wake attempts and is removed when the root is answered, cancelled, failed, or superseded; ordinary interrupted runs without this intent are not automatically resumed. - -Externalized files in `data/tool-output/` have no age-based expiration. Creating a new tool-output file does not delete older observations. - -## Library Database - -Library uses: - -```text -data/library.db -``` - -It is a Bun SQLite database with WAL behavior and optional `sqlite-vec` tables for Memory and Experience embeddings. It is installation-global while records retain Scope/session metadata. SQLite sidecar files can exist while the server is active; copy the database only through a consistent backup workflow. - -## Credentials - -Credential files live under `data/auth/`, including: - -- `api-key.json` and `provider-auth.json` -- `holos-accounts.json` -- `mcp.json` -- integration-specific auth stores - -`holos-accounts.json` is the canonical multi-account Holos credential store. Its active account supplies the identity used by both the Holos runtime and the standalone Synergy Link transport. `api-key.json` is legacy migration input for Holos credentials and is not the steady-state source after migration. - -Synergy and Synergy Link serialize updates to `holos-accounts.json` with the shared `data/auth/.locks/` protocol. Writers use the `holos-accounts:write` lock key and atomic rename so lock-free readers never observe a partial account store. - -Holos account storage is permissioned to the local user. Treat the entire auth directory as sensitive. Diagnostics and SmartAllow use redaction/metadata paths rather than exposing raw secrets. - -Plugin-scoped credentials live separately at `data/plugin//auth.json`. Plugin approvals, audit history, runtime health, and the local registry use `data/plugin-approvals.json`, `data/plugin-audit.json`, `data/plugin-runtime-state.json`, and `data/registry/plugins.json`; `plugin.lock` at the installation root binds installed specs to resolved artifacts and integrity. Treat plugin auth and signing material under `keys/` as sensitive even when the plugin itself is trusted. - -## Browser, Worktrees, and Artifacts - -| Path | Content | -| ------------------------- | ----------------------------------------------------------------------- | -| `data/browser/sessions/` | canonical Browser session/page metadata | -| `data/browser/profiles/` | persistent browser profiles and storage state | -| `data/browser/uploads/` | owner-scoped upload staging | -| `data/browser/downloads/` | browser downloads grouped by Scope | -| `data/browser/chromium/` | managed Chromium assets | -| `data/worktree/` | Synergy-managed worktree metadata/resources | -| `data/snapshot/` | registered legacy file snapshot repositories pending migration | -| `data/snapshot-v2/` | Scope object stores, historical roots, owners, and maintenance journals | -| `data/tool-output/` | large tool outputs externalized from message records | -| `data/assets/` | product/plugin assets | -| `data/media/` | generated or captured media, including Browser screenshots | - -Archiving or deleting a session disposes its live Browser runtime, but persisted Browser state follows its own lifecycle and migration rules. - -## Daemon and Observability State - -Managed service state is under: - -```text -state/daemon/manifest.json -state/daemon/runtime-lock.json -state/daemon/logs/server.log -``` - -The lock records PID, server/daemon mode, command, and working directory. A stale or conflicting lock is inspected rather than blindly overwritten. - -Platform service definitions live in platform-owned locations: - -- macOS: `~/Library/LaunchAgents/dev.synergy.server.plist` -- Linux: `~/.config/systemd/user/synergy.service` -- Windows: Task Scheduler plus launch scripts in `state/daemon/` - -Structured observability traces live under `state/observability/traces/`. Performance and diagnostics state may add adjacent state/data records. `synergy status --verbose`, `synergy logs`, and `synergy diagnostics` are the supported inspection entry points. - -Indexed observability telemetry lives in `state/observability/observability.sqlite`. The database uses WAL plus incremental auto-vacuum. Retention and size maintenance evict the globally oldest eligible historical telemetry in bounded batches while preserving running spans and open issues. Existing observability databases and the previous `state/observability/performance/performance.sqlite` store are upgraded through central, transactional observability migrations; runtime request paths do not perform schema upgrades or full-database vacuum operations. - -Plugin installation stages artifacts and holds its transaction lock under `state/plugin-install/`. Cached plugin packages, extracted archives, marketplace records, models, provider catalogs, and downloaded runtime dependencies live under `cache/`; they may be recreated and must not be treated as approval or credential records. Live provider model snapshots are versioned, atomically written, and keyed by opaque identity hashes rather than credentials or raw account identifiers. LSP process bookkeeping is kept in `state/lsp-pids.json`. - -## Project-Local `.synergy` - -A repository's `.synergy/` is project configuration and extension source, not the installation data root: - -```text -/.synergy/synergy.d/ -/.synergy/agent/ -/.synergy/command/ -/.synergy/skill/ +Synergy keeps installation state under `/.synergy/`. `SYNERGY_HOME` selects the parent home, not the `.synergy` suffix. For example, `SYNERGY_HOME=/tmp/example` produces `/tmp/example/.synergy/`. + +## Physical layout + +| Path | Responsibility | +| -------------------------------------------------- | ------------------------------------------------------------------------------------------ | +| `bin/` | Installed launchers and binaries | +| `config/` | Global domain configuration, agents, commands, skills and instructions | +| `data/storage/` | Agent database bootstrap identity, SQLite database, migration backups and transfer records | +| `data/auth/` | Provider, Holos, MCP and other account credentials | +| `data/library.db` | Library's independently owned SQLite knowledge database | +| `data/plugin//auth.json` | Plugin credentials | +| `data/plugin-install-artifacts/` | Private installation recovery snapshots and directory backups | +| `data/browser/profiles/` | Persistent browser profiles and browser storage state | +| `data/browser/uploads/`, `data/browser/downloads/` | Browser file staging and downloads | +| `data/snapshot-v2//store.git/` | Shared Git snapshot objects and retained references | +| `data/snapshot/` | Historical Git snapshot repositories until explicit migration/cleanup | +| `data/sessions///` | Session-owned binary evidence; metadata is in the Agent database | +| `data/operations///` | Binary evidence for sessionless operations | +| `data/channel/workspaces/` | Channel-managed Project checkouts | +| `data/embedding/models/` | Local embedding models; overridable with `embedding.local.cacheDir` | +| `data/tool-output/` | Externalized tool output without age-based expiry | +| `state/` | Process ownership, daemon and transient runtime state | +| `cache/` | Rebuildable caches, including snapshot working indexes | +| `log/` | Process and diagnostic logs | +| `schema/` | Installed JSON schemas | + +Library, credentials, project files, browser profiles and observability remain separate stores with their own lifecycle. Do not copy an open Library/observability SQLite file without its owning backup protocol. Cache may be cleared on upgrade and is not a backup source. Treat auth, plugin recovery snapshots, logs, signing keys and exported Home archives as private data. + +## Agent database + +`Storage` reads and writes logical keys inside an explicit `Storage.Handle`. Keys no longer map to `.json` files. SQLite defaults to `data/storage/agent.sqlite`; PostgreSQL uses a configured namespace. `data/storage/manifest.json` binds the backend target, database identity and local artifact identity, preventing a missing or unrelated database from being silently accepted as an empty installation. + +| Logical collection | Owner and contents | +| --------------------------------------------------------------------------------------------------- | ---------------------------------------------------------------------------------------------------- | +| `projects` | Scope metadata | +| `sessions//` | Session info, messages, parts, Inbox, Todo, DAG, history, summary and Rollout | +| `operations//` | Sessionless Rollout metadata | +| `session_index`, `endpoint_session`, `sessions_page_index`, `session_child_index`, `session_nav_v2` | Session lookup and navigation projections | +| `session_message_order_v1`, `session_search_v1`, `session_search_dirty_v1` | Message ordering and rebuildable search projections | +| `permissions`, `permission-rules` | Persistent permission state | +| `agenda`, `blueprint_loops`, `superplan`, `lattice`, `notes` | Workflow and Note domain records | +| `channel` | Ownership indexes, response cards, thread bindings, provider dedup, outboxes and bounded diagnostics | +| `holos`, `synergy_link` | Routing, contacts, mailbox and target metadata; credentials stay separate | +| `browser` | Browser Session/page metadata; profiles stay on the filesystem | +| `plugin-lock`, `plugin-approvals`, `plugin-incompatible`, `registry` | Plugin installation, grants and registry records | +| `plugin-audit`, `plugin-runtime-state`, `plugin-install-intents` | Plugin audit, health and installation recovery | +| `snapshot-v2` | Snapshot repository metadata, ownership and cleanup records | +| `meta` | Versioned domain migration ledgers and recovery indexes | +| `storage_recovery`, `storage_staging`, `storage_transfer` | Quarantine, unpublished imports and transfer reports | + +Domain extensions keep their data and migration history when unloaded. Scope/session IDs and external account IDs are record data, not an implicit filesystem authority. Read [Agent storage](../architecture/agent-storage.md) for transaction, concurrency, notification and file-commit contracts. + +## Configuration and commands + +Global `config/synergy.d/130-storage.jsonc` selects storage. Omission selects SQLite. PostgreSQL configuration uses an environment-variable name rather than an inline password: + +```json +{ + "storage": { + "backend": "postgres", + "namespace": "my-agent-data", + "connectionEnv": "SYNERGY_DATABASE_URL", + "maxConnections": 8 + } +} ``` -Project worktrees may also be managed beneath a project-local Synergy area. Permission policy treats the active worktree as the write/execute boundary and the original checkout as readable but protected from autonomous modification. - -## Relocation and Backup - -Stop the server before raw filesystem backup or relocation. For supported selective movement, use `synergy data pack`, `merge`, `move`, and `set-home`. Use session export/import for portable session artifacts. - -Never include `data/auth/` in a public diagnostics bundle, issue attachment, or repository commit. - -Rollout runs also own `tools/` and `processes/` metadata through `RolloutLedger`. Tool inputs, original results, returned observations, and channel-framed process streams use the same private artifact store as model evidence. A process record can remain active after an explicitly backgrounded tool returns; exports must preserve its partial stream boundary rather than infer completion from the tool result. +Use `synergy data storage status` to inspect the active dataset, `verify` to check integrity and relationships, `resume` to finish interrupted upgrades or switches, and `migrate --target ` to change backend, namespace or SQLite location. These commands acquire the appropriate read-only or exclusive maintenance Handle. They do not stop the running Runtime. -## File snapshot persistence +`data pack`, `data merge` and `data move` use checksummed logical records and separately copy physical artifacts. The portable record stream is `data/agent-records.ndjson`; the target creates a new local bootstrap identity when restoring a Home archive. Target storage configuration is retained instead of importing another machine's connection settings. Conflicting Sessions are skipped as whole aggregates; source evidence and reports remain in `data/storage/transfers/`. -The server's Git executable initializes snapshot stores as SHA-1 repositories, including on Git 2.25.1. Initialization explicitly selects SHA-1 through `GIT_DEFAULT_HASH`, verifies the actual object hash format before recording success, and reports the Git exit code and stderr on initialization failure. A non-SHA-1 repository is rejected without conversion or deletion. The browser's operating system does not determine snapshot Git compatibility. +## Credentials and independent hosts -`data/snapshot-v2//store.git` holds self-contained Git objects and all historical retention refs. `repository.json` records object format; `owners/.json` selects `legacy`, `shared`, or the permanent deletion tombstone. `migrations/.json` and `deletions/.json` are durable recovery state. Scope `leases.json`, the root `leases.json`, and `.locks/` coordinate processes and are regenerated rather than merged into archives. `format.json` marks the installed layout version. +Holos account storage at `data/auth/holos-accounts.json` is the canonical multi-account credential store. Synergy and the standalone Link host serialize updates with `data/auth/.locks/`, using the `holos-accounts:write` lock key and atomic file replacement. `api-key.json` is historical migration input, not the steady-state Holos source. -`cache/snapshot-index////index` is rebuildable working state. It can be removed independently of historical objects. The workspace hash uses its canonical filesystem path. Legacy owners resolve only to `data/snapshot//` until explicit migration switches their ownership; unknown and reclaimed repositories remain intact and are reported separately. Legacy directories with no owner record and no session record (including the `__reclaimed__` scope) are reclaimable through `synergy data snapshots clean` and `POST /global/storage/snapshot/clean`: both default to a dry run, refuse a scope that fails its integrity check, and never touch the shared store or directories with owners. The HTTP endpoint rejects an empty `scopeID`; scope-targeted requests return 409 on busy or failed integrity checks, while batch requests (no `scopeID`) return `{ results, failures }` and keep the completed work of scopes processed before a failure. The `20260907-snapshot-release-orphan-owners` migration releases legacy owner records that the shared-store migration created for directories without session records, so such orphans reach `clean` on upgraded installations. Owned legacy repositories move through `synergy data snapshots migrate` or `POST /global/storage/snapshot/migrate`, and the shared store packs through `compact` or `POST /global/storage/snapshot/compact`; both HTTP endpoints default to a dry run and return 409 when storage is busy or a scope fails its integrity check. Clean before running `migrate` — a registered repository is migration's responsibility and is no longer a clean candidate. +The standalone Synergy Link host owns `SYNERGY_LINK_HOME` (default `~/.synergy-link/`), including its own `state.json`, `migrations.json`, `owner.json`, control socket and logs. It is independent of Agent database ownership and must not share one host state root across live instances. See [Link operations](../operations/qizhi-synergy-link.md). -JSON session export does not contain file objects. Complete `data pack`, `move`, and `merge` preserve file history through the snapshot domain's object/ref transfer. Owner-backend or maintenance-record conflicts abort that data transfer so the source remains available for resolution. These commands acquire offline ownership and never stop a running server. Migration changes are not backward-readable by an older runtime after shared snapshots have been captured. +Physical private writes use flushed temporary files, atomic rename and directory sync where supported. Transient Windows sharing violations retry up to four attempts. Database durability uses the database engine's commit protocol; it is not controlled by a per-record JSON formatting or durability option. diff --git a/packages/agent-integrations/src/acp/README.md b/packages/agent-integrations/src/acp/README.md index cea0fc3f0..fb1eda49b 100644 --- a/packages/agent-integrations/src/acp/README.md +++ b/packages/agent-integrations/src/acp/README.md @@ -4,7 +4,7 @@ ## Ownership -- `cli/acp.ts` runs migrations, starts a local Synergy HTTP server, creates a generated SDK client for the requested working directory, and binds ACP stdio transport. +- `cli/acp.ts` opens the product-injected Runtime Handle, including storage ownership, migrations, recovery and a local Synergy HTTP server, creates a generated SDK client for the requested working directory, and binds ACP stdio transport. - `agent.ts` implements protocol initialization, modes/models, session prompting, cancellation, permission bridging, history replay, and event-to-ACP updates. - `session.ts` maps ACP session state to durable Synergy session IDs and retains the ACP working directory, MCP descriptors, selected model, and mode. - `types.ts` defines the internal ACP configuration and session state. diff --git a/packages/agent-integrations/src/acp/cli/acp.ts b/packages/agent-integrations/src/acp/cli/acp.ts index 4070556a7..34872e791 100644 --- a/packages/agent-integrations/src/acp/cli/acp.ts +++ b/packages/agent-integrations/src/acp/cli/acp.ts @@ -2,66 +2,74 @@ import { Log } from "@ericsanchezok/synergy-harness/util/log" import { cmd } from "@ericsanchezok/synergy-cli/cli/cmd/cmd" import { AgentSideConnection, ndJsonStream } from "@agentclientprotocol/sdk" import { ACP } from "../agent" -import { Server } from "@ericsanchezok/synergy-server/server/server" +import type { RuntimeHandle } from "@ericsanchezok/synergy-harness/lifecycle" import { createSynergyClient } from "@ericsanchezok/synergy-sdk" import { withNetworkOptions, resolveNetworkOptions } from "@ericsanchezok/synergy-cli/cli/network" const log = Log.create({ service: "acp-command" }) -export const AcpCommand = cmd({ - command: "acp", - describe: "start ACP (Agent Client Protocol) server", - builder: (yargs) => { - return withNetworkOptions(yargs).option("cwd", { - describe: "working directory", - type: "string", - default: process.cwd(), - }) - }, - handler: async (args) => { - const opts = await resolveNetworkOptions(args, { output: "silent" }) - const server = Server.listen(opts) +export function createAcpCommand( + openRuntime: (options: { + mode: "oneshot" + network: { hostname: string; port: number } + }) => Promise }>, +) { + return cmd({ + command: "acp", + describe: "start ACP (Agent Client Protocol) server", + builder: (yargs) => { + return withNetworkOptions(yargs).option("cwd", { + describe: "working directory", + type: "string", + default: process.cwd(), + }) + }, + handler: async (args) => { + const opts = await resolveNetworkOptions(args) + await using runtime = await openRuntime({ mode: "oneshot", network: opts }) + const server = runtime.server - const sdk = createSynergyClient({ - baseUrl: `http://${server.hostname}:${server.port}`, - directory: args.cwd, - }) + const sdk = createSynergyClient({ + baseUrl: `http://${server.hostname}:${server.port}`, + directory: args.cwd, + }) - const input = new WritableStream({ - write(chunk) { - return new Promise((resolve, reject) => { - process.stdout.write(chunk, (err) => { - if (err) { - reject(err) - } else { - resolve() - } + const input = new WritableStream({ + write(chunk) { + return new Promise((resolve, reject) => { + process.stdout.write(chunk, (err) => { + if (err) { + reject(err) + } else { + resolve() + } + }) }) - }) - }, - }) - const output = new ReadableStream({ - start(controller) { - process.stdin.on("data", (chunk: Buffer) => { - controller.enqueue(new Uint8Array(chunk)) - }) - process.stdin.on("end", () => controller.close()) - process.stdin.on("error", (err) => controller.error(err)) - }, - }) + }, + }) + const output = new ReadableStream({ + start(controller) { + process.stdin.on("data", (chunk: Buffer) => { + controller.enqueue(new Uint8Array(chunk)) + }) + process.stdin.on("end", () => controller.close()) + process.stdin.on("error", (err) => controller.error(err)) + }, + }) - const stream = ndJsonStream(input, output) - const agent = await ACP.init({ sdk }) + const stream = ndJsonStream(input, output) + const agent = await ACP.init({ sdk }) - new AgentSideConnection((conn) => { - return agent.create(conn, { sdk }) - }, stream) + new AgentSideConnection((conn) => { + return agent.create(conn, { sdk }) + }, stream) - log.info("setup connection") - process.stdin.resume() - await new Promise((resolve, reject) => { - process.stdin.on("end", resolve) - process.stdin.on("error", reject) - }) - }, -}) + log.info("setup connection") + process.stdin.resume() + await new Promise((resolve, reject) => { + process.stdin.on("end", resolve) + process.stdin.on("error", reject) + }) + }, + }) +} diff --git a/packages/agent-integrations/test/synergy-link/target-store.test.ts b/packages/agent-integrations/test/synergy-link/target-store.test.ts index e2e75168b..e665d76d4 100644 --- a/packages/agent-integrations/test/synergy-link/target-store.test.ts +++ b/packages/agent-integrations/test/synergy-link/target-store.test.ts @@ -275,14 +275,14 @@ describe("Synergy Link target relink", () => { const continueWrite = Promise.withResolvers() const write = Storage.write let paused = false - const writeSpy = spyOn(Storage, "write").mockImplementation(async (key, content, options) => { + const writeSpy = spyOn(Storage, "write").mockImplementation(async (key, content) => { const candidate = content as { id?: string; linkID?: string } if (!paused && candidate.id === target.id && candidate.linkID === "link_new") { paused = true writeStarted.resolve() await continueWrite.promise } - return await write(key, content, options) + return await write(key, content) }) try { @@ -319,14 +319,14 @@ describe("Synergy Link target relink", () => { const continueWrite = Promise.withResolvers() const write = Storage.write let paused = false - const writeSpy = spyOn(Storage, "write").mockImplementation(async (key, content, options) => { + const writeSpy = spyOn(Storage, "write").mockImplementation(async (key, content) => { const candidate = content as { id?: string; linkID?: string } if (!paused && candidate.id === target.id && candidate.linkID === "link_new") { paused = true writeStarted.resolve() await continueWrite.promise } - return await write(key, content, options) + return await write(key, content) }) try { diff --git a/packages/browser-runtime/src/migration.ts b/packages/browser-runtime/src/migration.ts index 889c2a5c6..682f9f08a 100644 --- a/packages/browser-runtime/src/migration.ts +++ b/packages/browser-runtime/src/migration.ts @@ -1,3 +1,4 @@ +import { Storage } from "@ericsanchezok/synergy-harness/storage/storage" import fs from "fs/promises" import path from "path" import { Global } from "@ericsanchezok/synergy-harness/global" @@ -36,39 +37,16 @@ export namespace BrowserMigration { [key: string]: unknown } - function legacyStateFilePath(owner: BrowserOwner.Info): string { - const base = path.join(Global.Path.data, "browser", "sessions", legacyComponent(owner.scopeID, "scope")) - if (owner.mode === "scope") return path.join(base, "scope.json") + function legacyStateKey(owner: BrowserOwner.Info): string[] { + const base = ["browser", "sessions", legacyComponent(owner.scopeID, "scope")] + if (owner.mode === "scope") return [...base, "scope"] BrowserOwner.assertValid(owner) - return path.join(base, "session", `${legacyComponent(owner.sessionID!, "session")}.json`) + return [...base, "session", legacyComponent(owner.sessionID!, "session")] } - async function exists(filepath: string): Promise { - try { - await fs.access(filepath) - return true - } catch (error) { - if ((error as NodeJS.ErrnoException).code === "ENOENT") return false - throw error - } - } - - async function readState(filepath: string): Promise { - let text: string - try { - const info = await fs.lstat(filepath) - if (!info.isFile() || info.isSymbolicLink() || info.size > 64 * 1024 * 1024) return null - text = await Bun.file(filepath).text() - } catch (error) { - if ((error as NodeJS.ErrnoException).code === "ENOENT") return null - throw error - } - try { - const parsed = JSON.parse(text) - return parsed && typeof parsed === "object" ? (parsed as StoredState) : null - } catch { - return null - } + async function readState(key: string[]): Promise { + const [value] = await Storage.readMany([key]) + return value } function isPage(value: unknown): value is { id: string; url: string; title: string; lastActiveAt?: number | null } { @@ -226,14 +204,17 @@ export namespace BrowserMigration { } } - async function migrateFile(owner: BrowserOwner.Info, filepath: string): Promise { - const state = await readState(filepath) + async function migrateRecord(owner: BrowserOwner.Info, key: string[]): Promise { + const state = await readState(key) if (!state) return { ownerKey: BrowserOwner.key(owner), changed: false, version: BrowserStorage.CURRENT_VERSION } const next = migrateState(state) - const changed = JSON.stringify(state) !== JSON.stringify(next) - const target = BrowserStorage.pathForOwner(owner) - if (changed || filepath !== target) await BrowserStorage.save(owner, next) - if (filepath !== target) await fs.rm(filepath, { force: true }) + const target = BrowserStorage.keyForOwner(owner) + const moved = JSON.stringify(key) !== JSON.stringify(target) + const changed = moved || JSON.stringify(state) !== JSON.stringify(next) + await Storage.transaction(async () => { + if (changed) await BrowserStorage.save(owner, next) + if (moved) await Storage.remove(key) + }) await removeRetiredProfilePath(state.storageStatePath) await removeRetiredProfilePath(state.profileDir) return { ownerKey: BrowserOwner.key(owner), changed, version: BrowserStorage.CURRENT_VERSION } @@ -260,54 +241,26 @@ export namespace BrowserMigration { await fs.rm(realTarget, { recursive: true, force: true }) } - async function collectStateFiles(): Promise<{ owner: BrowserOwner.Info; filepath: string }[]> { - const sessionsRoot = path.join(Global.Path.data, "browser", "sessions") - const entries: { owner: BrowserOwner.Info; filepath: string }[] = [] - const scopes = await directoryEntries(sessionsRoot) - for (const scopeID of scopes) { - const scopeDir = path.join(sessionsRoot, scopeID) - const scopeInfo = await fs.lstat(scopeDir) - if (!scopeInfo.isDirectory() || scopeInfo.isSymbolicLink()) continue - const scopeFile = path.join(scopeDir, "scope.json") - if (await exists(scopeFile)) - entries.push({ owner: { mode: "scope", scopeID, directory: "" }, filepath: scopeFile }) - const sessionDir = path.join(scopeDir, "session") - let files: string[] = [] - try { - const sessionInfo = await fs.lstat(sessionDir) - if (sessionInfo.isDirectory() && !sessionInfo.isSymbolicLink()) files = await directoryEntries(sessionDir) - } catch (error) { - if ((error as NodeJS.ErrnoException).code !== "ENOENT") throw error - } - for (const filename of files) { - if (!filename.endsWith(".json")) continue - const filepath = path.join(sessionDir, filename) - const fileInfo = await fs.lstat(filepath) - if (!fileInfo.isFile() || fileInfo.isSymbolicLink() || fileInfo.size > 64 * 1024 * 1024) continue - entries.push({ - owner: { mode: "session", scopeID, directory: "", sessionID: filename.slice(0, -5) }, - filepath, - }) - } - } - return entries - } - export async function run(owner: BrowserOwner.Info): Promise { - const current = BrowserStorage.pathForOwner(owner) - if (await exists(current)) return migrateFile(owner, current) - return migrateFile(owner, legacyStateFilePath(owner)) + const current = BrowserStorage.keyForOwner(owner) + if (await readState(current)) return migrateRecord(owner, current) + return migrateRecord(owner, legacyStateKey(owner)) } export async function runAll(progress?: (current: number, total: number) => void): Promise { - const files = await collectStateFiles() + const keys = await Storage.list(["browser", "sessions"]) let current = 0 - for (const entry of files) { - await migrateFile(entry.owner, entry.filepath) - progress?.(++current, files.length) + for (const key of keys) { + const scopeID = key[2] + if (!scopeID) throw new Error("Historical Browser state has no owner") + const owner: BrowserOwner.Info = + key[3] === "scope" + ? { mode: "scope", scopeID, directory: "" } + : { mode: "session", scopeID, sessionID: key[4], directory: "" } + await migrateRecord(owner, key) + progress?.(++current, keys.length) } - await removeLegacySessionsRoot(path.join(Global.Path.data, "browser", "sessions")) - if (files.length === 0) progress?.(0, 0) + if (!keys.length) progress?.(0, 0) } } @@ -318,36 +271,6 @@ function legacyComponent(value: string, label: string): string { return value } -async function directoryEntries(directory: string): Promise { - try { - return await fs.readdir(directory) - } catch (error) { - if ((error as NodeJS.ErrnoException).code === "ENOENT") return [] - throw error - } -} - -async function removeLegacySessionsRoot(sessionsRoot: string): Promise { - let info: Awaited> - try { - info = await fs.lstat(sessionsRoot) - } catch (error) { - if ((error as NodeJS.ErrnoException).code === "ENOENT") return - throw error - } - if (!info.isDirectory() || info.isSymbolicLink()) { - throw new Error("Legacy Browser sessions root is unsafe.") - } - const [realSessionsRoot, realBrowserRoot] = await Promise.all([ - fs.realpath(sessionsRoot), - fs.realpath(path.join(Global.Path.data, "browser")), - ]) - if (!realSessionsRoot.startsWith(`${realBrowserRoot}${path.sep}`)) { - throw new Error("Legacy Browser sessions root escaped Browser storage.") - } - await fs.rm(realSessionsRoot, { recursive: true, force: true }) -} - export const migrations: Migration[] = [ { id: "20260710-browser-suspended-session-v4", diff --git a/packages/browser-runtime/src/storage.ts b/packages/browser-runtime/src/storage.ts index ad5ec3a8d..c22d90f4f 100644 --- a/packages/browser-runtime/src/storage.ts +++ b/packages/browser-runtime/src/storage.ts @@ -1,3 +1,4 @@ +import { Storage } from "@ericsanchezok/synergy-harness/storage/storage" import path from "path" import fs from "fs/promises" import z from "zod" @@ -70,9 +71,9 @@ export namespace BrowserStorage { export type StoredAnnotation = z.infer export type SessionState = Omit, "version"> & { version?: number } - function stateFilePath(owner: BrowserOwner.Info): string { + export function keyForOwner(owner: BrowserOwner.Info): string[] { BrowserOwner.assertValid(owner) - return path.join(Global.Path.data, "browser", "sessions-v4", `${BrowserOwner.storageID(owner)}.json`) + return ["browser", "sessions-v4", BrowserOwner.storageID(owner)] } export function profileDir(owner: BrowserOwner.Info): string { @@ -102,18 +103,11 @@ export namespace BrowserStorage { } } - /** Read state. Returns null if no state file or on any read error. */ export async function load(owner: BrowserOwner.Info): Promise { - const fp = stateFilePath(owner) - try { - await assertSecureDirectory(path.dirname(fp), path.join(Global.Path.data, "browser")) - const stat = await fs.lstat(fp) - if (!stat.isFile() || stat.isSymbolicLink() || stat.size > 64 * 1024 * 1024) return null - const state = StoredSessionSchema.parse(JSON.parse(await fs.readFile(fp, "utf8"))) - return { ...state, page: state.page ? { ...state.page, url: sanitizeUrl(state.page.url) } : null } - } catch { - return null - } + const [raw] = await Storage.readMany([keyForOwner(owner)]) + if (raw === undefined) return null + const state = StoredSessionSchema.parse(raw) + return { ...state, page: state.page ? { ...state.page, url: sanitizeUrl(state.page.url) } : null } } /** Persist session state. Creates parent dirs if needed. */ @@ -131,44 +125,15 @@ export namespace BrowserStorage { : "empty", page: state.page ? { ...state.page, url: sanitizeUrl(state.page.url) } : null, }) - const fp = stateFilePath(owner) - await ensureSecureDirectory(path.dirname(fp), path.join(Global.Path.data, "browser")) - const temporary = `${fp}.${crypto.randomUUID()}.tmp` - let failure: unknown - try { - await fs.writeFile(temporary, JSON.stringify(sanitized, null, 2), { flag: "wx", mode: 0o600 }) - await replaceFileAtomically(temporary, fp) - } catch (error) { - failure = error - } - try { - await fs.rm(temporary, { force: true }) - } catch (cleanupError) { - if (failure) throw new AggregateError([failure, cleanupError], "Browser state save and cleanup both failed.") - throw cleanupError - } - if (failure) throw failure + await Storage.write(keyForOwner(owner), sanitized) } - /** Remove session state. */ export async function remove(owner: BrowserOwner.Info): Promise { - const fp = stateFilePath(owner) - try { - await assertSecureDirectory(path.dirname(fp), path.join(Global.Path.data, "browser")) - await fs.unlink(fp) - } catch (error) { - if ((error as NodeJS.ErrnoException).code !== "ENOENT") throw error - } - } - - /** Get storage path for an owner. */ - export function pathForOwner(owner: BrowserOwner.Info): string { - return stateFilePath(owner) + await Storage.remove(keyForOwner(owner)) } export async function ensureOwnerDirs(owner: BrowserOwner.Info): Promise { const browserRoot = path.join(Global.Path.data, "browser") - await ensureSecureDirectory(path.dirname(stateFilePath(owner)), browserRoot) await ensureSecureDirectory(profileDir(owner), path.join(browserRoot, "profiles")) await ensureSecureDirectory(uploadsDir(owner), path.join(browserRoot, "uploads")) await ensureSecureDirectory(downloadsDir(owner), path.join(browserRoot, "downloads")) diff --git a/packages/browser-runtime/test/migration.test.ts b/packages/browser-runtime/test/migration.test.ts index ee096b39f..76ce26b42 100644 --- a/packages/browser-runtime/test/migration.test.ts +++ b/packages/browser-runtime/test/migration.test.ts @@ -1,9 +1,7 @@ +import { Storage } from "@ericsanchezok/synergy-harness/storage/storage" import { describe, expect, test } from "bun:test" -import fs from "fs/promises" -import path from "path" import { BrowserMigration } from "../src/migration.js" import { BrowserStorage } from "../src/storage.js" -import { Global } from "@ericsanchezok/synergy-harness/global" import type { BrowserOwner } from "../src/owner.js" describe("BrowserMigration", () => { @@ -14,48 +12,38 @@ describe("BrowserMigration", () => { directory: process.cwd(), sessionID: "session-a", } - const legacyStatePath = path.join( - Global.Path.data, - "browser", - "sessions", - owner.scopeID, - "session", - `${owner.sessionID}.json`, - ) - const statePath = BrowserStorage.pathForOwner(owner) - await fs.mkdir(path.dirname(legacyStatePath), { recursive: true }) - await Bun.write( - legacyStatePath, - JSON.stringify( + const legacyStatePath = ["browser", "sessions", owner.scopeID, "session", owner.sessionID!] + const statePath = BrowserStorage.keyForOwner(owner) + await Storage.write(legacyStatePath, { + tabs: [ { - tabs: [ - { - id: "page-1", - url: "https://example.com/path?q=1#hash", - title: "Example", - order: 0, - }, - ], - activeTabID: "missing-page", - timestamp: 1, - annotations: [ - { - id: "ann-1", - pageID: "page-1", - tabURL: "https://example.com/path?q=1#hash", - comment: "keep this", - resolved: false, - createdAt: 1, - }, - ], + id: "page-1", + url: "https://example.com/path?q=1#hash", + title: "Example", + order: 0, }, - null, - 2, - ), - ) + ], + activeTabID: "missing-page", + timestamp: 1, + annotations: [ + { + id: "ann-1", + pageID: "page-1", + tabURL: "https://example.com/path?q=1#hash", + comment: "keep this", + resolved: false, + createdAt: 1, + }, + ], + }) const result = await BrowserMigration.run(owner) - const upgraded = JSON.parse(await Bun.file(statePath).text()) + const upgraded = await Storage.read<{ + version: number + status: string + page: { id: string; [key: string]: unknown } + [key: string]: unknown + }>(statePath) expect(result.changed).toBe(true) expect(result.version).toBe(BrowserStorage.CURRENT_VERSION) @@ -89,9 +77,9 @@ describe("BrowserMigration", () => { scroll: { x: 0, y: 0 }, formState: [], }) - expect(await Bun.file(legacyStatePath).exists()).toBe(false) + expect(await Storage.readMany([legacyStatePath])).toEqual([undefined]) - await fs.rm(statePath, { force: true }) + await Storage.remove(statePath) }) test("preserves a recoverable failed descriptor with a structured error", async () => { @@ -101,21 +89,22 @@ describe("BrowserMigration", () => { directory: process.cwd(), sessionID: "session-failed", } - const statePath = BrowserStorage.pathForOwner(owner) - await fs.mkdir(path.dirname(statePath), { recursive: true }) - await Bun.write( - statePath, - JSON.stringify({ - version: 4, - status: "failed", - page: { id: "page-failed", url: "https://example.com/", title: "Example", lastActiveAt: 1 }, - timestamp: 1, - error: "Host restore failed", - }), - ) + const statePath = BrowserStorage.keyForOwner(owner) + await Storage.write(statePath, { + version: 4, + status: "failed", + page: { id: "page-failed", url: "https://example.com/", title: "Example", lastActiveAt: 1 }, + timestamp: 1, + error: "Host restore failed", + }) await BrowserMigration.run(owner) - const upgraded = JSON.parse(await Bun.file(statePath).text()) + const upgraded = await Storage.read<{ + version: number + status: string + page: { id: string; [key: string]: unknown } + [key: string]: unknown + }>(statePath) expect(upgraded.status).toBe("failed") expect(upgraded.page.id).toBe("page-failed") expect(upgraded.error).toEqual({ diff --git a/packages/browser-runtime/test/session-lazy-restore.test.ts b/packages/browser-runtime/test/session-lazy-restore.test.ts index 9ca1d3866..d287e4d98 100644 --- a/packages/browser-runtime/test/session-lazy-restore.test.ts +++ b/packages/browser-runtime/test/session-lazy-restore.test.ts @@ -1,5 +1,4 @@ import { afterEach, describe, expect, test } from "bun:test" -import fs from "fs/promises" import { BrowserOwner } from "../src/owner" import { BrowserSessionImpl } from "../src/session" import { BrowserStorage } from "../src/storage" @@ -17,7 +16,7 @@ const owner: BrowserOwner.Info = { } afterEach(async () => { - await fs.rm(BrowserStorage.pathForOwner(owner), { force: true }) + await BrowserStorage.remove(owner) BrowserEvent.remove(owner) }) diff --git a/packages/browser-runtime/test/storage-security.test.ts b/packages/browser-runtime/test/storage-security.test.ts index f33487c72..e16c11ad0 100644 --- a/packages/browser-runtime/test/storage-security.test.ts +++ b/packages/browser-runtime/test/storage-security.test.ts @@ -1,3 +1,4 @@ +import { Storage } from "@ericsanchezok/synergy-harness/storage/storage" import { afterEach, describe, expect, test } from "bun:test" import fs from "node:fs/promises" import os from "node:os" @@ -6,11 +7,11 @@ import { BrowserExport } from "../src/export.js" import { BrowserOwner } from "../src/owner.js" import { BrowserStorage } from "../src/storage.js" -const created = new Set() +const created = new Set() const createdOwners: BrowserOwner.Info[] = [] afterEach(async () => { - await Promise.all(Array.from(created, (filepath) => fs.rm(filepath, { force: true }))) + await Promise.all(Array.from(created, (filepath) => Storage.remove(filepath))) await Promise.all( createdOwners .splice(0) @@ -24,16 +25,16 @@ afterEach(async () => { }) describe("Browser storage owner isolation", () => { - test("hashes ambiguous and traversal-shaped owner fields into distinct contained paths", () => { + test("hashes ambiguous and traversal-shaped owner fields into distinct logical records", () => { const first = owner("scope:a", "b/c") const second = owner("scope", "a:b_c") const traversal = owner("../../outside", "../session") - const paths = [first, second, traversal].map(BrowserStorage.pathForOwner) + const paths = [first, second, traversal].map(BrowserStorage.keyForOwner) - expect(new Set(paths).size).toBe(3) + expect(new Set(paths.map((key) => JSON.stringify(key))).size).toBe(3) for (const filepath of paths) { - expect(path.basename(filepath)).toMatch(/^[a-f0-9]{64}\.json$/) - expect(path.basename(path.dirname(filepath))).toBe("sessions-v4") + expect(filepath[2]).toMatch(/^[a-f0-9]{64}$/) + expect(filepath.slice(0, 2)).toEqual(["browser", "sessions-v4"]) } expect(BrowserOwner.key(first)).not.toBe(BrowserOwner.key(second)) }) @@ -41,22 +42,18 @@ describe("Browser storage owner isolation", () => { test("rejects unknown or oversized persisted state instead of reviving it", async () => { const targetOwner = owner("storage-schema", "invalid-state") createdOwners.push(targetOwner) - const filepath = BrowserStorage.pathForOwner(targetOwner) + const filepath = BrowserStorage.keyForOwner(targetOwner) created.add(filepath) await BrowserStorage.ensureOwnerDirs(targetOwner) - await fs.writeFile( - filepath, - JSON.stringify({ - version: BrowserStorage.CURRENT_VERSION, - status: "suspended", - page: { id: "page", url: "https://example.com", title: "x".repeat(20_001) }, - timestamp: Date.now(), - unexpected: true, - }), - { mode: 0o600 }, - ) + await Storage.write(filepath, { + version: BrowserStorage.CURRENT_VERSION, + status: "suspended", + page: { id: "page", url: "https://example.com", title: "x".repeat(20_001) }, + timestamp: Date.now(), + unexpected: true, + }) - expect(await BrowserStorage.load(targetOwner)).toBeNull() + await expect(BrowserStorage.load(targetOwner)).rejects.toThrow() }) }) diff --git a/packages/cli/src/cli/cmd/data/index.ts b/packages/cli/src/cli/cmd/data/index.ts index 8dbd518d4..673deab2f 100644 --- a/packages/cli/src/cli/cmd/data/index.ts +++ b/packages/cli/src/cli/cmd/data/index.ts @@ -1,3 +1,4 @@ +import { DataStorageCommand } from "./storage" import type { CommandModule } from "yargs" import { DataSnapshotsCommand } from "./snapshots" import { cmd } from "../cmd" @@ -10,6 +11,7 @@ export function createDataCommand(commands: CommandModule[] = []) { describe: "manage synergy data location and storage", builder: (yargs) => yargs + .command(DataStorageCommand) .command(DataSnapshotsCommand) .command(DataPathCommand) .command(DataSetHomeCommand) diff --git a/packages/cli/src/cli/cmd/data/shared.ts b/packages/cli/src/cli/cmd/data/shared.ts index 35ca836ab..8daa2d445 100644 --- a/packages/cli/src/cli/cmd/data/shared.ts +++ b/packages/cli/src/cli/cmd/data/shared.ts @@ -4,7 +4,6 @@ import path from "path" import os from "os" import { UI } from "../../../util/ui" import { Global } from "@ericsanchezok/synergy-harness/global" -import { StoragePath } from "@ericsanchezok/synergy-harness/storage/path" export interface Category { key: string @@ -162,22 +161,13 @@ export async function isDirEmpty(dir: string): Promise { } export function archiveExclusions(directory: string): string[] { - if (directory === "data") return ["snapshot", "snapshot-v2"] + if (directory === "data") return ["snapshot", "snapshot-v2", "storage", "agent-records.ndjson"] + if (directory === "config") return [path.join("synergy.d", "130-storage.jsonc")] if (directory === "cache") return ["snapshot-index"] if (directory === "state") return [path.join("daemon", "runtime-lock.json")] return [] } -/** - * Invalidates the target home's rollout recovery ledger after owner trees - * were copied into it. copyDirSkipExisting retains an existing ledger, but - * imported owners can hold journals the target ledger never listed, so the - * next startup must fall back to the exhaustive recovery scan. - */ -export async function invalidatePendingRolloutLedger(dataDir: string) { - await fs.rm(path.join(dataDir, ...StoragePath.rolloutRecoveryPending()) + ".json", { force: true }) -} - export interface CopyProgress { copied: number skipped: number @@ -192,7 +182,7 @@ export async function copyDirSkipExisting( onProgress?: (progress: CopyProgress) => void, rootSrc?: string, totalFiles?: number, - exclude: string[] = [], + exclude: string[] | ((relative: string) => boolean) = [], ): Promise<{ copied: number; skipped: number }> { if (!rootSrc) { rootSrc = src @@ -212,7 +202,8 @@ export async function copyDirSkipExisting( for (const entry of entries) { const srcPath = path.join(currentSrc, entry.name) const dstPath = path.join(currentDst, entry.name) - if (exclude.includes(path.relative(src, srcPath))) continue + const relative = path.relative(src, srcPath) + if (typeof exclude === "function" ? exclude(relative) : exclude.includes(relative)) continue if (entry.isDirectory()) { await walk(srcPath, dstPath) @@ -244,7 +235,7 @@ export async function copyDirSkipExisting( acc.skipped++ } else { const linkTarget = await fs.readlink(srcPath) - await fs.symlink(linkTarget, dstPath).catch(() => {}) + await fs.symlink(linkTarget, dstPath) acc.copied++ } if (onProgress && totalFiles) { diff --git a/packages/cli/src/cli/cmd/data/snapshots.ts b/packages/cli/src/cli/cmd/data/snapshots.ts index cc48577bf..052074944 100644 --- a/packages/cli/src/cli/cmd/data/snapshots.ts +++ b/packages/cli/src/cli/cmd/data/snapshots.ts @@ -1,3 +1,5 @@ +import { Storage } from "@ericsanchezok/synergy-harness/storage/storage" +import { StorageMaintenance } from "@ericsanchezok/synergy-harness/storage/maintenance" import type { Argv } from "yargs" import { cmd } from "../cmd" import { SnapshotMaintenance } from "@ericsanchezok/synergy-harness/session/snapshot-maintenance" @@ -15,8 +17,10 @@ interface Input { export async function executeSnapshots(input: Input) { let lock: Awaited> | undefined + let maintenance: Awaited> | undefined try { - if (input.apply) lock = await ServerProcessLock.acquire() + if (!Storage.available()) maintenance = await StorageMaintenance.open({ readonly: !input.apply }) + else if (input.apply) lock = await ServerProcessLock.acquire() if (input.action === "inspect") return { ok: true, results: await SnapshotMaintenance.inspect(input.scope) } if (input.action === "clean") { // Skip registerLegacy: it would claim unowned legacy directories right @@ -64,6 +68,7 @@ export async function executeSnapshots(input: Input) { }, } } finally { + await maintenance?.close() await lock?.release() } } diff --git a/packages/cli/src/cli/cmd/data/storage.ts b/packages/cli/src/cli/cmd/data/storage.ts new file mode 100644 index 000000000..6231023b2 --- /dev/null +++ b/packages/cli/src/cli/cmd/data/storage.ts @@ -0,0 +1,92 @@ +import { cmd } from "../cmd" +import { Global } from "@ericsanchezok/synergy-harness/global" +import { StorageMaintenance } from "@ericsanchezok/synergy-harness/storage/maintenance" +import { StorageBootstrap } from "@ericsanchezok/synergy-harness/storage/bootstrap" +import { parseStorageConfiguration } from "@ericsanchezok/synergy-harness/storage/config" + +export const DataStorageCommand = cmd({ + command: "storage", + describe: "inspect, verify, recover, and move authoritative Agent storage", + builder: (yargs) => + yargs + .command( + "status", + "show the active dataset and outstanding recovery records", + () => {}, + async () => { + const manifest = await StorageBootstrap.status(Global.Path.root) + if (manifest?.phase !== "active") { + console.log( + JSON.stringify( + { phase: manifest?.phase ?? "uninitialized", backend: manifest?.backend, backupID: manifest?.backupID }, + null, + 2, + ), + ) + return + } + await using handle = await StorageMaintenance.open({ readonly: true }) + const recovery = await handle.store.list(["storage_recovery"]) + console.log( + JSON.stringify( + { + backend: handle.manifest.backend, + namespace: handle.manifest.namespace, + phase: handle.manifest.phase, + storeID: handle.manifest.storeID, + artifactStoreID: handle.manifest.artifactStoreID, + recoveryRecords: recovery.length, + pendingEvents: await handle.store.pendingEventCount(), + }, + null, + 2, + ), + ) + }, + ) + .command( + "verify", + "verify database integrity and record relationships without changing data", + () => {}, + async () => { + await using handle = await StorageMaintenance.open({ readonly: true }) + const report = await handle.store.verify() + console.log(JSON.stringify(report, null, 2)) + if (report.issues.length) process.exitCode = 1 + }, + ) + .command( + "resume", + "resume an interrupted upgrade or target switch after acquiring exclusive ownership", + () => {}, + async () => { + await using handle = await StorageMaintenance.open({ recover: true }) + console.log( + JSON.stringify( + { backend: handle.manifest.backend, phase: handle.manifest.phase, status: "ready" }, + null, + 2, + ), + ) + }, + ) + .command( + "migrate", + "copy authoritative data and atomically activate a verified storage target", + (yargs) => + yargs.option("target", { + type: "string", + demandOption: true, + describe: + "JSONC configuration file containing the target storage domain; credentials use an environment reference", + }), + async (args) => { + const configuration = parseStorageConfiguration(await Bun.file(args.target).text()) + await using handle = await StorageMaintenance.open() + await StorageBootstrap.migrateTarget({ root: Global.Path.root, store: handle.store, configuration }) + console.log("Storage target migrated and verified. The next Runtime will use the new target.") + }, + ) + .demandCommand(), + async handler() {}, +}) diff --git a/packages/cli/src/cli/commands.ts b/packages/cli/src/cli/commands.ts index b0675f228..672754408 100644 --- a/packages/cli/src/cli/commands.ts +++ b/packages/cli/src/cli/commands.ts @@ -4,6 +4,7 @@ import type { openLocalRuntime } from "@ericsanchezok/synergy-runtime-local" export interface CommandEntry { command: string | string[] describe: string + storage?: "maintenance" load(): Promise } @@ -24,6 +25,7 @@ export function coreCommands( }, { command: "agent", + storage: "maintenance", describe: "manage agents", load: async () => (await import("./cmd/agent")).AgentCommand as unknown as CommandModule, }, @@ -39,36 +41,43 @@ export function coreCommands( }, { command: "models [provider]", + storage: "maintenance", describe: "list all available models", load: async () => (await import("./cmd/models")).ModelsCommand as unknown as CommandModule, }, { command: "export [sessionID]", + storage: "maintenance", describe: "export a session transcript or self-contained rollout ZIP", load: async () => (await import("./cmd/export")).ExportCommand as unknown as CommandModule, }, { command: "import ", + storage: "maintenance", describe: "import a session transcript or rollout ZIP", load: async () => (await import("./cmd/import")).ImportCommand as unknown as CommandModule, }, { command: "session", + storage: "maintenance", describe: "manage sessions", load: async () => (await import("./cmd/session")).SessionCommand as unknown as CommandModule, }, { command: "config", + storage: "maintenance", describe: "manage synergy configuration", load: async () => (await import("./cmd/config")).ConfigCommand as unknown as CommandModule, }, { command: "doctor", + storage: "maintenance", describe: "diagnose synergy sandbox and environment", load: async () => (await import("./cmd/doctor")).DoctorCommand as unknown as CommandModule, }, { command: "diagnostics", + storage: "maintenance", describe: "create a local diagnostics package", load: async () => (await import("./cmd/diagnostics")).DiagnosticsCommand as unknown as CommandModule, }, @@ -80,6 +89,7 @@ export function coreCommands( }, { command: "migration", + storage: "maintenance", describe: "manage schema and data migrations", load: async () => (await import("./cmd/migration")).MigrationCommand as unknown as CommandModule, }, diff --git a/packages/cli/src/cli/network.ts b/packages/cli/src/cli/network.ts index da539e0c8..961661a6b 100644 --- a/packages/cli/src/cli/network.ts +++ b/packages/cli/src/cli/network.ts @@ -1,7 +1,20 @@ -import type { RunOptions } from "@ericsanchezok/synergy-harness/migration/types" +import { Global } from "@ericsanchezok/synergy-harness/global" +import { Storage } from "@ericsanchezok/synergy-harness/storage/storage" +import { StorageBootstrap } from "@ericsanchezok/synergy-harness/storage/bootstrap" +import { StorageMaintenance } from "@ericsanchezok/synergy-harness/storage/maintenance" +import { ensureMigrations } from "@ericsanchezok/synergy-harness/migration" import type { Argv, InferredOptionTypes } from "yargs" import { Config } from "@ericsanchezok/synergy-harness/config/config" -import { ensureMigrations } from "@ericsanchezok/synergy-harness/migration" + +export async function loadNetworkConfig() { + if (Storage.available()) { + await ensureMigrations({ output: "silent" }) + } else if ((await StorageBootstrap.status(Global.Path.root))?.phase !== "active") { + await using maintenance = await StorageMaintenance.open() + } + Config.global.reset() + return Config.global() +} interface ResolveNetworkInput { argv?: string[] @@ -55,15 +68,11 @@ export async function isServerReachable(url: string): Promise { } } -export async function resolveNetworkOptions( - args: NetworkOptions, - migrationOptions: Pick = { output: "interactive" }, -) { - await ensureMigrations(migrationOptions) +export async function resolveNetworkOptions(args: NetworkOptions) { Config.global.reset() return resolveNetworkArgv({ argv: process.argv, - config: await Config.global(), + config: await loadNetworkConfig(), defaults: { hostname: args.hostname, port: args.port, @@ -85,10 +94,9 @@ export async function resolveNetworkArgv( ) { const argv = input.argv ?? process.argv if (!input.config) { - await ensureMigrations({ output: "interactive" }) Config.global.reset() } - const config = input.config ?? (await Config.global()) + const config = input.config ?? (await loadNetworkConfig()) const portExplicitlySet = argv.includes("--port") const hostnameExplicitlySet = argv.includes("--hostname") const mdnsExplicitlySet = argv.includes("--mdns") diff --git a/packages/cli/src/daemon/spec.ts b/packages/cli/src/daemon/spec.ts index 5a48d6f00..f72d0a975 100644 --- a/packages/cli/src/daemon/spec.ts +++ b/packages/cli/src/daemon/spec.ts @@ -1,7 +1,6 @@ -import { resolveNetworkArgv } from "../cli/network" +import { resolveNetworkArgv, loadNetworkConfig } from "../cli/network" import { Config } from "@ericsanchezok/synergy-harness/config/config" import { DEFAULT_SERVER_PORT } from "@ericsanchezok/synergy-harness/util/server-defaults" -import { ensureMigrations } from "@ericsanchezok/synergy-harness/migration" import type { DaemonService } from "./service" import { DaemonCommand } from "./command" @@ -25,9 +24,8 @@ export namespace DaemonSpec { } export async function resolveNetwork(input?: { argv?: string[]; config?: GlobalConfig }): Promise { - await ensureMigrations({ output: "interactive" }) if (!input?.config) Config.global.reset() - const config = input?.config ?? (await Config.global()) + const config = input?.config ?? (await loadNetworkConfig()) const network = await resolveNetworkArgv({ argv: input?.argv, config, @@ -49,9 +47,8 @@ export namespace DaemonSpec { } export async function resolve(input?: { argv?: string[]; config?: GlobalConfig }): Promise { - await ensureMigrations({ output: "interactive" }) if (!input?.config) Config.global.reset() - const config = input?.config ?? (await Config.global()) + const config = input?.config ?? (await loadNetworkConfig()) const network = await resolveNetwork({ argv: input?.argv, config }) const command = DaemonCommand.resolve({ hostname: network.hostname, port: network.port }) diff --git a/packages/cli/src/index.ts b/packages/cli/src/index.ts index b461727ec..caa680fd3 100644 --- a/packages/cli/src/index.ts +++ b/packages/cli/src/index.ts @@ -1,8 +1,18 @@ export async function runCoreWorker(): Promise { const worker = process.argv.find((arg) => - ["__observability-worker-runner", "__agent-turn-runner", "__policy-worker-runner"].includes(arg), + [ + "__storage-worker-runner", + "__observability-worker-runner", + "__agent-turn-runner", + "__policy-worker-runner", + ].includes(arg), ) if (!worker) return false + if (worker === "__storage-worker-runner") { + await import("@ericsanchezok/synergy-harness/storage/sqlite-worker") + await new Promise(() => {}) + return true + } const { Global } = await import("@ericsanchezok/synergy-harness/global") await Global.initialize({ cache: false }) if (worker === "__observability-worker-runner") diff --git a/packages/cli/src/main.ts b/packages/cli/src/main.ts index c6c1c31d3..dbbde8f9d 100644 --- a/packages/cli/src/main.ts +++ b/packages/cli/src/main.ts @@ -55,6 +55,9 @@ export async function runCli(options: CliOptions): Promise { async function runCliImplementation(options: CliOptions): Promise { const argv = options.argv ?? hideBin(process.argv) + let storage: + | Awaited> + | undefined const builtinCommands = [...coreCommands(options.runtimeFactory, options.dataCommands), ...(options.commands ?? [])] const onRejection = (error: unknown) => { process.exitCode = 1 @@ -87,6 +90,16 @@ async function runCliImplementation(options: CliOptions): Promise { }) .middleware(async (opts) => { if (informational) return + const entry = builtinCommands.find((entry) => + (Array.isArray(entry.command) ? entry.command : [entry.command]).some( + (name) => name.split(" ")[0] === selectedCommand, + ), + ) + if (entry?.storage === "maintenance" && !storage) { + const { StorageMaintenance } = await import("@ericsanchezok/synergy-harness/storage/maintenance") + const inspect = selectedCommand === "migration" && (argv.includes("status") || argv.includes("--dry-run")) + storage = await StorageMaintenance.open({ readonly: inspect, migrate: selectedCommand !== "migration" }) + } if (!["send", "server"].includes(selectedCommand ?? "server")) await Global.initialize({ cache: false }) let configLogLevel: string | undefined try { @@ -258,6 +271,7 @@ async function runCliImplementation(options: CliOptions): Promise { process.exitCode = findRecordingError(e) ? 5 : process.exitCode || 2 } else process.exitCode = 1 } finally { + await storage?.close() if (!isLongRunningCommand()) await flushCliOutput() process.removeListener("unhandledRejection", onRejection) process.removeListener("uncaughtException", onException) diff --git a/packages/connections/src/channel/diagnostics.ts b/packages/connections/src/channel/diagnostics.ts index f78bbfdca..192c43632 100644 --- a/packages/connections/src/channel/diagnostics.ts +++ b/packages/connections/src/channel/diagnostics.ts @@ -218,9 +218,7 @@ export async function recording(channelType: string, accountId: string, input: D try { using _ = await Lock.write(`channel-diagnostics:${account}`) await pruneBeforeWrite(account, normalized.timestamp) - await Storage.write(StoragePath.channelDiagnosticsRecord(account, recordID(normalized.timestamp)), normalized, { - compact: true, - }) + await Storage.write(StoragePath.channelDiagnosticsRecord(account, recordID(normalized.timestamp)), normalized) } catch (err) { log.error("failed to persist diagnostic record", { error: err }) } diff --git a/packages/connections/src/channel/managed-project-ownership.ts b/packages/connections/src/channel/managed-project-ownership.ts index 61bd3f73a..e96b16d88 100644 --- a/packages/connections/src/channel/managed-project-ownership.ts +++ b/packages/connections/src/channel/managed-project-ownership.ts @@ -101,13 +101,19 @@ async function validateDirectoryChain(target: string, create: boolean): Promise< } async function readForward(hash: string): Promise { - const raw = await Storage.read(StoragePath.channelManagedOwnership(hash)).catch(() => undefined) + const raw = await Storage.read(StoragePath.channelManagedOwnership(hash)).catch((error) => { + if (error instanceof Storage.NotFoundError) return undefined + throw error + }) if (raw === undefined) return undefined return OwnershipRecord.parse(raw) } async function readReverse(scopeID: string): Promise { - const raw = await Storage.read(StoragePath.channelManagedOwnershipReverse(scopeID)).catch(() => undefined) + const raw = await Storage.read(StoragePath.channelManagedOwnershipReverse(scopeID)).catch((error) => { + if (error instanceof Storage.NotFoundError) return undefined + throw error + }) if (raw === undefined) return undefined const identity = z .object({ @@ -163,83 +169,85 @@ export namespace ManagedProjectOwnership { await validateDirectoryChain(directory, true) const scope = await resolveScope(directory) - const reverseIdentity = await readReverse(scope.id) - if (reverseIdentity && identityHash(reverseIdentity) !== hash) { - throw new OwnershipMismatchError({ - scopeID: scope.id, - actualChannelType: reverseIdentity.channelType, - actualAccountId: reverseIdentity.accountId, - actualExternalProjectId: reverseIdentity.externalProjectId, - }) - } - - const existing = await readForward(hash) - - if (existing) { - validateReverseIndex(hash, existing) - - if (existing.scopeID !== scope.id) { + return Storage.transaction(async () => { + const reverseIdentity = await readReverse(scope.id) + if (reverseIdentity && identityHash(reverseIdentity) !== hash) { throw new OwnershipMismatchError({ - scopeID: existing.scopeID, - actualChannelType: scope.id, - actualAccountId: input.accountId, - actualExternalProjectId: input.externalProjectId, + scopeID: scope.id, + actualChannelType: reverseIdentity.channelType, + actualAccountId: reverseIdentity.accountId, + actualExternalProjectId: reverseIdentity.externalProjectId, }) } - if (path.resolve(existing.directory) !== path.resolve(directory)) { - throw new OwnershipMismatchError({ - scopeID: existing.scopeID, - actualChannelType: existing.directory, - actualAccountId: directory, - actualExternalProjectId: input.externalProjectId, + const existing = await readForward(hash) + + if (existing) { + validateReverseIndex(hash, existing) + + if (existing.scopeID !== scope.id) { + throw new OwnershipMismatchError({ + scopeID: existing.scopeID, + actualChannelType: scope.id, + actualAccountId: input.accountId, + actualExternalProjectId: input.externalProjectId, + }) + } + + if (path.resolve(existing.directory) !== path.resolve(directory)) { + throw new OwnershipMismatchError({ + scopeID: existing.scopeID, + actualChannelType: existing.directory, + actualAccountId: directory, + actualExternalProjectId: input.externalProjectId, + }) + } + + const updated: OwnershipRecord = { + ...existing, + remoteState: input.remoteState, + lastSeenAt: Date.now(), + } + + if (input.projectName !== undefined && existing.lastSeenAt === existing.createdAt && scope.name === undefined) { + await Scope.updatePersisted({ scopeID: scope.id, name: input.projectName }) + } + + await writeForward(hash, updated) + await writeReverse(scope.id, { + channelType: input.channelType, + accountId: input.accountId, + externalProjectId: input.externalProjectId, }) + return { ...updated } } - const updated: OwnershipRecord = { - ...existing, - remoteState: input.remoteState, - lastSeenAt: Date.now(), - } + const now = Date.now() - if (input.projectName !== undefined && existing.lastSeenAt === existing.createdAt && scope.name === undefined) { + if (input.projectName !== undefined) { await Scope.updatePersisted({ scopeID: scope.id, name: input.projectName }) } - await writeForward(hash, updated) + const record: OwnershipRecord = { + channelType: input.channelType, + accountId: input.accountId, + externalProjectId: input.externalProjectId, + scopeID: scope.id, + directory: scope.directory, + remoteState: input.remoteState, + createdAt: now, + lastSeenAt: now, + } + + await writeForward(hash, record) await writeReverse(scope.id, { channelType: input.channelType, accountId: input.accountId, externalProjectId: input.externalProjectId, }) - return { ...updated } - } - const now = Date.now() - - if (input.projectName !== undefined) { - await Scope.updatePersisted({ scopeID: scope.id, name: input.projectName }) - } - - const record: OwnershipRecord = { - channelType: input.channelType, - accountId: input.accountId, - externalProjectId: input.externalProjectId, - scopeID: scope.id, - directory: scope.directory, - remoteState: input.remoteState, - createdAt: now, - lastSeenAt: now, - } - - await writeForward(hash, record) - await writeReverse(scope.id, { - channelType: input.channelType, - accountId: input.accountId, - externalProjectId: input.externalProjectId, + return { ...record } }) - - return { ...record } } export async function find(input: OwnershipIdentity): Promise { diff --git a/packages/harness/AGENTS.md b/packages/harness/AGENTS.md index 554d841cc..f043b459e 100644 --- a/packages/harness/AGENTS.md +++ b/packages/harness/AGENTS.md @@ -15,3 +15,5 @@ Run bun run typecheck and the affected tests, then the root package and dependen - Keep application-facing operations under the explicit session, scope, tools, context, lifecycle, config, persistence and rollout entries. Host adapters may use individually declared contract leaves; never export processor, resolver, scheduler or journal implementations to production. Published manifests omit `test/` exports, and production sources must not import them. Within this package use relative owner imports, not public entry points. - File browsing, indexing, Ripgrep, Hashline editing and conflict resolution belong to runtime-local; this package retains execution read evidence and locking. + +Agent authority is owned by `Storage.Handle`: read [Agent storage](../../docs/architecture/agent-storage.md) before persistence changes. Use SQL business transactions for records, indexes and outbox entries; keep files and external effects outside retryable callbacks. The central bootstrap owns historical JSON import and activation. Runtime and maintenance entry points run registered owner recovery before admission. Storage engine changes run `bun test test/storage`; the PostgreSQL CI matrix requires a real database via `SYNERGY_TEST_POSTGRES_URL`. diff --git a/packages/harness/package.json b/packages/harness/package.json index 01734d9ab..b66fd6f40 100644 --- a/packages/harness/package.json +++ b/packages/harness/package.json @@ -11,6 +11,9 @@ "./session/snapshot-git": "./src/session/snapshot-git.ts", "./scope": "./src/scope/index.ts", "./scope/context": "./src/scope/context.ts", + "./storage/sqlite-worker": "./src/storage/sqlite-worker.ts", + "./storage/bootstrap": "./src/storage/bootstrap.ts", + "./storage/transactional-store": "./src/storage/transactional-store.ts", "./storage/path": "./src/storage/path.ts", "./config/config": "./src/config/config.ts", "./provider/models-schemas": "./src/provider/models-schemas.ts", @@ -309,7 +312,13 @@ "./test/internal/session/rollout/call": "./src/session/rollout/call.ts", "./test/internal/config/migration": "./src/config/migration.ts", "./config/schema": "./src/config/schema.ts", - "./test/support/internals": "./test/support/internals.ts" + "./test/support/internals": "./test/support/internals.ts", + "./storage/maintenance": "./src/storage/maintenance.ts", + "./storage/sqlite-engine": "./src/storage/sqlite-engine.ts", + "./storage/config": "./src/storage/config.ts", + "./storage/portable": "./src/storage/portable.ts", + "./storage/legacy-import": "./src/storage/legacy-import.ts", + "./storage/recovery": "./src/storage/recovery.ts" }, "scripts": { "typecheck": "tsgo --noEmit", diff --git a/packages/harness/script/benchmark-storage.ts b/packages/harness/script/benchmark-storage.ts new file mode 100644 index 000000000..cfb1e3fd3 --- /dev/null +++ b/packages/harness/script/benchmark-storage.ts @@ -0,0 +1,63 @@ +import fs from "node:fs/promises" +import os from "node:os" +import path from "node:path" +import { TransactionalStore } from "../src/storage/transactional-store" + +const root = await fs.mkdtemp(path.join(os.tmpdir(), "synergy-storage-benchmark-")) +const count = 1000 +const concurrency = 32 +const payload = "x".repeat(1024) +try { + for (const backend of ["sqlite", ...(process.env.SYNERGY_TEST_POSTGRES_URL ? ["postgres"] : [])] as const) { + const namespace = `benchmark_${crypto.randomUUID()}` + const store = await TransactionalStore.open( + backend === "sqlite" + ? { backend, namespace, filename: path.join(root, "agent.sqlite") } + : { backend: "postgres", namespace, url: process.env.SYNERGY_TEST_POSTGRES_URL! }, + ) + try { + const started = performance.now() + const latencies: number[] = [] + for (let offset = 0; offset < count; offset += concurrency) { + await Promise.all( + Array.from({ length: Math.min(concurrency, count - offset) }, async (_, index) => { + const id = String(offset + index).padStart(8, "0") + const start = performance.now() + await store.transaction(async (tx) => { + await tx.write(["benchmark", "parts", id], { id, payload }) + await tx.write(["benchmark-index", id], { id }) + }) + latencies.push(performance.now() - start) + }), + ) + } + const elapsed = performance.now() - started + const readStarted = performance.now() + const page = await store.query({ kind: "benchmark", limit: 100, descending: true }) + const readMs = performance.now() - readStarted + if (page.length !== 100 || (await store.verify()).records !== count * 2) + throw new Error("Benchmark verification failed") + latencies.sort((a, b) => a - b) + console.log( + JSON.stringify({ + backend, + transactions: count, + records: count * 2, + concurrency, + payloadBytes: 1024, + elapsedMs: Math.round(elapsed), + transactionsPerSecond: Math.round((count / elapsed) * 1000), + queuedP50Ms: +latencies[Math.floor(count * 0.5)].toFixed(2), + queuedP95Ms: +latencies[Math.floor(count * 0.95)].toFixed(2), + read100Ms: +readMs.toFixed(2), + processRssMiB: +(process.memoryUsage().rss / 1024 / 1024).toFixed(1), + }), + ) + await store.transaction((tx) => tx.removeTree([])) + } finally { + await store.close() + } + } +} finally { + await fs.rm(root, { recursive: true, force: true }) +} diff --git a/packages/harness/script/build-sqlite.ts b/packages/harness/script/build-sqlite.ts new file mode 100644 index 000000000..4a8953db4 --- /dev/null +++ b/packages/harness/script/build-sqlite.ts @@ -0,0 +1,70 @@ +import fs from "node:fs/promises" +import path from "node:path" +import { createHash } from "node:crypto" + +// SQLite's WAL reset fix: https://www.sqlite.org/wal.html#walreset +const SOURCE = "https://www.sqlite.org/2026/sqlite-amalgamation-3510300.zip" +const SHA256 = "acb1e6f5d832484bf6d32b681e858c38add8b2acdfd42ac5df24b8afb46552b4" +const directory = path.resolve(import.meta.dirname, "../.artifacts/sqlite") +const filename = path.join(directory, "libsqlite3.dylib") + +export async function buildSqlite() { + if (await Bun.file(filename).exists()) { + const receipt = await Bun.file(path.join(directory, "receipt.json")).json() + const hash = createHash("sha256") + .update(await Bun.file(filename).bytes()) + .digest("hex") + if (receipt.source === SHA256 && receipt.binary === hash) return filename + throw new Error("SQLite build receipt does not match the staged engine") + } + if (process.platform !== "darwin") + throw new Error("Build the universal SQLite engine on macOS or download the sqlite-assets-darwin CI artifact") + await fs.mkdir(directory, { recursive: true }) + const stage = await fs.mkdtemp(path.join(directory, ".build-")) + try { + const response = await fetch(SOURCE) + if (!response.ok) throw new Error(`SQLite source download failed: ${response.status}`) + const bytes = new Uint8Array(await response.arrayBuffer()) + if (createHash("sha256").update(bytes).digest("hex") !== SHA256) throw new Error("SQLite source checksum mismatch") + const archive = path.join(stage, "source.zip") + await Bun.write(archive, bytes) + await run(["unzip", "-q", archive, "-d", stage]) + const source = path.join(stage, "sqlite-amalgamation-3510300/sqlite3.c") + const output = path.join(stage, "libsqlite3.dylib") + await run([ + "clang", + "-O2", + "-dynamiclib", + "-arch", + "arm64", + "-arch", + "x86_64", + "-mmacosx-version-min=11.0", + "-DSQLITE_THREADSAFE=1", + "-DSQLITE_ENABLE_FTS5", + "-DSQLITE_ENABLE_RTREE", + "-DSQLITE_ENABLE_COLUMN_METADATA", + "-DSQLITE_DQS=0", + "-install_name", + "@rpath/libsqlite3.dylib", + source, + "-o", + output, + ]) + await fs.rename(output, filename) + const binary = createHash("sha256") + .update(await Bun.file(filename).bytes()) + .digest("hex") + await Bun.write(path.join(directory, "receipt.json"), JSON.stringify({ source: SHA256, binary, version: "3.51.3" })) + return filename + } finally { + await fs.rm(stage, { recursive: true, force: true }) + } +} + +async function run(command: string[]) { + const process = Bun.spawn(command, { stdout: "inherit", stderr: "inherit" }) + if (await process.exited) throw new Error(`SQLite build failed: ${command[0]}`) +} + +if (import.meta.main) console.log(await buildSqlite()) diff --git a/packages/harness/src/bus/index.ts b/packages/harness/src/bus/index.ts index 34d8da35c..691e1fe05 100644 --- a/packages/harness/src/bus/index.ts +++ b/packages/harness/src/bus/index.ts @@ -1,3 +1,4 @@ +import { Storage } from "../storage/storage" import z from "zod" import { Log } from "../util/log" import { ScopeContext } from "../scope/context" @@ -62,7 +63,15 @@ export namespace Bus { export async function publish( def: Definition, properties: z.output, - ) { + ): Promise { + if (Storage.inTransaction()) { + const scope = ScopeContext.current.scope + const value = structuredClone(properties) + return Storage.enqueue( + { id: crypto.randomUUID(), scopeID: scope.id, type: def.type, payload: { scope, properties: value } }, + () => ScopeContext.provide({ scope, fn: () => publish(def, value) }), + ) + } const payload: { type: string properties: unknown diff --git a/packages/harness/src/config/config.ts b/packages/harness/src/config/config.ts index 8f8db837f..719eb8296 100644 --- a/packages/harness/src/config/config.ts +++ b/packages/harness/src/config/config.ts @@ -280,7 +280,9 @@ export namespace Config { // Inline config content has highest precedence if (Flag.SYNERGY_CONFIG_CONTENT) { - merge(Info.parse(LegacyExecutionConfig.migrate(JSON.parse(Flag.SYNERGY_CONFIG_CONTENT))), "inline_config") + const inline = Info.parse(LegacyExecutionConfig.migrate(JSON.parse(Flag.SYNERGY_CONFIG_CONTENT))) + if (inline.storage) throw new Error("Storage configuration must use the global 130-storage.jsonc domain file") + merge(inline, "inline_config") log.debug("loaded custom config from SYNERGY_CONFIG_CONTENT") } @@ -498,6 +500,8 @@ export namespace Config { try { const fragment = await loadFile(filepath, { addSchema: false }) ConfigDomain.validateKeys(fragment as Record, domain.id, { preserveUnregistered: true }) + if (domain.id === "storage" && fragment.storage && path.resolve(root) !== path.resolve(Global.Path.config)) + throw new Error("Storage configuration is global and cannot be overridden by a project") result = mergeConfigConcatArrays(result, fragment as Info) // A recovered file clears its historical diagnostic so the registry // reflects the most recent load. Only clear when the file really @@ -507,7 +511,7 @@ export namespace Config { clearIssueForPath(filepath) } } catch (error) { - if (strictExecution.getStore()) throw error + if (domain.id === "storage" || strictExecution.getStore()) throw error await quarantineDomainFile(domain.id, filepath, error) } } @@ -1395,6 +1399,8 @@ export namespace Config { options: { mode?: ConfigDomain.MergeMode; root?: string } = {}, ) { const parsed = ConfigDomain.Id.parse(id) + if (parsed === "storage") + throw new Error("Change the active storage target with data storage migrate --target; it cannot be hot-reloaded") using _ = await Lock.write(`config-domain:${ConfigDomain.filepath(parsed, options.root)}`) // `return await` (not bare `return promise`): with `using`, a bare return // disposes the lock before the async transaction has run, so concurrent @@ -1433,6 +1439,8 @@ export namespace Config { options: { mode?: ConfigDomain.MergeMode } = {}, ) { const parsed = ConfigDomain.Id.parse(id) + if (parsed === "storage") + throw new Error("Change the active storage target with data storage migrate --target; it cannot be hot-reloaded") using _ = await Lock.write(`config-domain:${ConfigDomain.filepath(parsed)}`) const oldConfig = await globalResolved() const current = await domainGet(parsed) @@ -1484,6 +1492,8 @@ export namespace Config { options: { mode?: ConfigDomain.MergeMode } = {}, ) { const parsed = ConfigDomain.Id.parse(id) + if (parsed === "storage") + throw new Error("Change the active storage target with data storage migrate --target; it cannot be hot-reloaded") using _ = await Lock.write(`config-domain:${ConfigDomain.filepath(parsed)}`) const oldConfig = await globalResolved() const result = await domainUpdateUnlocked(parsed, patch, options) diff --git a/packages/harness/src/config/domain.ts b/packages/harness/src/config/domain.ts index 4269f2901..5e01a1b24 100644 --- a/packages/harness/src/config/domain.ts +++ b/packages/harness/src/config/domain.ts @@ -116,6 +116,16 @@ export namespace ConfigDomain { uiSection: "runtime", importable: true, }, + { + id: "storage", + filename: "130-storage.jsonc", + label: "Storage", + ownedKeys: ["storage"], + mergePolicy: "replace-domain", + reloadTargets: [], + uiSection: "storage", + importable: false, + }, ] satisfies Definition[] export const byId = new Map(definitions.map((item) => [item.id, item])) diff --git a/packages/harness/src/config/schema.ts b/packages/harness/src/config/schema.ts index 6096e8544..ad01b5741 100644 --- a/packages/harness/src/config/schema.ts +++ b/packages/harness/src/config/schema.ts @@ -1,3 +1,4 @@ +import { StorageConfiguration } from "../storage/config" import { Log } from "../util/log" import z from "zod" import { MAX_EXECUTION_CANCEL_GRACE_MS } from "@ericsanchezok/synergy-util/runtime-shutdown" @@ -467,6 +468,9 @@ const CoreInfo = z .object({ $schema: z.string().optional().describe("JSON schema reference for configuration validation"), logLevel: Log.Level.optional().describe("Log level"), + storage: StorageConfiguration.optional().describe( + "Global authoritative storage; backend changes require an explicit storage migration", + ), server: Server.optional().describe("Server configuration for synergy serve and web commands"), command: z.record(z.string(), Command).optional().describe("Command configuration"), timeout: z diff --git a/packages/harness/src/global/index.ts b/packages/harness/src/global/index.ts index 0098e6b4b..c265eb9db 100644 --- a/packages/harness/src/global/index.ts +++ b/packages/harness/src/global/index.ts @@ -14,6 +14,8 @@ function homeDir() { } function root() { + if (process.argv.includes("__storage-maintenance-runner") && process.env.SYNERGY_MAINTENANCE_ROOT) + return path.resolve(process.env.SYNERGY_MAINTENANCE_ROOT) return path.join(homeDir(), "." + app) } diff --git a/packages/harness/src/lifecycle/runtime.ts b/packages/harness/src/lifecycle/runtime.ts index 06ca4556c..5eb2a01a9 100644 --- a/packages/harness/src/lifecycle/runtime.ts +++ b/packages/harness/src/lifecycle/runtime.ts @@ -1,3 +1,7 @@ +import { SessionStaging } from "../session/staging" +import { StorageRecovery } from "../storage/recovery" +import { Storage } from "../storage/storage" +import { StorageBootstrap } from "../storage/bootstrap" import { ConfigExtensions } from "../config/extensions" import { MigrationRegistry } from "../migration/registry" import { ensureMigrations, type MigrationReporter, type RunOptions } from "../migration/index" @@ -54,8 +58,9 @@ export namespace RuntimeHandle { export async function open(options: { experiment?: Experiment.File + storage?: Storage.Handle mode: "server" | "oneshot" - network?: RuntimeNetwork + network?: RuntimeNetwork | (() => Promise) services?: RuntimeServices reporter?: MigrationReporter migrationOutput?: RunOptions["output"] @@ -63,6 +68,8 @@ export namespace RuntimeHandle { }) { const services = options.services ?? {} const ownership = await ServerProcessLock.acquire(undefined, options.mode === "oneshot" ? "oneshot" : undefined) + let storage: StorageBootstrap.Prepared | undefined + let uninstallStorage: (() => void) | undefined let server: RuntimeServer | undefined let residentStarted = false let closing: Promise | undefined @@ -134,6 +141,8 @@ export namespace RuntimeHandle { ObservabilityResources.stop() await cleanup(() => Observability.flush()) await cleanup(() => ObservabilityStore.close()) + await cleanup(() => storage?.store.close()) + uninstallStorage?.() await cleanup(() => ownership.release()) ScopeStartup.configure("server") Experiment.configureRuntime() @@ -146,10 +155,21 @@ export namespace RuntimeHandle { MigrationRegistry.lock() ConfigExtensions.lock() await Global.initialize({ configSchemaPath: options.services?.configSchemaPath }) + if (options.storage) uninstallStorage = Storage.install(options.storage) + else { + storage = await StorageBootstrap.prepare({ root: Global.Path.root }) + uninstallStorage = Storage.install({ store: storage.store, artifactDirectory: Global.Path.data }) + } + await SessionStaging.recover() const migration = await ensureMigrations({ output: options.migrationOutput ?? "silent", reporter: options.reporter, }) + if (storage && storage.manifest.phase !== "active") await StorageRecovery.validate() + await storage?.activate() + await StorageRecovery.recoverOwners() + await StorageRecovery.load() + await StorageRecovery.reconcileNotifications() const resolved = await ScopeContext.provide({ scope: Scope.home(), fn: () => Config.resolveExecution() }) const requested = Experiment.applyRuntime(resolved, options.experiment?.runtime ?? {}) const shutdownTimeoutMs = configureExecution(requested) @@ -173,7 +193,10 @@ export namespace RuntimeHandle { }, }) if (services.transport) { - const network = options.network ?? { hostname: "127.0.0.1", port: 0 } + const network = (typeof options.network === "function" ? await options.network() : options.network) ?? { + hostname: "127.0.0.1", + port: 0, + } server = services.transport.listen(network, options.mode) configureRuntimeEndpoint({ hostname: server.hostname ?? network.hostname, port: server.port ?? network.port }) } diff --git a/packages/harness/src/migration/index.ts b/packages/harness/src/migration/index.ts index 90203ffa4..8f7bfb1c8 100644 --- a/packages/harness/src/migration/index.ts +++ b/packages/harness/src/migration/index.ts @@ -1,14 +1,11 @@ -import path from "path" import { Storage } from "../storage/storage" import { StoragePath } from "../storage/path" import { Log } from "../util/log" import { MigrationRegistry } from "./registry" import { orderMigrations } from "./order" import { progressBar, stageWrite, disableWrap, enableWrap, PROGRESS_INTERVAL } from "./format" -import { Global } from "../global" import { Installation } from "../global/installation" import { setActiveMigrationContext } from "./context" -import { withFileLock } from "@ericsanchezok/synergy-util/fs-lock" // Side-effect imports: register harness-core domain migrations in // MigrationRegistry. Product-domain migrations register through the L4 // product manifest (src/product-registration.ts) loaded by real entry points. @@ -22,9 +19,7 @@ export type { Migration, RunOptions, RunResult, MigrationContext, MigrationSumma const log = Log.create({ service: "migration" }) -let runningMigrations: Promise | undefined -let migrationsCompleted = false -let lastSummary: MigrationSummary | undefined +const states = new WeakMap; summary?: MigrationSummary }>() function collectByDomain(options?: { targetDomain?: string }): Map { const result = new Map() @@ -49,7 +44,7 @@ function collectByDomain(options?: { targetDomain?: string }): Map { const oldLogKey = StoragePath.metaMigrationLog() - const oldData = await Storage.read>(oldLogKey).catch(() => null) + const oldData = await Storage.read>(oldLogKey).catch(missingLog) if (!oldData) return const remaining = { ...oldData } @@ -89,7 +84,7 @@ async function migrateOldTrackingData(): Promise { async function migrateRegisteredLegacyTrackingData(): Promise { for (const owner of MigrationRegistry.legacyTracking()) { const oldKey = StoragePath.metaMigrationLogDomain(owner.sourceDomain) - const oldData = await Storage.read>(oldKey).catch(() => null) + const oldData = await Storage.read>(oldKey).catch(missingLog) if (!oldData) continue const migrated = Object.fromEntries( Object.entries(oldData).map(([id, timestamp]) => [owner.aliases[id] ?? owner.rename?.(id) ?? id, timestamp]), @@ -101,32 +96,37 @@ async function migrateRegisteredLegacyTrackingData(): Promise { } export async function ensureMigrations(options?: RunOptions): Promise { - if (migrationsCompleted) { - const summary = lastSummary ?? emptySummary() - options?.reporter?.summary(summary) - return summary + const store = Storage.current().store + let state = states.get(store) + if (!state) { + state = {} + states.set(store, state) + } + if (state.summary) { + options?.reporter?.summary(state.summary) + return state.summary } - runningMigrations ??= runMigrations({ ...options, output: options?.output ?? "silent" }) + const current = state + current.running ??= runMigrations({ ...options, output: options?.output ?? "silent" }) .then((summary) => { - migrationsCompleted = true - lastSummary = summary + current.summary = summary return summary }) .finally(() => { - runningMigrations = undefined + current.running = undefined }) - return runningMigrations + return current.running } export function resetMigrations(): void { - migrationsCompleted = false - runningMigrations = undefined - lastSummary = undefined + if (Storage.available()) states.delete(Storage.current().store) } export async function runMigrations(options?: RunOptions): Promise { - await migrateOldTrackingData() - await migrateRegisteredLegacyTrackingData() + if (!options?.dryRun) { + await migrateOldTrackingData() + await migrateRegisteredLegacyTrackingData() + } const dryRun = options?.dryRun ?? false const output = options?.output ?? "interactive" @@ -267,31 +267,20 @@ function emptySummary(): MigrationSummary { } async function loadLogForDomain(domain: string): Promise> { - return Storage.read>(StoragePath.metaMigrationLogDomain(domain)).catch(() => ({})) + return Storage.read>(StoragePath.metaMigrationLogDomain(domain)).catch( + (error) => missingLog(error) ?? {}, + ) } -function migrationLockDirectory() { - return path.join(Global.Path.data, "meta", "migration", ".locks") +function missingLog(error: unknown): undefined { + if (error instanceof Storage.NotFoundError) return + throw error } -function migrationLockOptions(domain: string) { - return { - directory: migrationLockDirectory(), - key: `migration-log:${domain}`, - timeoutMessage: `Timed out acquiring migration tracking lock for ${domain}`, - } -} - -/** - * Merge completion markers into the per-domain migration log under a cross-process - * file lock. Multiple Synergy instances sharing one home may run migrations - * concurrently; a plain read-modify-write would let the last writer drop markers - * persisted by the other process, re-pending completed migrations on the next boot. - */ async function mergeDomainLog(domain: string, entries: Record): Promise { const key = StoragePath.metaMigrationLogDomain(domain) - await withFileLock(migrationLockOptions(domain), async () => { - const current = await Storage.read>(key).catch(() => ({})) + await Storage.transaction(async () => { + const current = await loadLogForDomain(domain) await Storage.write(key, { ...current, ...entries }) }) } @@ -303,8 +292,10 @@ async function saveLogForDomain(domain: string, data: Record): P /** Remove a single completion marker under the same cross-process lock, preserving concurrent markers. */ async function deleteDomainLogEntry(domain: string, migrationID: string): Promise { const key = StoragePath.metaMigrationLogDomain(domain) - await withFileLock(migrationLockOptions(domain), async () => { - const current: Record = await Storage.read>(key).catch(() => ({})) + await Storage.transaction(async () => { + const current: Record = await Storage.read>(key).catch( + (error) => missingLog(error) ?? {}, + ) delete current[migrationID] await Storage.write(key, current) }) @@ -384,8 +375,6 @@ export async function rollbackMigrations(domain: string, targetId: string): Prom export async function getMigrationStatus( domain?: string, ): Promise> { - await migrateOldTrackingData() - const domains = collectByDomain({ targetDomain: domain }) const result: Record = {} diff --git a/packages/harness/src/observability/migration.ts b/packages/harness/src/observability/migration.ts index 8ad54b159..e34ceff6f 100644 --- a/packages/harness/src/observability/migration.ts +++ b/packages/harness/src/observability/migration.ts @@ -1,3 +1,4 @@ +import { initializeSqliteEngine } from "../storage/sqlite-engine" import fs from "fs/promises" import { Database } from "bun:sqlite" import { MigrationRegistry } from "../migration/registry" @@ -26,6 +27,7 @@ export namespace ObservabilityMigration { progress(1, 1) return } + initializeSqliteEngine() const legacy = new Database(legacyPath, { readonly: true }) try { const steps = [ diff --git a/packages/harness/src/observability/store.ts b/packages/harness/src/observability/store.ts index a448273a5..4b051467c 100644 --- a/packages/harness/src/observability/store.ts +++ b/packages/harness/src/observability/store.ts @@ -1,3 +1,4 @@ +import { initializeSqliteEngine } from "../storage/sqlite-engine" import { Database } from "bun:sqlite" import fsSync from "fs" import { ObservabilityConfig } from "./config" @@ -82,6 +83,7 @@ export namespace ObservabilityStore { if (!inlineMode()) { if (!readonlyDb) { try { + initializeSqliteEngine() const conn = new Database(pathName(), { readonly: true }) conn.exec("PRAGMA busy_timeout=5000") readonlyDb = conn @@ -774,6 +776,7 @@ export namespace ObservabilityStore { function createConnection() { fsSync.mkdirSync(ObservabilityPaths.dir(), { recursive: true }) const fresh = !fsSync.existsSync(ObservabilityPaths.pathName()) + initializeSqliteEngine() const conn = new Database(ObservabilityPaths.pathName(), { create: true }) ObservabilityDbSchema.configureWriteConnection(conn, fresh) return conn diff --git a/packages/harness/src/observability/telemetry-worker.ts b/packages/harness/src/observability/telemetry-worker.ts index a1d8b401f..7e4315001 100644 --- a/packages/harness/src/observability/telemetry-worker.ts +++ b/packages/harness/src/observability/telemetry-worker.ts @@ -1,3 +1,4 @@ +import { initializeSqliteEngine } from "../storage/sqlite-engine" import { Database } from "bun:sqlite" import fs from "fs" import path from "path" @@ -130,6 +131,7 @@ function handle(message: TelemetryProtocol.HostToWorker): void { if (db) return fs.mkdirSync(path.dirname(message.dbPath), { recursive: true }) const fresh = !fs.existsSync(message.dbPath) + initializeSqliteEngine() const conn = new Database(message.dbPath, { create: true }) ObservabilityDbSchema.configureWriteConnection(conn, fresh) const autoVacuum = conn.query("PRAGMA auto_vacuum").get() as { auto_vacuum?: number } | undefined diff --git a/packages/harness/src/public/persistence.ts b/packages/harness/src/public/persistence.ts index 391b7910e..206868492 100644 --- a/packages/harness/src/public/persistence.ts +++ b/packages/harness/src/public/persistence.ts @@ -1,3 +1,6 @@ export { Storage } from "../storage/storage" export { MigrationRegistry, MigrationRegistrationLockedError } from "../migration/registry" export type { Migration } from "../migration/types" +export { TransactionalStore, StoreTransaction } from "../storage/transactional-store" +export type { StoreOptions, TransactionOptions, StoredRecord, RecordQuery } from "../storage/transactional-store" +export { StorageBootstrap } from "../storage/bootstrap" diff --git a/packages/harness/src/scope/index.ts b/packages/harness/src/scope/index.ts index 668132d8c..5a9b27a5d 100644 --- a/packages/harness/src/scope/index.ts +++ b/packages/harness/src/scope/index.ts @@ -68,7 +68,10 @@ export namespace Scope { } async function readPersisted(scopeID: string) { - return Storage.read>(StoragePath.scope(pid(scopeID))).catch(() => undefined) + return Storage.read>(StoragePath.scope(pid(scopeID))).catch((error) => { + if (error instanceof Storage.NotFoundError) return undefined + throw error + }) } export async function fromID(scopeID: string): Promise { @@ -89,6 +92,17 @@ export namespace Scope { } } + function publish( + definition: Definition, + properties: z.output, + scopeID: string, + ) { + const payload = { type: definition.type, properties: structuredClone(properties) } + return Storage.enqueue({ id: crypto.randomUUID(), scopeID, type: definition.type, payload }, async () => { + GlobalBus.emit("event", { payload }) + }) + } + async function writePersisted(data: z.infer) { await Storage.write(StoragePath.scope(pid(data.id)), data) } @@ -96,7 +110,7 @@ export namespace Scope { async function findByWorktree(worktree: string): Promise | undefined> { const resolved = path.resolve(worktree) for (const rawID of await Storage.scan(StoragePath.scopeRoot())) { - const data = await readPersisted(rawID).catch(() => undefined) + const data = await readPersisted(rawID) if (!data || data.time?.archived) continue if (path.resolve(data.worktree) === resolved) return data } @@ -352,7 +366,23 @@ export namespace Scope { project.vcs !== existing.vcs || project.sandboxes.length !== previousSandboxes.length || project.sandboxes.some((entry, index) => entry !== previousSandboxes[index]) - if (persist && recordChanged) await writePersisted(project) + if (persist && recordChanged) + await Storage.transaction(async () => { + const latest = await readPersisted(project.id) + const merged = latest + ? { + ...latest, + directory: project.directory, + worktree: project.worktree, + vcs: project.vcs, + sandboxes: [...new Set([...(latest.sandboxes ?? []), ...project.sandboxes])].filter((entry) => + existsSync(entry), + ), + } + : project + await writePersisted(merged) + await publish(Event.Updated, merged, merged.id) + }) const scope: Scope.Project = { type: "project", @@ -367,15 +397,6 @@ export namespace Scope { time: project.time, } - if (persist && recordChanged) { - GlobalBus.emit("event", { - payload: { - type: Event.Updated.type, - properties: project, - }, - }) - } - return { scope, sandbox } } @@ -441,60 +462,54 @@ export namespace Scope { archived?: number | null sandboxes?: string[] }) { - if (input.scopeID === "home") return undefined - if (input.archived !== undefined && input.archived !== null) { - for (const guard of archiveGuards) await guard(input.scopeID) - } - const result = await Storage.update>(StoragePath.scope(pid(input.scopeID)), (draft) => { - if (input.name !== undefined) draft.name = input.name - if (input.icon !== undefined) { - draft.icon = { ...draft.icon } - if (input.icon.url !== undefined) draft.icon!.url = input.icon.url - if (input.icon.color !== undefined) draft.icon!.color = input.icon.color - } - if (input.pinned !== undefined) { - draft.pinned = input.pinned ?? undefined - } - if (input.archived !== undefined) { - draft.time.archived = input.archived ?? undefined + return Storage.transaction(async () => { + if (input.scopeID === "home") return undefined + if (input.archived !== undefined && input.archived !== null) { + for (const guard of archiveGuards) await guard(input.scopeID) } - if (input.sandboxes !== undefined) { - const worktree = path.resolve(draft.worktree) - const seen = new Set() - draft.sandboxes = input.sandboxes - .filter((s) => path.isAbsolute(s)) - .filter((s) => { - const resolved = path.resolve(s) - if (resolved === worktree) return false - if (seen.has(resolved)) return false - seen.add(resolved) - return true - }) - } - draft.time.updated = Date.now() - }) - GlobalBus.emit("event", { - payload: { - type: Event.Updated.type, - properties: result, - }, + const result = await Storage.update>(StoragePath.scope(pid(input.scopeID)), (draft) => { + if (input.name !== undefined) draft.name = input.name + if (input.icon !== undefined) { + draft.icon = { ...draft.icon } + if (input.icon.url !== undefined) draft.icon!.url = input.icon.url + if (input.icon.color !== undefined) draft.icon!.color = input.icon.color + } + if (input.pinned !== undefined) { + draft.pinned = input.pinned ?? undefined + } + if (input.archived !== undefined) { + draft.time.archived = input.archived ?? undefined + } + if (input.sandboxes !== undefined) { + const worktree = path.resolve(draft.worktree) + const seen = new Set() + draft.sandboxes = input.sandboxes + .filter((s) => path.isAbsolute(s)) + .filter((s) => { + const resolved = path.resolve(s) + if (resolved === worktree) return false + if (seen.has(resolved)) return false + seen.add(resolved) + return true + }) + } + draft.time.updated = Date.now() + }) + await publish(Event.Updated, result, result.id) + return result }) - return result } export async function remove(scopeID: string) { - if (scopeID === "home") return undefined - for (const guard of archiveGuards) await guard(scopeID) - const result = await Storage.update>(StoragePath.scope(pid(scopeID)), (draft) => { - draft.time.archived = Date.now() - }) - GlobalBus.emit("event", { - payload: { - type: Event.Removed.type, - properties: { id: scopeID, directory: result.worktree }, - }, + return Storage.transaction(async () => { + if (scopeID === "home") return undefined + for (const guard of archiveGuards) await guard(scopeID) + const result = await Storage.update>(StoragePath.scope(pid(scopeID)), (draft) => { + draft.time.archived = Date.now() + }) + await publish(Event.Removed, { id: scopeID, directory: result.worktree }, scopeID) + return result }) - return result } export async function sandboxes(scopeID: string) { diff --git a/packages/harness/src/scope/migration.ts b/packages/harness/src/scope/migration.ts index 9dad9af83..86a0d725d 100644 --- a/packages/harness/src/scope/migration.ts +++ b/packages/harness/src/scope/migration.ts @@ -49,7 +49,6 @@ export const migrations: Migration[] = [ description: "Consolidate orphan scope data (no active project, no worktree) into a reclaimed scope", async up(progress) { const now = Date.now() - const dataDir = Global.Path.data const reclaimedSID = Identifier.asScopeID(RECLAIMED_SCOPE_ID) // 1. Collect all scopeIDs that have file-based data @@ -121,7 +120,7 @@ export const migrations: Migration[] = [ let done = 0 for (const orphanID of orphanIDs) { - await moveFileBasedData(orphanID, RECLAIMED_SCOPE_ID, dataDir) + await moveRecordsByScope(orphanID, RECLAIMED_SCOPE_ID) await removeOrphanProjectRecord(orphanID) done++ progress(done, totalSteps) @@ -190,28 +189,18 @@ export const migrations: Migration[] = [ id: "20260624-scope-global-to-home", description: "Rename legacy global scope data to the home scope", async up(progress) { - const dataDir = Global.Path.data const fromSID = Identifier.asScopeID(LEGACY_GLOBAL_SCOPE_ID) const toSID = Identifier.asScopeID(HOME_SCOPE_ID) const steps = 12 let done = 0 - await moveFileBasedData(LEGACY_GLOBAL_SCOPE_ID, HOME_SCOPE_ID, dataDir) - await moveFile( - path.join(dataDir, ...StoragePath.sessionNavIndex(fromSID)) + ".json", - path.join(dataDir, ...StoragePath.sessionNavIndex(toSID)) + ".json", - ) + await moveRecordsByScope(LEGACY_GLOBAL_SCOPE_ID, HOME_SCOPE_ID) + await moveRecord(StoragePath.sessionNavIndex(fromSID), StoragePath.sessionNavIndex(toSID)) done++ progress(done, steps) - await moveDir( - path.join(dataDir, ...StoragePath.blueprintLoopsRoot(fromSID)), - path.join(dataDir, ...StoragePath.blueprintLoopsRoot(toSID)), - ) - await moveFile( - path.join(dataDir, ...StoragePath.permission(fromSID)) + ".json", - path.join(dataDir, ...StoragePath.permission(toSID)) + ".json", - ) + await moveRecords(StoragePath.blueprintLoopsRoot(fromSID), StoragePath.blueprintLoopsRoot(toSID)) + await moveRecord(StoragePath.permission(fromSID), StoragePath.permission(toSID)) await moveDir( path.join(Global.Path.snapshot, LEGACY_GLOBAL_SCOPE_ID), path.join(Global.Path.snapshot, HOME_SCOPE_ID), @@ -507,44 +496,41 @@ async function removeLegacyGlobalRoots() { await Storage.remove(StoragePath.permission(Identifier.asScopeID(LEGACY_GLOBAL_SCOPE_ID))).catch(() => undefined) } -async function moveFileBasedData(fromScopeID: string, toScopeID: string, dataDir: string) { - const fromSID = Identifier.asScopeID(fromScopeID) - const toSID = Identifier.asScopeID(toScopeID) - - // Move sessions directory (contains session dirs with info/messages/etc) - await moveDir( - path.join(dataDir, ...StoragePath.sessionsRoot(fromSID)), - path.join(dataDir, ...StoragePath.sessionsRoot(toSID)), - ) - - // Move sessions page index - await moveFile( - path.join(dataDir, ...StoragePath.sessionsPageIndex(fromSID)) + ".json", - path.join(dataDir, ...StoragePath.sessionsPageIndex(toSID)) + ".json", - ) - - // Move notes directory - await moveDir( - path.join(dataDir, ...StoragePath.notesRoot(fromSID)), - path.join(dataDir, ...StoragePath.notesRoot(toSID)), - ) - - // Move agenda items directory - await moveDir( - path.join(dataDir, ...StoragePath.agendaItemsRoot(fromSID)), - path.join(dataDir, ...StoragePath.agendaItemsRoot(toSID)), - ) +async function moveRecordsByScope(fromScopeID: string, toScopeID: string) { + const from = Identifier.asScopeID(fromScopeID) + const to = Identifier.asScopeID(toScopeID) + await moveRecords(StoragePath.sessionsRoot(from), StoragePath.sessionsRoot(to)) + await moveRecord(StoragePath.sessionsPageIndex(from), StoragePath.sessionsPageIndex(to)) + await moveRecords(StoragePath.notesRoot(from), StoragePath.notesRoot(to)) + await moveRecords(StoragePath.agendaItemsRoot(from), StoragePath.agendaItemsRoot(to)) + await moveRecords(["agenda", "runs", fromScopeID], ["agenda", "runs", toScopeID]) + await moveRecord(StoragePath.agendaRunIndex(from), StoragePath.agendaRunIndex(to)) +} - // Move agenda runs directory - const runsPrefix = ["agenda", "runs", fromScopeID] - const runsTarget = ["agenda", "runs", toScopeID] - await moveDir(path.join(dataDir, ...runsPrefix), path.join(dataDir, ...runsTarget)) +async function moveRecord(from: string[], to: string[]) { + await Storage.transaction(async (tx) => { + const [source, target] = await tx.readMany([from, to]) + if (source === undefined) return + if (target === undefined) await tx.write(to, source) + await tx.remove(from) + }) +} - // Move agenda run index - await moveFile( - path.join(dataDir, ...StoragePath.agendaRunIndex(fromSID)) + ".json", - path.join(dataDir, ...StoragePath.agendaRunIndex(toSID)) + ".json", - ) +async function moveRecords(from: string[], to: string[]) { + for (const child of await Storage.scan(from)) { + await Storage.transaction(async (tx) => { + const source = [...from, child] + const target = [...to, child] + const existing = await tx.readMany([target]) + if (existing[0] === undefined && !(await tx.list(target)).length) { + const [own] = await tx.readMany([source]) + if (own !== undefined) await tx.write(target, own) + for (const key of await tx.list(source)) + await tx.write([...target, ...key.slice(source.length)], await tx.read(key)) + } + await tx.removeTree(source) + }) + } } async function moveDir(from: string, to: string) { @@ -564,12 +550,6 @@ async function moveDir(from: string, to: string) { } } -async function moveFile(from: string, to: string) { - if (!existsSync(from)) return - await fs.mkdir(path.dirname(to), { recursive: true }) - await fs.rename(from, to).catch(() => {}) -} - async function removeOrphanProjectRecord(scopeID: string) { await Storage.remove(StoragePath.scope(Identifier.asScopeID(scopeID))).catch(() => {}) } diff --git a/packages/harness/src/session/dag.ts b/packages/harness/src/session/dag.ts index 29a3ac2b7..573787ed9 100644 --- a/packages/harness/src/session/dag.ts +++ b/packages/harness/src/session/dag.ts @@ -113,10 +113,12 @@ export namespace Dag { } export async function update(input: { sessionID: string; nodes: Node[] }) { - const ready = computeReady(input.nodes) - const scopeID = await resolveScopeID(input.sessionID) - await Storage.write(StoragePath.sessionDag(scopeID, asSessionID(input.sessionID)), input.nodes) - Bus.publish(Event.Updated, { sessionID: input.sessionID, nodes: input.nodes, ready }) + return Storage.transaction(async () => { + const ready = computeReady(input.nodes) + const scopeID = await resolveScopeID(input.sessionID) + await Storage.write(StoragePath.sessionDag(scopeID, asSessionID(input.sessionID)), input.nodes) + Bus.publish(Event.Updated, { sessionID: input.sessionID, nodes: input.nodes, ready }) + }) } export async function get(sessionID: string) { diff --git a/packages/harness/src/session/history.ts b/packages/harness/src/session/history.ts index 926bdc024..6f0c47799 100644 --- a/packages/harness/src/session/history.ts +++ b/packages/harness/src/session/history.ts @@ -151,60 +151,61 @@ export namespace SessionHistory { numTurns: z.number().int().min(1).optional(), cutMessageID: z.string().optional(), }), - async (input) => { - if ((input.numTurns == null) === (input.cutMessageID == null)) { - throw new Error("Provide exactly one of numTurns or cutMessageID") - } - SessionManager.assertIdle(input.sessionID) - const [raw, events] = await Promise.all([ - rawMessages({ sessionID: input.sessionID }), - readEvents(input.sessionID), - ]) - const effective = applyEvents(raw, events) - - let cutMessageID: string | undefined - let dropped: MessageV2.WithParts[] = [] - - if (input.cutMessageID) { - // cutMessageID mode: drop everything from cutMessageID onward - cutMessageID = input.cutMessageID - const cutIndex = effective.findIndex((msg) => msg.info.id === cutMessageID) - if (cutIndex >= 0) { - dropped = effective.slice(cutIndex) + async (input) => + Storage.transaction(async () => { + if ((input.numTurns == null) === (input.cutMessageID == null)) { + throw new Error("Provide exactly one of numTurns or cutMessageID") + } + SessionManager.assertIdle(input.sessionID) + const [raw, events] = await Promise.all([ + rawMessages({ sessionID: input.sessionID }), + readEvents(input.sessionID), + ]) + const effective = applyEvents(raw, events) + + let cutMessageID: string | undefined + let dropped: MessageV2.WithParts[] = [] + + if (input.cutMessageID) { + // cutMessageID mode: drop everything from cutMessageID onward + cutMessageID = input.cutMessageID + const cutIndex = effective.findIndex((msg) => msg.info.id === cutMessageID) + if (cutIndex >= 0) { + dropped = effective.slice(cutIndex) + } + } else { + // numTurns mode (must be defined due to .refine) + const numTurns = input.numTurns! + const turnStarts = effective.map((msg, index) => ({ msg, index })).filter(({ msg }) => isRollbackUser(msg)) + if (turnStarts.length === 0) return latestInfo(input.sessionID, raw, events) + const selected = turnStarts.slice(-numTurns) + const cutoff = selected[0].index + dropped = effective.slice(cutoff) + if (dropped.length === 0) return latestInfo(input.sessionID, raw, events) + cutMessageID = selected[0].msg.info.id } - } else { - // numTurns mode (must be defined due to .refine) - const numTurns = input.numTurns! - const turnStarts = effective.map((msg, index) => ({ msg, index })).filter(({ msg }) => isRollbackUser(msg)) - if (turnStarts.length === 0) return latestInfo(input.sessionID, raw, events) - const selected = turnStarts.slice(-numTurns) - const cutoff = selected[0].index - dropped = effective.slice(cutoff) - if (dropped.length === 0) return latestInfo(input.sessionID, raw, events) - cutMessageID = selected[0].msg.info.id - } - const selectedTurns = dropped.filter(isRollbackUser).length - - const event: RollbackEvent = { - id: Identifier.ascending("history"), - sessionID: input.sessionID, - type: "rollback", - time: { - created: Date.now(), - }, - numTurns: selectedTurns, - cutMessageID, - droppedMessageIDs: dropped.map((msg) => msg.info.id), - droppedUserMessageIDs: dropped.filter(isRollbackUser).map((msg) => msg.info.id), - ...summarizePatches(dropped), - } - await writeEvent(event) + const selectedTurns = dropped.filter(isRollbackUser).length + + const event: RollbackEvent = { + id: Identifier.ascending("history"), + sessionID: input.sessionID, + type: "rollback", + time: { + created: Date.now(), + }, + numTurns: selectedTurns, + cutMessageID, + droppedMessageIDs: dropped.map((msg) => msg.info.id), + droppedUserMessageIDs: dropped.filter(isRollbackUser).map((msg) => msg.info.id), + ...summarizePatches(dropped), + } + await writeEvent(event) - const nextEvents = [...events, event] - await updateSessionHistory(input.sessionID, info(input.sessionID, raw, nextEvents)) - return event - }, + const nextEvents = [...events, event] + await updateSessionHistory(input.sessionID, info(input.sessionID, raw, nextEvents)) + return event + }), ) export const unrollback = fn( @@ -212,47 +213,48 @@ export namespace SessionHistory { sessionID: Identifier.schema("session"), rollbackID: Identifier.schema("history").optional(), }), - async (input) => { - SessionManager.assertIdle(input.sessionID) - const [raw, events] = await Promise.all([ - rawMessages({ sessionID: input.sessionID }), - readEvents(input.sessionID), - ]) - const target = input.rollbackID - ? activeRollbacks(events).find((event) => event.id === input.rollbackID) - : latest(events) - if (!target) return latestInfo(input.sessionID, raw, events) - - const latestRollback = latest(events) - if (!latestRollback || latestRollback.id !== target.id) { - throw new UnrollbackConflictError({ - message: "Only the latest rollback can be restored.", - rollbackID: target.id, - }) - } + async (input) => + Storage.transaction(async () => { + SessionManager.assertIdle(input.sessionID) + const [raw, events] = await Promise.all([ + rawMessages({ sessionID: input.sessionID }), + readEvents(input.sessionID), + ]) + const target = input.rollbackID + ? activeRollbacks(events).find((event) => event.id === input.rollbackID) + : latest(events) + if (!target) return latestInfo(input.sessionID, raw, events) + + const latestRollback = latest(events) + if (!latestRollback || latestRollback.id !== target.id) { + throw new UnrollbackConflictError({ + message: "Only the latest rollback can be restored.", + rollbackID: target.id, + }) + } - if (!canUnrollback(raw, target)) { - throw new UnrollbackConflictError({ - message: "Cannot redo this rollback after new session messages have been added.", - rollbackID: target.id, - }) - } + if (!canUnrollback(raw, target)) { + throw new UnrollbackConflictError({ + message: "Cannot redo this rollback after new session messages have been added.", + rollbackID: target.id, + }) + } - const event: UnrollbackEvent = { - id: Identifier.ascending("history"), - sessionID: input.sessionID, - type: "unrollback", - time: { - created: Date.now(), - }, - rollbackID: target.id, - } - await writeEvent(event) + const event: UnrollbackEvent = { + id: Identifier.ascending("history"), + sessionID: input.sessionID, + type: "unrollback", + time: { + created: Date.now(), + }, + rollbackID: target.id, + } + await writeEvent(event) - const nextEvents = [...events, event] - await updateSessionHistory(input.sessionID, info(input.sessionID, raw, nextEvents)) - return event - }, + const nextEvents = [...events, event] + await updateSessionHistory(input.sessionID, info(input.sessionID, raw, nextEvents)) + return event + }), ) export const restoreFiles = fn( @@ -629,17 +631,17 @@ export namespace SessionHistory { async function writeEvent(event: Event) { const { SessionSummary } = await import("./summary") - SessionManager.bumpHistoryRevision(event.sessionID) const session = await SessionManager.requireSession(event.sessionID) const scopeID = asScopeID((session.scope as Scope).id) - await SessionSummary.invalidateDerivedState(event.sessionID, scopeID) await Storage.write( StoragePath.sessionHistoryEvent(scopeID, asSessionID(event.sessionID), asHistoryID(event.id)), event, ) await SessionSummary.invalidateDerivedState(event.sessionID, scopeID) - SessionManager.bumpHistoryRevision(event.sessionID) - SessionMessageCache.invalidate(event.sessionID) + Storage.afterCommit(() => { + SessionManager.bumpHistoryRevision(event.sessionID) + SessionMessageCache.invalidate(event.sessionID) + }) } async function updateSessionHistory(sessionID: string, history: Info["history"] | undefined) { diff --git a/packages/harness/src/session/inbox.ts b/packages/harness/src/session/inbox.ts index 474d2fb0e..3b23a0867 100644 --- a/packages/harness/src/session/inbox.ts +++ b/packages/harness/src/session/inbox.ts @@ -2,12 +2,10 @@ import z from "zod" import { NamedError } from "@ericsanchezok/synergy-util/error" import { Bus } from "../bus" import { BusEvent } from "../bus/bus-event" -import { GlobalBus } from "../bus/global" import { Identifier } from "../id/id" import { Scope } from "../scope" import { Storage } from "../storage/storage" import { StoragePath } from "../storage/path" -import { Context } from "../util/context" import { Lock } from "../util/lock" import { sha256Content } from "../util/crypto" import { Log } from "../util/log" @@ -246,51 +244,48 @@ export namespace SessionInbox { } async function writeItem(item: StoredItem, preserveCreated = false): Promise { - { - using lock = await Lock.write(`session-inbox-write:${item.sessionID}`) - if (!preserveCreated) { - const { SessionManager } = await import("./manager") - item.time.created = Math.max( - item.time.created, - Date.now(), - SessionManager.fenceQueuedBefore(item.sessionID) ?? 0, + return Storage.transaction(async () => { + { + if (!preserveCreated) { + const { SessionManager } = await import("./manager") + item.time.created = Math.max( + item.time.created, + Date.now(), + SessionManager.fenceQueuedBefore(item.sessionID) ?? 0, + ) + } + const session = await readSession(item.sessionID) + const scopeID = Identifier.asScopeID((session.scope as Scope).id) + await Storage.write( + StoragePath.sessionInboxItem(scopeID, Identifier.asSessionID(item.sessionID), item.id), + item, ) } - const session = await readSession(item.sessionID) - const scopeID = Identifier.asScopeID((session.scope as Scope).id) - await Storage.write(StoragePath.sessionInboxItem(scopeID, Identifier.asSessionID(item.sessionID), item.id), item) - } - await publish(item.sessionID) - return item + await publish(item.sessionID) + return item + }) } async function removeItems(sessionID: string, itemIDs: string[], notify = true): Promise { - if (itemIDs.length === 0) return - const session = await readSession(sessionID) - const scopeID = Identifier.asScopeID((session.scope as Scope).id) - await Promise.all( - itemIDs.map((id) => Storage.remove(StoragePath.sessionInboxItem(scopeID, Identifier.asSessionID(sessionID), id))), - ) - if (notify) await publish(sessionID) + return Storage.transaction(async () => { + if (itemIDs.length === 0) return + const session = await readSession(sessionID) + const scopeID = Identifier.asScopeID((session.scope as Scope).id) + await Promise.all( + itemIDs.map((id) => + Storage.remove(StoragePath.sessionInboxItem(scopeID, Identifier.asSessionID(sessionID), id)), + ), + ) + if (notify) await publish(sessionID) + }) } async function publish(sessionID: string): Promise { const items = await list(sessionID) const payload = { sessionID, items } - try { - await Bus.publish(Event.Updated, payload) - } catch (e) { - if (!(e instanceof Context.NotFound)) throw e - const session = await readSession(sessionID) - const scope = session.scope as Scope - GlobalBus.emit("event", { - directory: scope.type === "home" ? "home" : scope.directory, - payload: { - type: Event.Updated.type, - properties: payload, - }, - }) - } + const session = await readSession(sessionID) + const scope = session.scope as Scope + await ScopeContext.provide({ scope, fn: () => Bus.publish(Event.Updated, payload) }) } function summarizeParts(parts: Array<{ type: string; text?: unknown; filename?: unknown }>): Item["summary"] & { @@ -395,7 +390,7 @@ export namespace SessionInbox { : stored.filter((item) => item.time.created >= (options.createdAfter ?? 0)) if (items.some((item) => item.mode === "task" && item.status !== "failed")) return true if (options?.allowSteer === false) return false - if (!items.some((item) => item.mode === "steer")) return false + if (!items.some((item) => item.mode === "steer" && item.status !== "failed")) return false return !!(await latestRootID(sessionID)) } @@ -463,16 +458,11 @@ export namespace SessionInbox { return publicItem(await getStored(sessionID, itemID)) } - function stableDeliveryItemID(sessionID: string, deliveryKey: string): string { + export function stableDeliveryItemID(sessionID: string, deliveryKey: string): string { const hash = sha256Content(`${sessionID}:${deliveryKey}`).slice(0, 26) return `inb_${hash}` } - function legacyStableMessageID(sessionID: string, deliveryKey: string): string { - const hash = sha256Content(`${sessionID}:${deliveryKey}`).slice(0, 26) - return `msg_${hash}` - } - function deliveryItem(input: z.infer, ids: { itemID: string; messageID: string }): StoredItem { const summarized = summarizeParts(input.message.parts) const mode = input.mode @@ -537,15 +527,22 @@ export namespace SessionInbox { async function findExistingDelivery(sessionID: string, deliveryKey: string) { const itemID = stableDeliveryItemID(sessionID, deliveryKey) - const existing = await getStored(sessionID, itemID).catch(() => undefined) + const existing = await getStored(sessionID, itemID).catch((error) => { + if (error instanceof Storage.NotFoundError) return + throw error + }) if (existing) return { itemID: existing.id, messageID: existing.messageID } - const legacyMessageID = legacyStableMessageID(sessionID, deliveryKey) - const materialized = (await SessionHistory.messageInfos(sessionID)).find( - (info) => info.id === legacyMessageID || info.metadata?.inboxDeliveryKey === deliveryKey, - ) - if (!materialized) return undefined - return { itemID, messageID: materialized.id } + const session = await readSession(sessionID) + const [receipt] = await Storage.readMany<{ itemID: string; messageID: string }>([ + [ + ...StoragePath.sessionRoot(Identifier.asScopeID(session.scope.id), Identifier.asSessionID(sessionID)), + "inbox-materialized", + itemID, + ], + ]) + if (receipt) return { itemID: receipt.itemID, messageID: receipt.messageID } + return undefined } async function deliverUniqueWithPreparedMessage( @@ -723,14 +720,8 @@ export namespace SessionInbox { const itemID = stableDeliveryItemID(input.sessionID, input.deliveryKey) using _ = await Lock.write(`session-inbox-delivery:${input.sessionID}:${input.deliveryKey}`) - const existing = await getStored(input.sessionID, itemID).catch(() => undefined) - if (existing) return { itemID: existing.id, messageID: existing.messageID, created: false } - - const legacyMessageID = legacyStableMessageID(input.sessionID, input.deliveryKey) - const materialized = (await SessionHistory.messageInfos(input.sessionID)).find( - (info) => info.id === legacyMessageID || info.metadata?.inboxDeliveryKey === input.deliveryKey, - ) - if (materialized) return { itemID, messageID: materialized.id, created: false } + const existing = await findExistingDelivery(input.sessionID, input.deliveryKey) + if (existing) return { ...existing, created: false } const orderKey = Identifier.ascending("inbox") const messageID = Identifier.ascending("message") @@ -759,27 +750,27 @@ export namespace SessionInbox { * A steer item becomes task (queues for after-turn). */ export async function guide(input: { sessionID: string; itemID: string }): Promise { - const item = await assertMutable(input) - if (item.mode === "context") return publicItem(item) - if (item.status === "failed") { - // The loop deletes steer items before materialization, so guiding a - // parked failure would permanently drop its payload mid-run; only the - // retry path may re-drive it. - throw new ItemFailedError({ - message: "A failed item cannot be guided; retry delivery or delete it instead.", - sessionID: input.sessionID, - itemID: input.itemID, - }) - } - const updated: StoredItem = { - ...item, - mode: item.mode === "task" ? "steer" : "task", - time: { - ...item.time, - updated: Date.now(), - }, - } - return publicItem(await writeItem(updated, true)) + return Storage.transaction(async () => { + const item = await assertMutable(input) + if (item.mode === "context") return publicItem(item) + if (item.status === "failed") { + // Retry must reopen the failed rollout before its payload becomes runnable. + throw new ItemFailedError({ + message: "A failed item cannot be guided; retry delivery or delete it instead.", + sessionID: input.sessionID, + itemID: input.itemID, + }) + } + const updated: StoredItem = { + ...item, + mode: item.mode === "task" ? "steer" : "task", + time: { + ...item.time, + updated: Date.now(), + }, + } + return publicItem(await writeItem(updated, true)) + }) } /** @@ -790,15 +781,17 @@ export namespace SessionInbox { } async function drainWhere(sessionID: string, predicate: (item: StoredItem) => boolean): Promise { - const items = await listStored(sessionID) - const drained = items.filter(predicate) - if (drained.length === 0) return [] - await removeItems( - sessionID, - drained.map((item) => item.id), - ) - log.info("drained inbox items", { sessionID, count: drained.length }) - return drained + return Storage.transaction(async () => { + const items = await listStored(sessionID) + const drained = items.filter(predicate) + if (drained.length === 0) return [] + await removeItems( + sessionID, + drained.map((item) => item.id), + ) + log.info("drained inbox items", { sessionID, count: drained.length }) + return drained + }) } export async function drainReady(sessionID: string): Promise { @@ -827,12 +820,14 @@ export namespace SessionInbox { // --- Mode-based drains --- - export async function drainSteer(sessionID: string): Promise { - return drainWhere(sessionID, (item) => item.mode === "steer") + export async function peekSteer(sessionID: string): Promise { + return (await listStored(sessionID)).filter((item) => item.mode === "steer" && item.status !== "failed") } - export async function drainContext(sessionID: string): Promise { - return drainWhere(sessionID, (item) => item.mode === "context" && item.message?.role === "user") + export async function peekContext(sessionID: string): Promise { + return (await listStored(sessionID)).filter( + (item) => item.mode === "context" && item.message?.role === "user" && item.status !== "failed", + ) } export async function peekTask(sessionID: string): Promise { @@ -841,29 +836,31 @@ export namespace SessionInbox { } export async function fenceQueuedWork(sessionID: string, onFence: (createdBefore: number) => void): Promise { - let removed: number - { - using lock = await Lock.write(`session-inbox-write:${sessionID}`) - const { SessionManager } = await import("./manager") - const items = await listStored(sessionID) - const createdBefore = - SessionManager.fenceQueuedBefore(sessionID) ?? - Math.max(Date.now(), ...items.map((item) => item.time.created)) + 1 - onFence(createdBefore) - removed = await removeByModesUnlocked(sessionID, ["task", "steer", "context"], createdBefore) - } - if (removed > 0) await publish(sessionID) - return removed + return Storage.transaction(async () => { + let removed: number + { + const { SessionManager } = await import("./manager") + const items = await listStored(sessionID) + const createdBefore = + SessionManager.fenceQueuedBefore(sessionID) ?? + Math.max(Date.now(), ...items.map((item) => item.time.created)) + 1 + Storage.afterCommit(() => onFence(createdBefore)) + removed = await removeByModesUnlocked(sessionID, ["task", "steer", "context"], createdBefore) + } + if (removed > 0) await publish(sessionID) + return removed + }) } export async function removeByModes(sessionID: string, modes: ItemMode[], createdBefore?: number): Promise { - let removed: number - { - using lock = await Lock.write(`session-inbox-write:${sessionID}`) - removed = await removeByModesUnlocked(sessionID, modes, createdBefore) - } - if (removed > 0) await publish(sessionID) - return removed + return Storage.transaction(async () => { + let removed: number + { + removed = await removeByModesUnlocked(sessionID, modes, createdBefore) + } + if (removed > 0) await publish(sessionID) + return removed + }) } async function removeByModesUnlocked(sessionID: string, modes: ItemMode[], createdBefore?: number): Promise { @@ -884,11 +881,42 @@ export namespace SessionInbox { rootID?: string, options?: { guiding?: boolean }, ): Promise { - // Pre-allocated messageID ensures idempotent write - const existing = await MessageV2.get({ sessionID: item.sessionID, messageID: item.messageID }).catch( - () => undefined, - ) - if (existing) return existing + try { + return await materializeStoredItem(item, rootID, options) + } catch (error) { + if (item.mode !== "task" && error instanceof Attachment.InvalidUrlError) + await parkTaskFailure(item.sessionID, item, error.message) + throw error + } + } + + async function materializeStoredItem( + item: StoredItem, + rootID?: string, + options?: { guiding?: boolean }, + ): Promise { + const commitOptions: SessionUserMessageMaterialization.CommitOptions = { + commit: async () => { + const session = await readSession(item.sessionID) + const scopeID = Identifier.asScopeID(session.scope.id) + const sid = Identifier.asSessionID(item.sessionID) + await Storage.write([...StoragePath.sessionRoot(scopeID, sid), "inbox-materialized", item.id], { + itemID: item.id, + messageID: item.messageID, + deliveryKey: item.deliveryKey, + completedAt: Date.now(), + }) + await removeItems(item.sessionID, [item.id]) + }, + } + const existing = await MessageV2.get({ sessionID: item.sessionID, messageID: item.messageID }).catch((error) => { + if (error instanceof Storage.NotFoundError) return + throw error + }) + if (existing) { + await Storage.transaction(() => commitOptions.commit!(existing)) + return existing + } const payload = item.message if (!payload) return undefined @@ -918,6 +946,7 @@ export namespace SessionInbox { noReply: item.mode === "task" ? item.input.noReply : true, }, rootID, + commitOptions, ) } @@ -960,7 +989,7 @@ export namespace SessionInbox { ...(payload.tools ? { tools: payload.tools } : {}), ...(variant ? { variant } : {}), } - return SessionUserMessageMaterialization.write({ info, parts }) + return SessionUserMessageMaterialization.write({ info, parts }, commitOptions) } // Assistant messages @@ -991,10 +1020,7 @@ export namespace SessionInbox { } : {}), } - await Session.updateMessage(info) - for (const part of parts) { - await Session.updatePart(part) - } + const result = await SessionUserMessageMaterialization.write({ info, parts }, commitOptions) await SessionContextContributions.onAssistantComplete(info) await Plugin.trigger( "session.turn.after", @@ -1037,7 +1063,6 @@ export namespace SessionInbox { await parkTaskFailure(sessionID, task, error.message) return { status: "failed", itemID: task.id, reason: error.message } } - await commitReady(sessionID, [task.id]) return { status: "materialized", itemID: task.id, messageID: task.messageID } } diff --git a/packages/harness/src/session/index.ts b/packages/harness/src/session/index.ts index f064e083a..0ef3a360d 100644 --- a/packages/harness/src/session/index.ts +++ b/packages/harness/src/session/index.ts @@ -1,3 +1,5 @@ +import type { StoreTransaction } from "../storage/transactional-store" +import { SessionStaging } from "./staging" import { RolloutAttachment } from "./rollout/attachment" import { RolloutContext } from "./rollout/context" import { SnapshotLifecycle } from "./snapshot-lifecycle" @@ -115,6 +117,63 @@ export namespace Session { } } + export async function rebuildStorageIndexes(tx: StoreTransaction) { + for (const root of [ + "session_index", + "endpoint_session", + "sessions_page_index", + "session_child_index", + "session_nav_v2", + ]) + await tx.removeTree([root]) + const scopes = await tx.scan(["sessions"]) + for (const scopeID of scopes) { + const page: PageIndex = { entries: [] } + const children = new Map() + const nav: SessionNavEntry[] = [] + let after: string[] | undefined + for (;;) { + const batch = await tx.query({ kind: "session", scopeID, after, limit: 128 }) + if (!batch.length) break + for (const record of batch) { + const session = record.value + const index = toIndex(session) + if (index.scopeID !== scopeID || session.id !== record.key[2]) + throw new Error("Session identity does not match its storage owner") + await tx.write(["session_index", session.id], index) + if (session.endpoint) + await tx.write( + StoragePath.endpointSession(SessionEndpoint.toKey(session.endpoint), asSessionID(session.id)), + { sessionID: session.id, scopeID }, + ) + page.entries.push(toPageIndexEntry(session)) + nav.push(toNavEntry(session)) + if (session.parentID) { + const child = children.get(session.parentID) ?? { + version: 1, + scopeID, + parentID: session.parentID, + updatedAt: Date.now(), + entries: [], + } + child.entries.push(toChildIndexEntry(session)) + children.set(session.parentID, child) + } + } + after = batch.at(-1)!.key + } + page.entries.sort((a, b) => b.updated - a.updated || b.id.localeCompare(a.id)) + nav.sort((a, b) => b.lastActivityAt - a.lastActivityAt || b.id.localeCompare(a.id)) + await tx.write(["sessions_page_index", scopeID], page) + await tx.write(["session_nav_v2", scopeID], { version: 1, scopeID, updatedAt: Date.now(), entries: nav }) + for (const [parentID, child] of children) { + sortChildIndexEntries(child.entries) + await tx.write(["session_child_index", scopeID, parentID], child) + } + } + await tx.remove(StoragePath.rolloutRecoveryPending()) + } + export function withoutRuntimeInfo(session: Info): Info { const { working: _working, ...rest } = session return rest @@ -186,7 +245,10 @@ export namespace Session { export type WorkspaceSelection = z.infer export async function readPageIndex(scopeID: string): Promise { - return Storage.read(StoragePath.sessionsPageIndex(asScopeID(scopeID))).catch(() => ({ entries: [] })) + return Storage.read(StoragePath.sessionsPageIndex(asScopeID(scopeID))).catch((error) => { + if (error instanceof Storage.NotFoundError) return { entries: [] } + throw error + }) } export async function writePageIndex(scopeID: string, index: PageIndex) { @@ -194,21 +256,23 @@ export namespace Session { } export async function upsertPageIndexEntry(scopeID: string, entry: PageIndex["entries"][number]) { - using _ = await Lock.write(`session-page-index:${scopeID}`) - const index = await readPageIndex(scopeID) - const existing = index.entries.findIndex((e) => e.id === entry.id) - if (existing >= 0) index.entries.splice(existing, 1) - const insertAt = index.entries.findIndex((e) => e.updated <= entry.updated) - if (insertAt === -1) index.entries.push(entry) - else index.entries.splice(insertAt, 0, entry) - await writePageIndex(scopeID, index) + return Storage.transaction(async () => { + const index = await readPageIndex(scopeID) + const existing = index.entries.findIndex((e) => e.id === entry.id) + if (existing >= 0) index.entries.splice(existing, 1) + const insertAt = index.entries.findIndex((e) => e.updated <= entry.updated) + if (insertAt === -1) index.entries.push(entry) + else index.entries.splice(insertAt, 0, entry) + await writePageIndex(scopeID, index) + }) } export async function removePageIndexEntry(scopeID: string, sessionID: string) { - using _ = await Lock.write(`session-page-index:${scopeID}`) - const index = await readPageIndex(scopeID) - index.entries = index.entries.filter((e) => e.id !== sessionID) - await writePageIndex(scopeID, index) + return Storage.transaction(async () => { + const index = await readPageIndex(scopeID) + index.entries = index.entries.filter((e) => e.id !== sessionID) + await writePageIndex(scopeID, index) + }) } function toPageIndexEntry(session: Info): PageIndex["entries"][number] { @@ -238,13 +302,10 @@ export namespace Session { export async function readChildIndex(scopeID: string, parentID: string): Promise { return Storage.read(StoragePath.sessionChildIndex(asScopeID(scopeID), asSessionID(parentID))).catch( - () => ({ - version: 1, - scopeID, - parentID, - updatedAt: 0, - entries: [], - }), + (error) => { + if (error instanceof Storage.NotFoundError) return { version: 1, scopeID, parentID, updatedAt: 0, entries: [] } + throw error + }, ) } @@ -257,25 +318,26 @@ export namespace Session { } export async function upsertChildIndexEntry(scopeID: string, parentID: string, entry: ChildIndexEntry) { - using _ = await Lock.write(`session-child-index:${scopeID}:${parentID}`) - const index = await readChildIndex(scopeID, parentID) - const existing = index.entries.findIndex((e) => e.id === entry.id) - if (existing >= 0) index.entries.splice(existing, 1) - index.entries.push(entry) - await writeChildIndex(scopeID, parentID, index) + return Storage.transaction(async () => { + const index = await readChildIndex(scopeID, parentID) + const existing = index.entries.findIndex((e) => e.id === entry.id) + if (existing >= 0) index.entries.splice(existing, 1) + index.entries.push(entry) + await writeChildIndex(scopeID, parentID, index) + }) } export async function removeChildIndexEntry(scopeID: string, parentID: string, sessionID: string) { - using _ = await Lock.write(`session-child-index:${scopeID}:${parentID}`) - const index = await readChildIndex(scopeID, parentID) - const nextEntries = index.entries.filter((e) => e.id !== sessionID) - if (nextEntries.length === index.entries.length) return - index.entries = nextEntries - await writeChildIndex(scopeID, parentID, index) + return Storage.transaction(async () => { + const index = await readChildIndex(scopeID, parentID) + const nextEntries = index.entries.filter((e) => e.id !== sessionID) + if (nextEntries.length === index.entries.length) return + index.entries = nextEntries + await writeChildIndex(scopeID, parentID, index) + }) } export async function removeChildIndex(scopeID: string, parentID: string) { - using _ = await Lock.write(`session-child-index:${scopeID}:${parentID}`) await Storage.remove(StoragePath.sessionChildIndex(asScopeID(scopeID), asSessionID(parentID))) } @@ -334,7 +396,7 @@ export namespace Session { if (!session.endpoint) return const endpointKey = SessionEndpoint.toKey(session.endpoint) - await Storage.remove(StoragePath.endpointSession(endpointKey, asSessionID(session.id))).catch(() => undefined) + await Storage.remove(StoragePath.endpointSession(endpointKey, asSessionID(session.id))) } export async function withRuntimeInfo(session: Info): Promise { @@ -387,8 +449,12 @@ export namespace Session { ) { return } - if (info.time.archived) lastPublish.delete(session.id) - else lastPublish.set(session.id, { key, at: now }) + const remember = () => { + if (info.time.archived) lastPublish.delete(session.id) + else lastPublish.set(session.id, { key, at: now }) + } + if (Storage.inTransaction()) Storage.afterCommit(remember) + else remember() Bus.publish(event, { info, navEntry }) } @@ -469,22 +535,28 @@ export namespace Session { } log.info("created", result) - await Storage.write( - StoragePath.sessionInfo(asScopeID(scope.id), asSessionID(result.id)), - withoutRuntimeInfo(result), - ) - await Storage.write(StoragePath.sessionIndex(asSessionID(result.id)), toIndex(result)) - await writeEndpointIndex(result) - await upsertPageIndexEntry(scope.id, toPageIndexEntry(result)) - if (result.parentID) await upsertChildIndexEntry(scope.id, result.parentID, toChildIndexEntry(result)) - const navEntry = await SessionNav.upsertNavEntry(toNavEntry(result)) + await Storage.transaction(async () => { + if (result.parentID && !(await SessionManager.getSession(result.parentID))) + throw new Storage.NotFoundError({ message: "Parent Session no longer exists" }) + await Storage.write( + StoragePath.sessionInfo(asScopeID(scope.id), asSessionID(result.id)), + withoutRuntimeInfo(result), + ) + await Storage.write(StoragePath.sessionIndex(asSessionID(result.id)), toIndex(result)) + await writeEndpointIndex(result) + await upsertPageIndexEntry(scope.id, toPageIndexEntry(result)) + if (result.parentID) await upsertChildIndexEntry(scope.id, result.parentID, toChildIndexEntry(result)) + const navEntry = await SessionNav.upsertNavEntry(toNavEntry(result)) - await SessionSchemaRegistry.created(result) + await SessionSchemaRegistry.created(result) - SessionManager.registerRuntime(result.id) - Scope.touch(scope.id) + Storage.afterCommit(() => { + SessionManager.registerRuntime(result.id) + }) + await Scope.touch(scope.id) - await publishInfo(SessionEvent.Updated, result, navEntry) + await publishInfo(SessionEvent.Updated, result, navEntry) + }) return withRuntimeInfo(result) } @@ -552,7 +624,9 @@ export namespace Session { message: "The fork point message is no longer part of the effective session history.", }) } - let session = await create({ + const sessionID = Identifier.descending("session") + const createInput = { + id: sessionID, scope: source.scope as Scope, workspace: source.workspace, title: input.title, @@ -562,35 +636,39 @@ export namespace Session { messageID: forkPoint, title: source.title, }, - }) + } const selected = forkPoint ? msgs.slice(0, msgs.findIndex((msg) => msg.info.id === forkPoint) + (includeForkPoint ? 1 : 0)) : msgs + const stagingID = await SessionStaging.begin(source.scope.id, [sessionID]) + let session: Info | undefined try { await SnapshotLifecycle.adopt({ scopeID: source.scope.id, sourceSessionID: source.id, - targetSessionID: session.id, + targetSessionID: sessionID, workspace: source.workspace?.path ?? ScopeContext.current.directory, hashes: selected.flatMap((msg) => msg.parts.flatMap(SnapshotRecords.partRoots)), }) + const prepared: MessageV2.WithParts[] = [] const messageMap = new Map() for (const msg of selected) { const id = Identifier.ascending("message") messageMap.set(msg.info.id, id) - const cloned = await updateMessage({ + const cloned: MessageV2.Info = { ...msg.info, ...(msg.info.role === "assistant" ? { accounting: MessageV2.copyAccounting(msg.info, "inherited") } : {}), - sessionID: session.id, + sessionID, id, ...("parentID" in msg.info && typeof msg.info.parentID === "string" ? { parentID: messageMap.get(msg.info.parentID) ?? msg.info.parentID } : {}), - }) + } + const parts: MessageV2.Part[] = [] for (const part of msg.parts) { const from = { kind: "session" as const, scopeID: source.scope.id, sessionID: source.id } - const to = { ...from, sessionID: session.id } + const to = { ...from, sessionID } const state = part.type === "tool" && part.state.status === "completed" ? { ...part.state } : undefined if (state?.outputArtifact) state.outputArtifact = await RolloutArtifact.copy(from, to, state.outputArtifact) if (state?.attachments) { @@ -608,20 +686,36 @@ export namespace Session { part.type === "attachment" && part.artifact ? await RolloutArtifact.copy(from, to, part.artifact) : undefined - await updatePart({ - ...part, - ...(artifact ? { artifact } : {}), - ...(state ? { state } : {}), - id: Identifier.ascending("part"), - messageID: cloned.id, - sessionID: session.id, - }) + parts.push( + await preparePart( + { + ...part, + ...(artifact ? { artifact } : {}), + ...(state ? { state } : {}), + id: Identifier.ascending("part"), + messageID: cloned.id, + sessionID, + }, + source.scope.id, + ), + ) } + prepared.push({ info: cloned, parts }) } + session = await Storage.transaction(async () => { + const created = await create(createInput) + for (const message of prepared) { + await updateMessage(message.info) + for (const part of message.parts) await updatePart(part) + } + await SessionStaging.finish(stagingID) + return created + }) session = await applyWorkspaceSelection(session.id, input.workspace) } catch (error) { - await remove(session.id) + if (session) await remove(session.id) + else await SessionStaging.discard(stagingID) throw error } return session @@ -763,37 +857,39 @@ export namespace Session { } export async function acknowledgeRollback(id: string, rollbackID: string): Promise { - const session = await SessionManager.requireSession(id) - const scope = session.scope as Scope - const scopeID = asScopeID(scope.id) - const sessionID = asSessionID(id) - using _ = await SessionMutation.write(scopeID, sessionID) - const currentRollbackID = (await SessionHistory.storedInfo(id))?.rollback?.id - if (currentRollbackID !== rollbackID) { - throw new RollbackAckConflictError({ - message: currentRollbackID - ? "Only the current rollback can be acknowledged." - : "No active rollback can be acknowledged.", - rollbackID, - currentRollbackID, - }) - } + return Storage.transaction(async () => { + const session = await SessionManager.requireSession(id) + const scope = session.scope as Scope + const scopeID = asScopeID(scope.id) + const sessionID = asSessionID(id) - let changed = false - const result = await Storage.update(StoragePath.sessionInfo(scopeID, sessionID), (draft) => { - if (draft.rollbackAck?.rollbackID === rollbackID) return - draft.rollbackAck = { rollbackID, acknowledgedAt: Date.now() } - changed = true - }) - const rollbackAck = result.rollbackAck - if (!rollbackAck) { - throw new RollbackAckConflictError({ - message: "No active rollback can be acknowledged.", - rollbackID, + const currentRollbackID = (await SessionHistory.storedInfo(id))?.rollback?.id + if (currentRollbackID !== rollbackID) { + throw new RollbackAckConflictError({ + message: currentRollbackID + ? "Only the current rollback can be acknowledged." + : "No active rollback can be acknowledged.", + rollbackID, + currentRollbackID, + }) + } + + let changed = false + const result = await Storage.update(StoragePath.sessionInfo(scopeID, sessionID), (draft) => { + if (draft.rollbackAck?.rollbackID === rollbackID) return + draft.rollbackAck = { rollbackID, acknowledgedAt: Date.now() } + changed = true }) - } - if (changed) await publishInfo(SessionEvent.Updated, result) - return rollbackAck + const rollbackAck = result.rollbackAck + if (!rollbackAck) { + throw new RollbackAckConflictError({ + message: "No active rollback can be acknowledged.", + rollbackID, + }) + } + if (changed) await publishInfo(SessionEvent.Updated, result) + return rollbackAck + }) } async function acknowledgeCompletionNoticeResult( @@ -801,31 +897,33 @@ export namespace Session { acknowledgedCount: number, options?: { repairNavOnNoop?: boolean }, ) { - if (!Number.isSafeInteger(acknowledgedCount) || acknowledgedCount < 0) { - throw new TypeError("acknowledgedCount must be a non-negative safe integer") - } - - return serializeCompletionNoticeMutation(id, async () => { - const session = await SessionManager.requireSession(id) - const scope = session.scope as Scope - const scopeID = asScopeID(scope.id) - const sessionID = asSessionID(id) - using _ = await SessionMutation.write(scopeID, sessionID) - let actualAcknowledgedCount = 0 - const result = await Storage.update(StoragePath.sessionInfo(scopeID, sessionID), (draft) => { - const current = draft.completionNotice.unreadCount ?? (draft.completionNotice.unread ? 1 : 0) - const next = Math.max(0, current - acknowledgedCount) - actualAcknowledgedCount = current - next - draft.completionNotice.unread = next > 0 - draft.completionNotice.unreadCount = next - }) - if (actualAcknowledgedCount === 0 && !options?.repairNavOnNoop) { - return { info: await withRuntimeInfo(result), acknowledgedCount: 0 } + return Storage.transaction(async () => { + if (!Number.isSafeInteger(acknowledgedCount) || acknowledgedCount < 0) { + throw new TypeError("acknowledgedCount must be a non-negative safe integer") } - const navEntry = await SessionNav.upsertNavEntry(toNavEntry(result)) - await publishInfo(SessionEvent.Updated, result, navEntry) - return { info: await withRuntimeInfo(result), acknowledgedCount: actualAcknowledgedCount } + return serializeCompletionNoticeMutation(id, async () => { + const session = await SessionManager.requireSession(id) + const scope = session.scope as Scope + const scopeID = asScopeID(scope.id) + const sessionID = asSessionID(id) + + let actualAcknowledgedCount = 0 + const result = await Storage.update(StoragePath.sessionInfo(scopeID, sessionID), (draft) => { + const current = draft.completionNotice.unreadCount ?? (draft.completionNotice.unread ? 1 : 0) + const next = Math.max(0, current - acknowledgedCount) + actualAcknowledgedCount = current - next + draft.completionNotice.unread = next > 0 + draft.completionNotice.unreadCount = next + }) + if (actualAcknowledgedCount === 0 && !options?.repairNavOnNoop) { + return { info: await withRuntimeInfo(result), acknowledgedCount: 0 } + } + + const navEntry = await SessionNav.upsertNavEntry(toNavEntry(result)) + await publishInfo(SessionEvent.Updated, result, navEntry) + return { info: await withRuntimeInfo(result), acknowledgedCount: actualAcknowledgedCount } + }) }) } @@ -880,49 +978,49 @@ export namespace Session { editor: (session: Info) => void, options?: { preserveActivityAt?: boolean; forcePublish?: boolean }, ) { - const session = await SessionManager.requireSession(id) - const scope = session.scope as Scope - const scopeID = asScopeID(scope.id) - const sessionID = asSessionID(id) - using _ = await SessionMutation.write(scopeID, sessionID) - let before: Info | undefined - const result = await Storage.update(StoragePath.sessionInfo(scopeID, sessionID), (draft) => { - before = structuredClone(draft) - editor(draft) - draft.time.updated = Date.now() - }) - if (!before) throw new Error(`Session ${id} was not available before mutation`) - - await Storage.write(StoragePath.sessionIndex(asSessionID(result.id)), toIndex(result)) - await upsertPageIndexEntry(scope.id, toPageIndexEntry(result)) - if (before.parentID && before.parentID !== result.parentID) { - await removeChildIndexEntry(scope.id, before.parentID, result.id) - } - if (result.parentID) { - await upsertChildIndexEntry(scope.id, result.parentID, toChildIndexEntry(result)) - } - const shouldPreserveActivityAt = - options?.preserveActivityAt ?? (before.pendingReply === true && result.pendingReply === true) - const navEntry = await SessionNav.upsertNavEntry(toNavEntry(result), { - preserveActivityAt: shouldPreserveActivityAt, - }) + return Storage.transaction(async () => { + const session = await SessionManager.requireSession(id) + const scope = session.scope as Scope + const scopeID = asScopeID(scope.id) + const sessionID = asSessionID(id) - const beforeKey = before.endpoint ? SessionEndpoint.toKey(before.endpoint) : undefined - const afterKey = result.endpoint ? SessionEndpoint.toKey(result.endpoint) : undefined - if (beforeKey && beforeKey !== afterKey) { - await removeEndpointIndex(before) - } - if (result.endpoint) { - await writeEndpointIndex(result) - } + let before: Info | undefined + const result = await Storage.update(StoragePath.sessionInfo(scopeID, sessionID), (draft) => { + before = structuredClone(draft) + editor(draft) + draft.time.updated = Date.now() + }) + if (!before) throw new Error(`Session ${id} was not available before mutation`) - if (!before.time.archived && result.time.archived) { - await SessionProjectHealth.detachWorktreeSession(result.id).catch((error) => { - log.warn("failed to detach worktree during session archive", { sessionID: result.id, error }) + await Storage.write(StoragePath.sessionIndex(asSessionID(result.id)), toIndex(result)) + await upsertPageIndexEntry(scope.id, toPageIndexEntry(result)) + if (before.parentID && before.parentID !== result.parentID) { + await removeChildIndexEntry(scope.id, before.parentID, result.id) + } + if (result.parentID) { + await upsertChildIndexEntry(scope.id, result.parentID, toChildIndexEntry(result)) + } + const shouldPreserveActivityAt = + options?.preserveActivityAt ?? (before.pendingReply === true && result.pendingReply === true) + const navEntry = await SessionNav.upsertNavEntry(toNavEntry(result), { + preserveActivityAt: shouldPreserveActivityAt, }) - } - await publishInfo(SessionEvent.Updated, result, navEntry, { force: options?.forcePublish }) - return withRuntimeInfo(result) + + const beforeKey = before.endpoint ? SessionEndpoint.toKey(before.endpoint) : undefined + const afterKey = result.endpoint ? SessionEndpoint.toKey(result.endpoint) : undefined + if (beforeKey && beforeKey !== afterKey) { + await removeEndpointIndex(before) + } + if (result.endpoint) { + await writeEndpointIndex(result) + } + + if (!before.time.archived && result.time.archived) { + Storage.afterCommit(() => SessionProjectHealth.detachWorktreeSession(result.id)) + } + await publishInfo(SessionEvent.Updated, result, navEntry, { force: options?.forcePublish }) + return withRuntimeInfo(result) + }) } export async function update(id: string, editor: (session: Info) => void) { @@ -1095,75 +1193,80 @@ export namespace Session { return page.items }) - async function removeInternal(sessionID: string, removed: Info[]): Promise { - try { - const indexed = await SessionManager.requireSession(sessionID) - const scope = indexed.scope as Scope - const scopeID = asScopeID(scope.id) - const canonicalSessionID = asSessionID(sessionID) - using _ = await SessionMutation.write(scopeID, canonicalSessionID) - const session = await Storage.read(StoragePath.sessionInfo(scopeID, canonicalSessionID)) - for (const child of await children(sessionID)) { - await removeInternal(child.id, removed) - } - await SnapshotLifecycle.beginDelete(scope.id, sessionID) - await SessionProjectHealth.detachWorktreeSession(sessionID).catch((error) => { - log.warn("failed to detach worktree during session removal", { sessionID, error }) - }) - SessionManager.unregisterRuntime(sessionID) - SessionManager.forgetSession(sessionID) - SessionMessageCache.disable(sessionID) - await removeEndpointIndex(session) - await MessageV2.removeOrderIndex(scopeID, canonicalSessionID) - await SessionNav.removeNavEntry(scope.id, sessionID) - await Storage.removeTree(StoragePath.sessionRoot(scopeID, canonicalSessionID)) - await Storage.remove(StoragePath.sessionIndex(canonicalSessionID)) - await removePageIndexEntry(scope.id, sessionID) - if (session.parentID) await removeChildIndexEntry(scope.id, session.parentID, sessionID) - await SessionSearchIndex.removeRecords(scopeID, canonicalSessionID) - await removeChildIndex(scope.id, sessionID) - await SnapshotLifecycle.completeDelete(scope.id, sessionID) - removed.push(session) - } catch (e) { - log.error(e) - } - } - export const remove = fn(Identifier.schema("session"), async (sessionID) => { + const pending = [sessionID] + const drained = new Set() + while (pending.length) { + const id = pending.pop()! + if (drained.has(id)) continue + drained.add(id) + if (!(await SessionManager.getSession(id))) continue + await flushPartWrites(id) + for (const child of await children(id)) pending.push(child.id) + } const removed: Info[] = [] - await removeInternal(sessionID, removed) - try { - for (const info of removed) { - await Bus.publish(SessionEvent.Deleted, { info }) + await Storage.transaction(async () => { + const visiting = new Set() + async function removeTree(id: string) { + if (visiting.has(id)) throw new Error("Session ancestry contains a cycle") + visiting.add(id) + const session = await SessionManager.getSession(id) + if (!session) return + const scope = session.scope as Scope + const scopeID = asScopeID(scope.id) + const sid = asSessionID(id) + for (const child of await children(id)) await removeTree(child.id) + await SnapshotLifecycle.scheduleDelete(scope.id, id) + await removeEndpointIndex(session) + await MessageV2.removeOrderIndex(scopeID, sid) + await SessionNav.removeNavEntry(scope.id, id) + await Storage.removeTree(StoragePath.sessionRoot(scopeID, sid)) + await Storage.remove(StoragePath.sessionIndex(sid)) + await removePageIndexEntry(scope.id, id) + if (session.parentID) await removeChildIndexEntry(scope.id, session.parentID, id) + await SessionSearchIndex.removeRecords(scopeID, sid) + await removeChildIndex(scope.id, id) + Storage.afterCommit(() => { + SessionManager.unregisterRuntime(id) + SessionManager.forgetSession(id) + SessionMessageCache.disable(id) + }) + await ScopeContext.provide({ scope, fn: () => Bus.publish(SessionEvent.Deleted, { info: session }) }) + removed.push(session) } - } catch (e) { - log.error(e) + await removeTree(sessionID) + }) + for (const session of removed) { + await SessionProjectHealth.detachWorktreeSession(session.id) + await SnapshotLifecycle.completeDelete(session.scope.id, session.id) } }) export async function updateLastExchange(sessionID: string) { - const session = await SessionManager.requireSession(sessionID) - const scopeID = asScopeID((session.scope as Scope).id) - const lastExchange: NonNullable = {} - const msgs = await SessionHistory.modelMessages({ sessionID }) - for (let i = msgs.length - 1; i >= 0; i--) { - const msg = msgs[i] - if (!lastExchange.assistant && msg.info.role === "assistant") { - const text = MessageV2.extractText(msg.parts, { maxLength: 200 }) - if (text) lastExchange.assistant = text - } - if (!lastExchange.user && msg.info.role === "user") { - const text = MessageV2.extractText(msg.parts, { maxLength: 200 }) - if (text) lastExchange.user = text + return Storage.transaction(async () => { + const session = await SessionManager.requireSession(sessionID) + const scopeID = asScopeID((session.scope as Scope).id) + const lastExchange: NonNullable = {} + const msgs = await SessionHistory.modelMessages({ sessionID }) + for (let i = msgs.length - 1; i >= 0; i--) { + const msg = msgs[i] + if (!lastExchange.assistant && msg.info.role === "assistant") { + const text = MessageV2.extractText(msg.parts, { maxLength: 200 }) + if (text) lastExchange.assistant = text + } + if (!lastExchange.user && msg.info.role === "user") { + const text = MessageV2.extractText(msg.parts, { maxLength: 200 }) + if (text) lastExchange.user = text + } + if (lastExchange.user && lastExchange.assistant) break } - if (lastExchange.user && lastExchange.assistant) break - } - // Write lastExchange directly without bumping time.updated or republishing, - // since the caller (processor) already performs a proper Session.update(). - using _ = await SessionMutation.write(scopeID, asSessionID(sessionID)) - const infoPath = StoragePath.sessionInfo(scopeID, asSessionID(sessionID)) - await Storage.update(infoPath, (draft) => { - draft.lastExchange = lastExchange + // Write lastExchange directly without bumping time.updated or republishing, + // since the caller (processor) already performs a proper Session.update(). + + const infoPath = StoragePath.sessionInfo(scopeID, asSessionID(sessionID)) + await Storage.update(infoPath, (draft) => { + draft.lastExchange = lastExchange + }) }) } @@ -1192,49 +1295,53 @@ export namespace Session { } } - export const updateMessage = fn(MessageV2.Info, async (msg) => { - const canonical = MessageV2.canonicalMessage(msg) - const session = await SessionManager.requireSession(msg.sessionID) - const scopeID = asScopeID((session.scope as Scope).id) - // Invalidate the search index BEFORE the content write so a crash between - // the write and the post-write mark can never leave a clean-but-stale - // record trusted; the post-write mark below refreshes the marker for any - // scan that races this write (commitRebuild only clears markers older - // than its scan start). - await SessionSearchIndex.markDirty(scopeID, asSessionID(canonical.sessionID)) - await MessageV2.writeInfo({ scopeID, info: canonical }) - SessionMessageCache.upsertMessage(canonical.sessionID, canonical) - // Flip the rollback projection before publishing the replacement root: the - // frontend prefix-cut hides everything after the cut while canUnrollback is - // true, so the new branch must never arrive ahead of its invalidation. - await publishRollbackInvalidation(canonical, session.history) - Bus.publish(MessageV2.Event.Updated, { - info: canonical, - }) - await SessionSearchIndex.markDirty(scopeID, asSessionID(canonical.sessionID)) - return canonical - }) + export const updateMessage = fn(MessageV2.Info, async (msg) => + Storage.transaction(async () => { + const canonical = MessageV2.canonicalMessage(msg) + const session = await SessionManager.requireSession(msg.sessionID) + const scopeID = asScopeID((session.scope as Scope).id) + // Invalidate the search index BEFORE the content write so a crash between + // the write and the post-write mark can never leave a clean-but-stale + // record trusted; the post-write mark below refreshes the marker for any + // scan that races this write (commitRebuild only clears markers older + // than its scan start). + await SessionSearchIndex.markDirty(scopeID, asSessionID(canonical.sessionID)) + await MessageV2.writeInfo({ scopeID, info: canonical }) + SessionMessageCache.upsertMessage(canonical.sessionID, canonical) + // Flip the rollback projection before publishing the replacement root: the + // frontend prefix-cut hides everything after the cut while canUnrollback is + // true, so the new branch must never arrive ahead of its invalidation. + await publishRollbackInvalidation(canonical, session.history) + Bus.publish(MessageV2.Event.Updated, { + info: canonical, + }) + await SessionSearchIndex.markDirty(scopeID, asSessionID(canonical.sessionID)) + return canonical + }), + ) export async function updateAssistantContextUsage(input: { sessionID: string messageID: string contextUsage: NonNullable }) { - const session = await SessionManager.requireSession(input.sessionID) - const scopeID = asScopeID((session.scope as Scope).id) - const result = await Storage.update( - StoragePath.messageInfo(scopeID, asSessionID(input.sessionID), asMessageID(input.messageID)), - (draft) => { - if (draft.role !== "assistant") throw new Error("Context Usage can only be attached to assistant messages") - draft.contextUsage = input.contextUsage - }, - ) - const canonical = MessageV2.canonicalMessage(result) - SessionMessageCache.upsertMessage(canonical.sessionID, canonical) - Bus.publish(MessageV2.Event.Updated, { - info: canonical, + return Storage.transaction(async () => { + const session = await SessionManager.requireSession(input.sessionID) + const scopeID = asScopeID((session.scope as Scope).id) + const result = await Storage.update( + StoragePath.messageInfo(scopeID, asSessionID(input.sessionID), asMessageID(input.messageID)), + (draft) => { + if (draft.role !== "assistant") throw new Error("Context Usage can only be attached to assistant messages") + draft.contextUsage = input.contextUsage + }, + ) + const canonical = MessageV2.canonicalMessage(result) + SessionMessageCache.upsertMessage(canonical.sessionID, canonical) + Bus.publish(MessageV2.Event.Updated, { + info: canonical, + }) + return canonical as MessageV2.Assistant }) - return canonical as MessageV2.Assistant } export const mergeMessageMetadata = fn( @@ -1244,22 +1351,24 @@ export namespace Session { metadata: z.record(z.string(), z.any()), }), async (input) => { - const session = await SessionManager.requireSession(input.sessionID) - const scopeID = asScopeID((session.scope as Scope).id) - const result = await Storage.update( - StoragePath.messageInfo(scopeID, asSessionID(input.sessionID), asMessageID(input.messageID)), - (draft) => { - draft.metadata = { - ...draft.metadata, - ...input.metadata, - } - }, - ) - SessionMessageCache.upsertMessage(result.sessionID, result) - Bus.publish(MessageV2.Event.Updated, { - info: result, + return Storage.transaction(async () => { + const session = await SessionManager.requireSession(input.sessionID) + const scopeID = asScopeID((session.scope as Scope).id) + const result = await Storage.update( + StoragePath.messageInfo(scopeID, asSessionID(input.sessionID), asMessageID(input.messageID)), + (draft) => { + draft.metadata = { + ...draft.metadata, + ...input.metadata, + } + }, + ) + SessionMessageCache.upsertMessage(result.sessionID, result) + Bus.publish(MessageV2.Event.Updated, { + info: result, + }) + return result }) - return result }, ) @@ -1269,23 +1378,26 @@ export namespace Session { messageID: Identifier.schema("message"), }), async (input) => { - const session = await SessionManager.requireSession(input.sessionID) - const scopeID = asScopeID((session.scope as Scope).id) - // See updateMessage: invalidate before the content write, refresh after. - await SessionSearchIndex.markDirty(scopeID, asSessionID(input.sessionID)) - await MessageV2.removeInfo({ - scopeID, - sessionID: asSessionID(input.sessionID), - messageID: asMessageID(input.messageID), - }) - // Structural change: drop the cache and let the next read repopulate. - SessionMessageCache.invalidate(input.sessionID) - Bus.publish(MessageV2.Event.Removed, { - sessionID: input.sessionID, - messageID: input.messageID, + await flushPartWrites(input.sessionID) + return Storage.transaction(async () => { + const session = await SessionManager.requireSession(input.sessionID) + const scopeID = asScopeID((session.scope as Scope).id) + // See updateMessage: invalidate before the content write, refresh after. + await SessionSearchIndex.markDirty(scopeID, asSessionID(input.sessionID)) + await MessageV2.removeInfo({ + scopeID, + sessionID: asSessionID(input.sessionID), + messageID: asMessageID(input.messageID), + }) + // Structural change: drop the cache and let the next read repopulate. + SessionMessageCache.invalidate(input.sessionID) + Bus.publish(MessageV2.Event.Removed, { + sessionID: input.sessionID, + messageID: input.messageID, + }) + await SessionSearchIndex.markDirty(scopeID, asSessionID(input.sessionID)) + return input.messageID }) - await SessionSearchIndex.markDirty(scopeID, asSessionID(input.sessionID)) - return input.messageID }, ) @@ -1296,26 +1408,29 @@ export namespace Session { partID: Identifier.schema("part"), }), async (input) => { - const session = await SessionManager.requireSession(input.sessionID) - const scopeID = asScopeID((session.scope as Scope).id) - // See updateMessage: invalidate before the content write, refresh after. - await SessionSearchIndex.markDirty(scopeID, asSessionID(input.sessionID)) - await Storage.remove( - StoragePath.messagePart( - scopeID, - asSessionID(input.sessionID), - asMessageID(input.messageID), - asPartID(input.partID), - ), - ) - SessionMessageCache.invalidate(input.sessionID) - Bus.publish(MessageV2.Event.PartRemoved, { - sessionID: input.sessionID, - messageID: input.messageID, - partID: input.partID, + await flushPartWrites(input.sessionID) + return Storage.transaction(async () => { + const session = await SessionManager.requireSession(input.sessionID) + const scopeID = asScopeID((session.scope as Scope).id) + // See updateMessage: invalidate before the content write, refresh after. + await SessionSearchIndex.markDirty(scopeID, asSessionID(input.sessionID)) + await Storage.remove( + StoragePath.messagePart( + scopeID, + asSessionID(input.sessionID), + asMessageID(input.messageID), + asPartID(input.partID), + ), + ) + SessionMessageCache.invalidate(input.sessionID) + Bus.publish(MessageV2.Event.PartRemoved, { + sessionID: input.sessionID, + messageID: input.messageID, + partID: input.partID, + }) + await SessionSearchIndex.markDirty(scopeID, asSessionID(input.sessionID)) + return input.partID }) - await SessionSearchIndex.markDirty(scopeID, asSessionID(input.sessionID)) - return input.partID }, ) @@ -1338,9 +1453,23 @@ export namespace Session { // Part files are the highest-frequency writes and are never hand-edited, so // they persist as compact JSON (no pretty-print) to cut serialization and disk // bytes on the streaming path. - const partWriteBuffer = new PartWriteBuffer((path, value) => - Storage.write(path, value, { compact: true }), - ) + const partWriteBuffer = Storage.state(() => { + const handle = { store: Storage.current().store, artifactDirectory: Storage.current().artifactDirectory } + return new PartWriteBuffer((key, value) => + Storage.provide(handle, () => + Storage.transaction(async (tx) => { + await assertPartOwner(tx, key) + await tx.write(key, value) + }), + ), + ) + }) + + async function assertPartOwner(tx: StoreTransaction, key: string[]) { + await tx.read(["sessions", key[1], key[2], "info"]) + await tx.read(["sessions", key[1], key[2], "messages", key[4], "info"]) + await tx.assertNotDeleted(key) + } /** * Flush all buffered streaming part writes to disk and await them. Called at @@ -1349,8 +1478,8 @@ export namespace Session { * write that normally flushes never fired (issue #327). */ export function flushPartWrites(sessionID?: string) { - if (!sessionID) return partWriteBuffer.flushAll() - return partWriteBuffer.flushWhere((part) => part.sessionID === sessionID) + if (!sessionID) return partWriteBuffer().flushAll() + return partWriteBuffer().flushWhere((part) => part.sessionID === sessionID) } type UpdatePartInternalInput = @@ -1358,14 +1487,9 @@ export namespace Session { | { part: MessageV2.TextPart; delta: string } | { part: MessageV2.ReasoningPart; delta: string } - async function updatePartInternal(input: UpdatePartInternalInput) { - let part = "delta" in input ? input.part : input - const delta = "delta" in input ? input.delta : undefined - // Streaming hot path (issue #350 H1): resolve the scopeID from the permanent - // sessionID -> scopeID cache instead of loading full session info on every - // delta. A session's scope is immutable, so this is safe; on a cold cache it - // reads only the small session-index record. - const scopeID = asScopeID(await SessionManager.resolveScopeID(part.sessionID)) + export async function preparePart(input: MessageV2.Part, ownerScopeID?: string): Promise { + let part = input + const scopeID = asScopeID(ownerScopeID ?? (await SessionManager.resolveScopeID(part.sessionID))) try { const owner = { kind: "session" as const, scopeID, sessionID: part.sessionID } if (part.type === "attachment") part = await RolloutAttachment.capture(owner, part) @@ -1407,6 +1531,18 @@ export namespace Session { } throw error } + return part + } + + async function updatePartInternal(input: UpdatePartInternalInput) { + let part = "delta" in input ? input.part : input + const delta = "delta" in input ? input.delta : undefined + // Streaming hot path (issue #350 H1): resolve the scopeID from the permanent + // sessionID -> scopeID cache instead of loading full session info on every + // delta. A session's scope is immutable, so this is safe; on a cold cache it + // reads only the small session-index record. + const scopeID = asScopeID(await SessionManager.resolveScopeID(part.sessionID)) + part = await preparePart(part) if (delta === undefined) part = MessageV2.canonicalPart(part) const path = StoragePath.messagePart( scopeID, @@ -1414,36 +1550,19 @@ export namespace Session { asMessageID(part.messageID), asPartID(part.id), ) - const searchableDiscreteWrite = - delta === undefined && (part.type === "text" || part.type === "tool" || part.type === "attachment") - // A discrete content-bearing part write is a searchable-content boundary: - // user messages persist their parts AFTER Session.updateMessage marks the - // session dirty, assistant parts stream before the terminal updateMessage, - // and interrupted turns may flush parts with no following message write. - // A rebuild that runs in any of those windows must observe the part. - // - // Invalidate BEFORE the write (crash between write and post-mark cannot - // strand a clean-but-stale record) and refresh AFTER it (a scan racing - // this write sees a marker newer than its scan start). Streaming deltas - // stay unmarked (hot path) because a discrete terminal write always - // follows before the message settles. - if (searchableDiscreteWrite) { - await SessionSearchIndex.markDirty(scopeID, asSessionID(part.sessionID)) - } if (delta !== undefined) { - partWriteBuffer.defer(part.id, path, part) + partWriteBuffer().defer(part.id, path, part) } else { - // Discrete/terminal update: cancel any pending streamed write and persist - // durably before returning (preserves the original write-through contract). - partWriteBuffer.cancel(part.id) - await Storage.write(path, part, { compact: true }) - // Maintain the loop-scoped message cache on durable writes only (#350 D2); - // per-delta streamed updates are coalesced by the write-behind buffer and - // do not need to advance the cache — the terminal write for each part does. - SessionMessageCache.upsertPart(part.sessionID, part) - } - if (searchableDiscreteWrite) { - await SessionSearchIndex.markDirty(scopeID, asSessionID(part.sessionID)) + await partWriteBuffer().writeNow(part.id, path, part, (key, value) => + Storage.transaction(async (tx) => { + await assertPartOwner(tx, key) + await Storage.write(key, value) + if (value.type === "text" || value.type === "tool" || value.type === "attachment") + await SessionSearchIndex.markDirty(scopeID, asSessionID(value.sessionID)) + SessionMessageCache.upsertPart(value.sessionID, value) + await Bus.publish(MessageV2.Event.PartUpdated, { part: value }) + }), + ) } if (part.type === "tool") { // Tool parts are published as unsequenced streaming events. Keep a @@ -1460,10 +1579,7 @@ export namespace Session { durable: delta === undefined, }) } - Bus.publish(MessageV2.Event.PartUpdated, { - part, - delta, - }) + if (delta !== undefined) await Bus.publish(MessageV2.Event.PartUpdated, { part, delta }) return part } diff --git a/packages/harness/src/session/input.ts b/packages/harness/src/session/input.ts index 697874e14..78d1b72fc 100644 --- a/packages/harness/src/session/input.ts +++ b/packages/harness/src/session/input.ts @@ -183,10 +183,14 @@ export type CreateUserMessageInput = InvokeInput & { origin?: MessageV2.OriginUser } -export async function createUserMessage(input: CreateUserMessageInput, rootIDOverride?: string) { +export async function createUserMessage( + input: CreateUserMessageInput, + rootIDOverride?: string, + commitOptions?: SessionUserMessageMaterialization.CommitOptions, +) { if (input.noReply === true) { if (input.experiment) throw new Error("Experiment configuration requires a new root task") - return materializeUserMessage(input, rootIDOverride) + return materializeUserMessage(input, rootIDOverride, commitOptions) } const { Session } = await import(".") const { RolloutLifecycle } = await import("./rollout/lifecycle") @@ -201,7 +205,7 @@ export async function createUserMessage(input: CreateUserMessageInput, rootIDOve try { return await Experiment.provide(configuration, () => RolloutContext.provide({ owner: RolloutLifecycle.owner(session), runID: rootIDOverride ?? messageID }, () => - materializeUserMessage({ ...input, messageID }, rootIDOverride), + materializeUserMessage({ ...input, messageID }, rootIDOverride, commitOptions), ), ) } catch (error) { @@ -210,7 +214,11 @@ export async function createUserMessage(input: CreateUserMessageInput, rootIDOve } } -async function materializeUserMessage(input: CreateUserMessageInput, rootIDOverride?: string) { +async function materializeUserMessage( + input: CreateUserMessageInput, + rootIDOverride?: string, + commitOptions?: SessionUserMessageMaterialization.CommitOptions, +) { const { Session } = await import(".") const { Agent } = await import("../agent/agent") const session = await Session.get(input.sessionID).catch(() => undefined) @@ -757,7 +765,7 @@ async function materializeUserMessage(input: CreateUserMessageInput, rootIDOverr ;(part as MessageV2.TextPart).origin = "user" } } - return SessionUserMessageMaterialization.write({ info, parts }) + return SessionUserMessageMaterialization.write({ info, parts }, commitOptions) } async function effectiveMessages(sessionID: string) { diff --git a/packages/harness/src/session/invoke.ts b/packages/harness/src/session/invoke.ts index c40c51535..ed8a248b4 100644 --- a/packages/harness/src/session/invoke.ts +++ b/packages/harness/src/session/invoke.ts @@ -470,7 +470,7 @@ export namespace SessionInvoke { // so they can trigger a model call in this iteration. Context items follow // in ② after the predicate confirms a call is needed (piggyback). if (!rollbackActive) { - const steerItems = await SessionInbox.drainSteer(sessionID) + const steerItems = await SessionInbox.peekSteer(sessionID) if (steerItems.length > 0) { log.info("drained steer items into session", { sessionID, count: steerItems.length }) for (const item of steerItems) { @@ -531,7 +531,7 @@ export namespace SessionInvoke { // Mode-based drain ②: context items piggyback on confirmed model call. // Materialized after needsModelCall is true; do NOT wake idle sessions. if (!rollbackActive) { - const contextItems = await SessionInbox.drainContext(sessionID) + const contextItems = await SessionInbox.peekContext(sessionID) if (contextItems.length > 0) { log.info("drained context items (piggyback)", { sessionID, count: contextItems.length }) for (const item of contextItems) { diff --git a/packages/harness/src/session/manager.ts b/packages/harness/src/session/manager.ts index ea5cda626..00485361c 100644 --- a/packages/harness/src/session/manager.ts +++ b/packages/harness/src/session/manager.ts @@ -3,6 +3,7 @@ import { GlobalBus } from "../bus/global" import { Context } from "../util/context" import { Identifier } from "../id/id" import { Log } from "../util/log" +import { StorageRecovery } from "../storage/recovery" import { Storage } from "../storage/storage" import { StoragePath } from "../storage/path" import type { MessageV2 } from "./message-v2" @@ -136,30 +137,30 @@ export namespace SessionManager { // `updatePart` only needs the scopeID to build the storage path, not the full // session info. Entries are tiny (ULID -> scopeID strings) and dropped when a // session is deleted (`forgetSession`). - const scopeIDCache = new Map() - const historyRevisions = new Map() + const scopeIDCache = Storage.state(() => new Map()) + const historyRevisions = Storage.state(() => new Map()) function rememberScopeID(sessionID: string, scopeID: string) { - scopeIDCache.set(sessionID, scopeID) + scopeIDCache().set(sessionID, scopeID) } export function forgetSession(sessionID: string) { - scopeIDCache.delete(sessionID) - historyRevisions.delete(sessionID) + scopeIDCache().delete(sessionID) + historyRevisions().delete(sessionID) } /** Cached scopeID lookup, warm during an active loop. */ export function cachedScopeID(sessionID: string): string | undefined { - return scopeIDCache.get(sessionID) + return scopeIDCache().get(sessionID) } export function historyRevision(sessionID: string) { - return historyRevisions.get(sessionID) ?? 0 + return historyRevisions().get(sessionID) ?? 0 } export function bumpHistoryRevision(sessionID: string) { const revision = historyRevision(sessionID) + 1 - historyRevisions.set(sessionID, revision) + historyRevisions().set(sessionID, revision) return revision } @@ -170,11 +171,14 @@ export namespace SessionManager { * path so per-delta persistence never re-reads session state. */ export async function resolveScopeID(sessionID: string): Promise { - const cached = scopeIDCache.get(sessionID) + const cached = scopeIDCache().get(sessionID) if (cached) return cached const indexed = await Storage.read<{ scopeID: string }>( StoragePath.sessionIndex(Identifier.asSessionID(sessionID)), - ).catch(() => undefined) + ).catch((error) => { + if (error instanceof Storage.NotFoundError) return undefined + throw error + }) if (!indexed) throw new Storage.NotFoundError({ message: `Session ${sessionID} not found` }) rememberScopeID(sessionID, indexed.scopeID) return indexed.scopeID @@ -205,7 +209,10 @@ export namespace SessionManager { sweepTimer.unref() async function readSessionInfo(scopeID: string, sessionID: Identifier.SessionID): Promise { - return Storage.read(StoragePath.sessionInfo(Identifier.asScopeID(scopeID), sessionID)).catch(() => undefined) + return Storage.read(StoragePath.sessionInfo(Identifier.asScopeID(scopeID), sessionID)).catch((error) => { + if (error instanceof Storage.NotFoundError) return undefined + throw error + }) } export async function getSessionID(endpoint: SessionEndpoint.Info, scopeID?: string): Promise { @@ -216,7 +223,10 @@ export namespace SessionManager { const sessionID = Identifier.asSessionID(candidateSessionID) const indexed = await Storage.read<{ scopeID: string }>( StoragePath.endpointSession(endpointKey, sessionID), - ).catch(() => undefined) + ).catch((error) => { + if (error instanceof Storage.NotFoundError) return undefined + throw error + }) if (!indexed || (scopeID && indexed.scopeID !== scopeID)) continue const info = await readSessionInfo(indexed.scopeID, sessionID) @@ -232,12 +242,18 @@ export namespace SessionManager { if (!sessionID) return undefined const indexed = await Storage.read<{ scopeID: string }>( StoragePath.sessionIndex(Identifier.asSessionID(sessionID)), - ).catch(() => undefined) + ).catch((error) => { + if (error instanceof Storage.NotFoundError) return undefined + throw error + }) if (!indexed) return undefined rememberScopeID(sessionID, indexed.scopeID) return Storage.read( StoragePath.sessionInfo(Identifier.asScopeID(indexed.scopeID), Identifier.asSessionID(sessionID)), - ).catch(() => undefined) + ).catch((error) => { + if (error instanceof Storage.NotFoundError) return undefined + throw error + }) } export async function requireSession(input: string | SessionEndpoint.Info): Promise { @@ -450,6 +466,7 @@ export namespace SessionManager { } export function acquire(sessionID: string): LoopLease | undefined { + StorageRecovery.assertRunnable(sessionID) if (!accepting) throw new Error("Synergy runtime is shutting down") const runtime = registerRuntime(sessionID) if (occupied(runtime)) return undefined @@ -785,7 +802,10 @@ export namespace SessionManager { for (const sessionID of ids) { const info = await Storage.read( StoragePath.sessionInfo(Identifier.asScopeID(scopeID), Identifier.asSessionID(sessionID)), - ).catch(() => undefined) + ).catch((error) => { + if (error instanceof Storage.NotFoundError) return undefined + throw error + }) if (!info || !info.time || info.time.archived || info.pendingReply !== true) continue sessionIDs.add(info.id) } @@ -804,7 +824,10 @@ export namespace SessionManager { if (isRunning(sessionID)) continue const info = await Storage.read( StoragePath.sessionInfo(Identifier.asScopeID(scopeID), Identifier.asSessionID(sessionID)), - ).catch(() => undefined) + ).catch((error) => { + if (error instanceof Storage.NotFoundError) return undefined + throw error + }) if (!info || !info.time || info.time.archived) continue if (info.cortex?.status !== "queued" && info.cortex?.status !== "running") continue sessionIDs.add(info.id) @@ -823,7 +846,10 @@ export namespace SessionManager { for (const sessionID of ids) { const info = await Storage.read( StoragePath.sessionInfo(Identifier.asScopeID(scopeID), Identifier.asSessionID(sessionID)), - ).catch(() => undefined) + ).catch((error) => { + if (error instanceof Storage.NotFoundError) return undefined + throw error + }) if (!info || !info.time || info.time.archived || !info.cortex) continue if ( info.cortex.status !== "completed" && diff --git a/packages/harness/src/session/message-cache.ts b/packages/harness/src/session/message-cache.ts index 20b64b869..d1e720fe9 100644 --- a/packages/harness/src/session/message-cache.ts +++ b/packages/harness/src/session/message-cache.ts @@ -1,7 +1,7 @@ +import { Storage } from "../storage/storage" import { MessageV2 } from "./message-v2" import { applyModelWorkingSetProjection, modelWorkingSetProjection } from "./model-working-set" import { LLMTurnMemory } from "./llm-memory" - // Loop-scoped in-memory model working-set cache (issue #350 D2). // // The invoke loop assembles model context on every step. The cache holds only @@ -15,18 +15,19 @@ import { LLMTurnMemory } from "./llm-memory" // Maintenance is immutable so a list already handed to a caller remains a valid // snapshot while later writes advance the cache. export namespace SessionMessageCache { - const active = new Set() - const cache = new Map() - + const state = Storage.state(() => ({ + active: new Set(), + cache: new Map(), + sizes: new Map(), + lru: [] as string[], + totalBytes: 0, + hits: 0, + misses: 0, + evictions: 0, + protectedOverbudget: 0, + })) // Bound the aggregate footprint of concurrent model working sets. Eviction is // transparent because the next read reconstructs the working set from disk. - const sizes = new Map() - const lru: string[] = [] - let totalBytes = 0 - let hits = 0 - let misses = 0 - let evictions = 0 - let protectedOverbudget = 0 const DEFAULT_BYTE_BUDGET = 256 * 1024 * 1024 // Read on each eviction so SYNERGY_SESSION_CACHE_MAX_BYTES can be tuned (and // set by tests) without a restart; the cost is a trivial env parse on writes. @@ -34,48 +35,48 @@ export namespace SessionMessageCache { const env = Number.parseInt(process.env.SYNERGY_SESSION_CACHE_MAX_BYTES ?? "", 10) return Number.isFinite(env) && env > 0 ? env : DEFAULT_BYTE_BUDGET } - /** Begin the single-writer window for a session (loop start). */ export function enable(sessionID: string) { - active.add(sessionID) + state().active.add(sessionID) } - /** End the window and drop the entry (loop exit). */ export function disable(sessionID: string) { - active.delete(sessionID) + state().active.delete(sessionID) drop(sessionID) } - /** Drop the cached list but keep the window open; the next read repopulates. */ export function invalidate(sessionID: string) { drop(sessionID) } - export function isActive(sessionID: string) { - return active.has(sessionID) + return state().active.has(sessionID) } - /** Cached model working set, or undefined when closed or unpopulated. */ export function get(sessionID: string): MessageV2.WithParts[] | undefined { return read(sessionID, true) } - function read(sessionID: string, countStats: boolean): MessageV2.WithParts[] | undefined { - if (!active.has(sessionID)) return undefined - const hit = cache.get(sessionID) + if (Storage.inTransaction() || !state().active.has(sessionID)) return undefined + const hit = state().cache.get(sessionID) if (hit) { - if (countStats) hits++ + if (countStats) state().hits++ touch(sessionID) } else if (countStats) { - misses++ + state().misses++ } return hit } - - export function stats(input: { entryLimit?: number } = {}) { + export function stats( + input: { + entryLimit?: number + } = {}, + ) { const entryLimit = Math.max(0, Math.floor(input.entryLimit ?? 100)) - const entries: Array<{ sessionID: string; estimatedBytes: number }> = [] - for (const [sessionID, estimatedBytes] of sizes) { + const entries: Array<{ + sessionID: string + estimatedBytes: number + }> = [] + for (const [sessionID, estimatedBytes] of state().sizes) { let lo = 0 let hi = entries.length while (lo < hi) { @@ -93,51 +94,55 @@ export namespace SessionMessageCache { if (entries.length > entryLimit) entries.pop() } return { - totalBytes, - activeCount: active.size, - entryCount: cache.size, - hits, - misses, - evictions, - protectedOverbudget, + totalBytes: state().totalBytes, + activeCount: state().active.size, + entryCount: state().cache.size, + hits: state().hits, + misses: state().misses, + evictions: state().evictions, + protectedOverbudget: state().protectedOverbudget, entries, - truncatedEntryCount: Math.max(0, sizes.size - entries.length), + truncatedEntryCount: Math.max(0, state().sizes.size - entries.length), } } - export function resetStatsForTest() { - hits = 0 - misses = 0 - evictions = 0 - protectedOverbudget = 0 + state().hits = 0 + state().misses = 0 + state().evictions = 0 + state().protectedOverbudget = 0 } - /** Full teardown for tests: clears windows, entries, and counters. */ export function resetForTest() { - active.clear() - cache.clear() - sizes.clear() - lru.length = 0 - totalBytes = 0 + state().active.clear() + state().cache.clear() + state().sizes.clear() + state().lru.length = 0 + state().totalBytes = 0 resetStatsForTest() } - /** Seed from a fresh compaction-aware disk read (no-op outside the window). */ export function set(sessionID: string, messages: MessageV2.WithParts[]) { - if (!active.has(sessionID)) return + if (Storage.inTransaction()) { + Storage.afterCommit(() => set(sessionID, messages)) + return + } + if (!state().active.has(sessionID)) return const workingSet = projectModelWorkingSet(messages) const size = estimateList(workingSet) if (size > byteBudget()) { drop(sessionID) return } - cache.set(sessionID, workingSet) + state().cache.set(sessionID, workingSet) setSize(sessionID, size) touch(sessionID) evict(sessionID) } - export function upsertMessage(sessionID: string, info: MessageV2.Info) { + if (Storage.inTransaction()) { + Storage.afterCommit(() => upsertMessage(sessionID, info)) + return + } const list = read(sessionID, false) if (!list) return const idx = list.findIndex((m) => m.info.id === info.id) @@ -152,13 +157,16 @@ export namespace SessionMessageCache { replaceProjected(sessionID, next) return } - cache.set(sessionID, next) + state().cache.set(sessionID, next) addSize(sessionID, estimateInfo(info) - (previous ? estimateInfo(previous) : 0)) touch(sessionID) evict(sessionID) } - export function upsertPart(sessionID: string, part: MessageV2.Part) { + if (Storage.inTransaction()) { + Storage.afterCommit(() => upsertPart(sessionID, part)) + return + } const list = read(sessionID, false) if (!list) return const mi = list.findIndex((m) => m.info.id === part.messageID) @@ -187,20 +195,18 @@ export namespace SessionMessageCache { replaceProjected(sessionID, next) return } - cache.set(sessionID, next) + state().cache.set(sessionID, next) addSize(sessionID, estimatePart(part) - (previous ? estimatePart(previous) : 0)) touch(sessionID) evict(sessionID) } - function replaceProjected(sessionID: string, messages: MessageV2.WithParts[]) { const workingSet = projectModelWorkingSet(messages) - cache.set(sessionID, workingSet) + state().cache.set(sessionID, workingSet) setSize(sessionID, estimateList(workingSet)) touch(sessionID) evict(sessionID) } - function projectModelWorkingSet(messages: MessageV2.WithParts[]) { const projection = modelWorkingSetProjection(messages.map((message) => message.info)) if (!projection) return messages @@ -213,65 +219,56 @@ export namespace SessionMessageCache { (message) => ({ ...message, info: { ...message.info, includeInContext: false } }), ) } - // --- Footprint accounting & LRU eviction --- - function drop(sessionID: string) { - cache.delete(sessionID) - const size = sizes.get(sessionID) + state().cache.delete(sessionID) + const size = state().sizes.get(sessionID) if (size !== undefined) { - totalBytes -= size - sizes.delete(sessionID) + state().totalBytes -= size + state().sizes.delete(sessionID) } - const i = lru.indexOf(sessionID) - if (i !== -1) lru.splice(i, 1) + const i = state().lru.indexOf(sessionID) + if (i !== -1) state().lru.splice(i, 1) } - function touch(sessionID: string) { - const i = lru.indexOf(sessionID) - if (i !== -1) lru.splice(i, 1) - lru.push(sessionID) + const i = state().lru.indexOf(sessionID) + if (i !== -1) state().lru.splice(i, 1) + state().lru.push(sessionID) } - function setSize(sessionID: string, bytes: number) { - totalBytes += bytes - (sizes.get(sessionID) ?? 0) - sizes.set(sessionID, bytes) + state().totalBytes += bytes - (state().sizes.get(sessionID) ?? 0) + state().sizes.set(sessionID, bytes) } - function addSize(sessionID: string, bytes: number) { - const current = sizes.get(sessionID) + const current = state().sizes.get(sessionID) if (current === undefined) return - totalBytes += bytes - sizes.set(sessionID, current + bytes) + state().totalBytes += bytes + state().sizes.set(sessionID, current + bytes) } - // Evict least-recently-used entries until under budget. The current writer is // protected only while its own entry fits the budget; a single oversized // working set must not make the aggregate limit ineffective. function evict(protect: string) { const budget = byteBudget() - if (totalBytes <= budget) return - if ((sizes.get(protect) ?? 0) > budget) drop(protect) - for (let i = 0; i < lru.length && totalBytes > budget; ) { - const victim = lru[i] + if (state().totalBytes <= budget) return + if ((state().sizes.get(protect) ?? 0) > budget) drop(protect) + for (let i = 0; i < state().lru.length && state().totalBytes > budget; ) { + const victim = state().lru[i] if (victim === protect) { i++ continue } drop(victim) - evictions++ + state().evictions++ } - if (totalBytes > budget) protectedOverbudget++ + if (state().totalBytes > budget) state().protectedOverbudget++ } - function estimatePart(part: MessageV2.Part): number { return LLMTurnMemory.estimateBytes(part) } - function estimateInfo(info: MessageV2.Info): number { return LLMTurnMemory.estimateBytes(info) } - function estimateList(list: MessageV2.WithParts[]): number { let total = 0 for (const m of list) { @@ -280,7 +277,6 @@ export namespace SessionMessageCache { } return total } - function messageInsertionIndex(messages: MessageV2.WithParts[], info: MessageV2.Info): number { let lo = 0 let hi = messages.length @@ -291,7 +287,6 @@ export namespace SessionMessageCache { } return lo } - function insertionIndex(arr: T[], id: string, idOf: (t: T) => string): number { let lo = 0 let hi = arr.length diff --git a/packages/harness/src/session/message-v2.ts b/packages/harness/src/session/message-v2.ts index c810cad67..e2813d4c0 100644 --- a/packages/harness/src/session/message-v2.ts +++ b/packages/harness/src/session/message-v2.ts @@ -13,7 +13,6 @@ import { SnapshotSchema } from "./snapshot-schema" import { fn } from "../util/fn" import { Storage } from "../storage/storage" import { StoragePath } from "../storage/path" -import { Lock } from "../util/lock" import { ProviderTransform } from "../provider/transform" import { ProviderAuthRecoveryError } from "../provider/auth-recovery-error" import { STATUS_CODES } from "http" @@ -1327,7 +1326,7 @@ export namespace MessageV2 { byMessageID: Map } - const messageOrderCache = new Map() + const messageOrderCache = Storage.state(() => new Map()) const MESSAGE_ORDER_CACHE_LIMIT = 64 const MESSAGE_ORDER_SIGN_BIT = 1n << 63n const MESSAGE_ORDER_MASK = (1n << 64n) - 1n @@ -1370,17 +1369,30 @@ export namespace MessageV2 { await Promise.all( markers .slice(index, index + MESSAGE_ORDER_REBUILD_CONCURRENCY) - .map((marker) => - Storage.write(StoragePath.sessionMessageOrderMarker(scopeID, sessionID, marker), {}, { compact: true }), - ), + .map((marker) => Storage.write(StoragePath.sessionMessageOrderMarker(scopeID, sessionID, marker), {})), ) } } function cacheMessageOrder(scopeID: Identifier.ScopeID, sessionID: Identifier.SessionID, markers: string[]) { const key = messageOrderKey(scopeID, sessionID) - messageOrderCache.delete(key) - messageOrderCache.set(key, { + const value = { + markers: markers.slice(), + byMessageID: new Map( + markers.flatMap((marker) => { + const id = markerMessageID(marker) + return id ? [[id, marker] as const] : [] + }), + ), + } + if (Storage.inTransaction()) { + Storage.afterCommit(() => { + cacheMessageOrder(scopeID, sessionID, markers) + }) + return value + } + messageOrderCache().delete(key) + messageOrderCache().set(key, { markers, byMessageID: new Map( markers.flatMap((marker) => { @@ -1389,16 +1401,16 @@ export namespace MessageV2 { }), ), }) - while (messageOrderCache.size > MESSAGE_ORDER_CACHE_LIMIT) { - const oldest = messageOrderCache.keys().next().value + while (messageOrderCache().size > MESSAGE_ORDER_CACHE_LIMIT) { + const oldest = messageOrderCache().keys().next().value if (oldest === undefined) break - messageOrderCache.delete(oldest) + messageOrderCache().delete(oldest) } - return messageOrderCache.get(key)! + return messageOrderCache().get(key)! } async function rebuildMessageOrder(scopeID: Identifier.ScopeID, sessionID: Identifier.SessionID) { - messageOrderCache.delete(messageOrderKey(scopeID, sessionID)) + messageOrderCache().delete(messageOrderKey(scopeID, sessionID)) await Storage.write(StoragePath.sessionMessageOrderState(scopeID, sessionID), { version: 1, ready: false, @@ -1407,20 +1419,20 @@ export namespace MessageV2 { await Storage.removeTree(StoragePath.sessionMessageOrderMarkersRoot(scopeID, sessionID)) const markers = infos.map(messageOrderMarker) await writeMessageOrderMarkers(scopeID, sessionID, markers) - await Storage.write( - StoragePath.sessionMessageOrderState(scopeID, sessionID), - { version: 1, ready: true, count: markers.length }, - { compact: true }, - ) + await Storage.write(StoragePath.sessionMessageOrderState(scopeID, sessionID), { + version: 1, + ready: true, + count: markers.length, + }) return cacheMessageOrder(scopeID, sessionID, markers) } async function loadMessageOrder(scopeID: Identifier.ScopeID, sessionID: Identifier.SessionID) { const key = messageOrderKey(scopeID, sessionID) - const cached = messageOrderCache.get(key) + const cached = Storage.inTransaction() ? undefined : messageOrderCache().get(key) if (cached) { - messageOrderCache.delete(key) - messageOrderCache.set(key, cached) + messageOrderCache().delete(key) + messageOrderCache().set(key, cached) return cached } @@ -1438,54 +1450,53 @@ export namespace MessageV2 { } async function messageOrderSnapshot(scopeID: Identifier.ScopeID, sessionID: Identifier.SessionID) { - const key = messageOrderKey(scopeID, sessionID) - if (messageOrderCache.has(key)) { - using _ = await Lock.read(`session-message-order:${scopeID}:${sessionID}`) - const cached = messageOrderCache.get(key) - if (cached) return cached.markers.slice() - } - using _ = await Lock.write(`session-message-order:${scopeID}:${sessionID}`) - return (await loadMessageOrder(scopeID, sessionID)).markers.slice() + return Storage.transaction(async () => { + const key = messageOrderKey(scopeID, sessionID) + if (messageOrderCache().has(key)) { + const cached = Storage.inTransaction() ? undefined : messageOrderCache().get(key) + if (cached) return cached.markers.slice() + } + + return (await loadMessageOrder(scopeID, sessionID)).markers.slice() + }) } export async function writeInfo(input: { scopeID: Identifier.ScopeID; info: Info }) { - const info = canonicalMessage(input.info) - const sessionID = Identifier.asSessionID(info.sessionID) - using _ = await Lock.write(`session-message-order:${input.scopeID}:${sessionID}`) - const order = await loadMessageOrder(input.scopeID, sessionID) - const previousMarker = order.byMessageID.get(info.id) - const nextMarker = messageOrderMarker(info) - if (previousMarker === nextMarker) { + return Storage.transaction(async () => { + const info = canonicalMessage(input.info) + const sessionID = Identifier.asSessionID(info.sessionID) + + const order = await loadMessageOrder(input.scopeID, sessionID) + const previousMarker = order.byMessageID.get(info.id) + const nextMarker = messageOrderMarker(info) + if (previousMarker === nextMarker) { + await Storage.write(StoragePath.messageInfo(input.scopeID, sessionID, Identifier.asMessageID(info.id)), info) + return info + } + + messageOrderCache().delete(messageOrderKey(input.scopeID, sessionID)) + await Storage.write(StoragePath.sessionMessageOrderState(input.scopeID, sessionID), { + version: 1, + ready: false, + }) await Storage.write(StoragePath.messageInfo(input.scopeID, sessionID, Identifier.asMessageID(info.id)), info) + if (previousMarker) { + await Storage.remove(StoragePath.sessionMessageOrderMarker(input.scopeID, sessionID, previousMarker)) + } + await Storage.write(StoragePath.sessionMessageOrderMarker(input.scopeID, sessionID, nextMarker), {}) + + const markers = order.markers + .filter((marker) => marker !== previousMarker) + .concat(nextMarker) + .sort(compareMessageOrderMarker) + await Storage.write(StoragePath.sessionMessageOrderState(input.scopeID, sessionID), { + version: 1, + ready: true, + count: markers.length, + }) + cacheMessageOrder(input.scopeID, sessionID, markers) return info - } - - messageOrderCache.delete(messageOrderKey(input.scopeID, sessionID)) - await Storage.write(StoragePath.sessionMessageOrderState(input.scopeID, sessionID), { - version: 1, - ready: false, }) - await Storage.write(StoragePath.messageInfo(input.scopeID, sessionID, Identifier.asMessageID(info.id)), info) - if (previousMarker) { - await Storage.remove(StoragePath.sessionMessageOrderMarker(input.scopeID, sessionID, previousMarker)) - } - await Storage.write( - StoragePath.sessionMessageOrderMarker(input.scopeID, sessionID, nextMarker), - {}, - { compact: true }, - ) - - const markers = order.markers - .filter((marker) => marker !== previousMarker) - .concat(nextMarker) - .sort(compareMessageOrderMarker) - await Storage.write( - StoragePath.sessionMessageOrderState(input.scopeID, sessionID), - { version: 1, ready: true, count: markers.length }, - { compact: true }, - ) - cacheMessageOrder(input.scopeID, sessionID, markers) - return info } export async function removeInfo(input: { @@ -1493,34 +1504,36 @@ export namespace MessageV2 { sessionID: Identifier.SessionID messageID: Identifier.MessageID }) { - using _ = await Lock.write(`session-message-order:${input.scopeID}:${input.sessionID}`) - const order = await loadMessageOrder(input.scopeID, input.sessionID) - const marker = order.byMessageID.get(input.messageID) - if (!marker) { - await Storage.remove(StoragePath.messageInfo(input.scopeID, input.sessionID, input.messageID)) - return - } + return Storage.transaction(async () => { + const order = await loadMessageOrder(input.scopeID, input.sessionID) + const marker = order.byMessageID.get(input.messageID) + if (!marker) { + await Storage.remove(StoragePath.messageInfo(input.scopeID, input.sessionID, input.messageID)) + return + } - messageOrderCache.delete(messageOrderKey(input.scopeID, input.sessionID)) - await Storage.write(StoragePath.sessionMessageOrderState(input.scopeID, input.sessionID), { - version: 1, - ready: false, + messageOrderCache().delete(messageOrderKey(input.scopeID, input.sessionID)) + await Storage.write(StoragePath.sessionMessageOrderState(input.scopeID, input.sessionID), { + version: 1, + ready: false, + }) + await Storage.remove(StoragePath.messageInfo(input.scopeID, input.sessionID, input.messageID)) + await Storage.remove(StoragePath.sessionMessageOrderMarker(input.scopeID, input.sessionID, marker)) + const markers = order.markers.filter((candidate) => candidate !== marker) + await Storage.write(StoragePath.sessionMessageOrderState(input.scopeID, input.sessionID), { + version: 1, + ready: true, + count: markers.length, + }) + cacheMessageOrder(input.scopeID, input.sessionID, markers) }) - await Storage.remove(StoragePath.messageInfo(input.scopeID, input.sessionID, input.messageID)) - await Storage.remove(StoragePath.sessionMessageOrderMarker(input.scopeID, input.sessionID, marker)) - const markers = order.markers.filter((candidate) => candidate !== marker) - await Storage.write( - StoragePath.sessionMessageOrderState(input.scopeID, input.sessionID), - { version: 1, ready: true, count: markers.length }, - { compact: true }, - ) - cacheMessageOrder(input.scopeID, input.sessionID, markers) } export async function removeOrderIndex(scopeID: Identifier.ScopeID, sessionID: Identifier.SessionID) { - using _ = await Lock.write(`session-message-order:${scopeID}:${sessionID}`) - messageOrderCache.delete(messageOrderKey(scopeID, sessionID)) - await Storage.removeTree(StoragePath.sessionMessageOrderRoot(scopeID, sessionID)) + return Storage.transaction(async () => { + messageOrderCache().delete(messageOrderKey(scopeID, sessionID)) + await Storage.removeTree(StoragePath.sessionMessageOrderRoot(scopeID, sessionID)) + }) } export async function readInfoList(input: { diff --git a/packages/harness/src/session/migration.ts b/packages/harness/src/session/migration.ts index 28a072d6b..0d125c1ca 100644 --- a/packages/harness/src/session/migration.ts +++ b/packages/harness/src/session/migration.ts @@ -401,9 +401,6 @@ function applyPrimaryAttachmentVisibility( return { value, changed } } -const legacyAttachmentPattern = String.raw`"type"\s*:\s*"file"|"artifact-only"|"attachment-only"|"primaryAttachmentIds"|"kind"\s*:\s*"artifact"|"artifact"\s*:|"mode"\s*:\s*"(inline|card|hidden)"|"primary"\s*:\s*true` -const legacyToolDisplayPattern = String.raw`"visibility"\s*:\s*"media"|"attachment-only"|"primaryAttachmentIds"` -const attachmentPartGlob = new Bun.Glob("sessions/**/parts/*.json") const legacyAttachmentMarkers = [ '"type":"file"', '"type": "file"', @@ -461,161 +458,22 @@ function candidateFromRelativePath(relativePath: string, text: string): Attachme } } -async function existingRipgrepPath() { - const system = Bun.which("rg") - if (system) return system - const bundled = path.join(Global.Path.bin, process.platform === "win32" ? "rg.exe" : "rg") - return (await Bun.file(bundled) - .exists() - .catch(() => false)) - ? bundled - : undefined -} - -async function readSpawnStdout(stdout: unknown) { - if (!stdout || typeof stdout === "string") return undefined - if (typeof (stdout as { text?: unknown }).text === "function") { - return (stdout as { text: () => Promise }).text() +async function collectPartCandidates(predicate: (text: string) => boolean): Promise { + const result: AttachmentPartCandidate[] = [] + for await (const record of Storage.records({ kind: "part" })) { + const text = JSON.stringify(record.value) + if (!predicate(text)) continue + const candidate = candidateFromRelativePath(record.key.join("/"), text) + if (candidate) result.push(candidate) } - if (typeof (stdout as { getReader?: unknown }).getReader === "function") - return Bun.readableStreamToText(stdout as ReadableStream) - return undefined + return result } -async function findLegacyAttachmentPartPaths() { - const sessionsRoot = path.join(Global.Path.data, "sessions") - if (!(await fs.stat(sessionsRoot).catch(() => undefined))?.isDirectory()) return [] - const rg = await existingRipgrepPath() - if (!rg) return undefined - - const proc = Bun.spawn( - [ - rg, - "--files-with-matches", - "--hidden", - "--follow", - "--glob=**/parts/*.json", - "--", - legacyAttachmentPattern, - sessionsRoot, - ], - { - stdout: "pipe", - stderr: "ignore", - maxBuffer: 1024 * 1024 * 50, - }, - ) - const [text, exitCode] = await Promise.all([readSpawnStdout(proc.stdout), proc.exited]) - if (text === undefined) return undefined - if (exitCode !== 0 && exitCode !== 1) { - log.warn("legacy attachment part candidate search failed", { exitCode }) - return [] - } - return text.split(/\r?\n/).filter(Boolean) +function collectLegacyAttachmentPartCandidates() { + return collectPartCandidates(needsAttachmentMigration) } - -async function findLegacyToolDisplayPartPaths() { - const sessionsRoot = path.join(Global.Path.data, "sessions") - if (!(await fs.stat(sessionsRoot).catch(() => undefined))?.isDirectory()) return [] - const rg = await existingRipgrepPath() - if (!rg) return undefined - - const proc = Bun.spawn( - [ - rg, - "--files-with-matches", - "--hidden", - "--follow", - "--glob=**/parts/*.json", - "--", - legacyToolDisplayPattern, - sessionsRoot, - ], - { - stdout: "pipe", - stderr: "ignore", - maxBuffer: 1024 * 1024 * 50, - }, - ) - const [text, exitCode] = await Promise.all([readSpawnStdout(proc.stdout), proc.exited]) - if (text === undefined) return undefined - if (exitCode !== 0 && exitCode !== 1) { - log.warn("legacy tool display part candidate search failed", { exitCode }) - return [] - } - return text.split(/\r?\n/).filter(Boolean) -} - -async function collectLegacyAttachmentPartCandidates(): Promise { - const candidates: AttachmentPartCandidate[] = [] - const pending: Promise[] = [] - const flush = async () => { - if (pending.length === 0) return - await Promise.all(pending.splice(0)) - } - - const paths = await findLegacyAttachmentPartPaths() - const scan = async function* () { - if (paths) { - for (const filepath of paths) yield filepath - return - } - for await (const relativePath of attachmentPartGlob.scan({ cwd: Global.Path.data, onlyFiles: true })) { - yield path.join(Global.Path.data, relativePath) - } - } - - for await (const filepath of scan()) { - pending.push( - Bun.file(filepath) - .text() - .then((text) => { - if (!needsAttachmentMigration(text)) return - const candidate = candidateFromRelativePath(path.relative(Global.Path.data, filepath), text) - if (candidate) candidates.push(candidate) - }) - .catch(() => undefined), - ) - if (pending.length >= 64) await flush() - } - await flush() - return candidates.sort((a, b) => a.key.join("/").localeCompare(b.key.join("/"))) -} - -async function collectLegacyToolDisplayPartCandidates(): Promise { - const candidates: AttachmentPartCandidate[] = [] - const pending: Promise[] = [] - const flush = async () => { - if (pending.length === 0) return - await Promise.all(pending.splice(0)) - } - - const paths = await findLegacyToolDisplayPartPaths() - const scan = async function* () { - if (paths) { - for (const filepath of paths) yield filepath - return - } - for await (const relativePath of attachmentPartGlob.scan({ cwd: Global.Path.data, onlyFiles: true })) { - yield path.join(Global.Path.data, relativePath) - } - } - - for await (const filepath of scan()) { - pending.push( - Bun.file(filepath) - .text() - .then((text) => { - if (!needsToolDisplayMigration(text)) return - const candidate = candidateFromRelativePath(path.relative(Global.Path.data, filepath), text) - if (candidate) candidates.push(candidate) - }) - .catch(() => undefined), - ) - if (pending.length >= 64) await flush() - } - await flush() - return candidates.sort((a, b) => a.key.join("/").localeCompare(b.key.join("/"))) +function collectLegacyToolDisplayPartCandidates() { + return collectPartCandidates(needsToolDisplayMigration) } async function migrateSessionAttachmentParts(progress: (current: number, total: number) => void) { @@ -2218,6 +2076,48 @@ export const migrations: Migration[] = [ await SnapshotMaintenance.releaseOrphanOwners(progress) }, }, + { + id: "20260914-transactional-session-indexes", + description: "Build Session lookup and navigation projections from transactional authority", + async up(progress) { + const { Session } = await import(".") + progress?.(0, 1) + await Storage.transaction((tx) => Session.rebuildStorageIndexes(tx)) + progress?.(1, 1) + }, + }, + { + id: "20260914-inbox-delivery-receipts", + description: "Index historical inbox materializations without replaying deliveries", + async up(progress) { + const { SessionInbox } = await import("./inbox") + let done = 0 + for await (const record of Storage.records({ kind: "message" })) { + const message = record.value + const deliveryKey = message.metadata?.inboxDeliveryKey + const itemID = + typeof deliveryKey === "string" + ? SessionInbox.stableDeliveryItemID(message.sessionID, deliveryKey) + : /^msg_[a-f0-9]{26}$/.test(message.id) + ? `inb_${message.id.slice(4)}` + : undefined + if (itemID) + await Storage.transaction(async () => { + const key = ["sessions", record.key[1], message.sessionID, "inbox-materialized", itemID] + if ((await Storage.readMany([key]))[0] === undefined) + await Storage.write(key, { + itemID, + messageID: message.id, + deliveryKey, + completedAt: message.time.created, + }) + }) + done++ + if (done % 128 === 0) progress?.(0, 0) + } + progress?.(done, done) + }, + }, ] function canonicalFieldsDiffer(before: any, after: any): boolean { diff --git a/packages/harness/src/session/nav.ts b/packages/harness/src/session/nav.ts index 33139768b..0d4613585 100644 --- a/packages/harness/src/session/nav.ts +++ b/packages/harness/src/session/nav.ts @@ -6,7 +6,6 @@ import { Identifier } from "../id/id" import { Storage } from "../storage/storage" import { StoragePath } from "../storage/path" import { Log } from "../util/log" -import { Lock } from "../util/lock" import { Info as SessionInfo } from "./types" import { SessionManagedProjects } from "./managed-projects" @@ -254,8 +253,9 @@ export namespace SessionNav { } export async function buildNavIndex(scopeID: string): Promise { - using _ = await Lock.write(mutationKey(scopeID)) - return buildNavIndexUnlocked(scopeID) + return Storage.transaction(async () => { + return buildNavIndexUnlocked(scopeID) + }) } async function readNavIndexUnlocked(scopeID: string): Promise { @@ -270,12 +270,14 @@ export namespace SessionNav { } export async function readNavIndex(scopeID: string): Promise { - const existing = await Storage.read( - StoragePath.sessionNavIndex(Identifier.asScopeID(scopeID)), - ).catch(() => undefined) - if (existing) return existing - using _ = await Lock.write(mutationKey(scopeID)) - return readNavIndexUnlocked(scopeID) + return Storage.transaction(async () => { + const existing = await Storage.read( + StoragePath.sessionNavIndex(Identifier.asScopeID(scopeID)), + ).catch(() => undefined) + if (existing) return existing + + return readNavIndexUnlocked(scopeID) + }) } export async function rebuildAllNavIndexes(progress?: (done: number, total: number) => void): Promise { @@ -437,31 +439,33 @@ export namespace SessionNav { entry: SessionNavEntry, options?: { preserveActivityAt?: boolean }, ): Promise { - using _ = await Lock.write(mutationKey(entry.scopeID)) - const index = await readNavIndexUnlocked(entry.scopeID) - const existing = index.entries.findIndex((e) => e.id === entry.id) - const nextEntry = - options?.preserveActivityAt && existing >= 0 - ? { ...entry, lastActivityAt: index.entries[existing].lastActivityAt } - : entry - if (existing >= 0) index.entries.splice(existing, 1) - const insertAt = index.entries.findIndex( - (e) => - e.lastActivityAt < nextEntry.lastActivityAt || - (e.lastActivityAt === nextEntry.lastActivityAt && e.id < nextEntry.id), - ) - if (insertAt === -1) index.entries.push(nextEntry) - else index.entries.splice(insertAt, 0, nextEntry) - index.updatedAt = Date.now() - await Storage.write(StoragePath.sessionNavIndex(Identifier.asScopeID(nextEntry.scopeID)), index) - return nextEntry + return Storage.transaction(async () => { + const index = await readNavIndexUnlocked(entry.scopeID) + const existing = index.entries.findIndex((e) => e.id === entry.id) + const nextEntry = + options?.preserveActivityAt && existing >= 0 + ? { ...entry, lastActivityAt: index.entries[existing].lastActivityAt } + : entry + if (existing >= 0) index.entries.splice(existing, 1) + const insertAt = index.entries.findIndex( + (e) => + e.lastActivityAt < nextEntry.lastActivityAt || + (e.lastActivityAt === nextEntry.lastActivityAt && e.id < nextEntry.id), + ) + if (insertAt === -1) index.entries.push(nextEntry) + else index.entries.splice(insertAt, 0, nextEntry) + index.updatedAt = Date.now() + await Storage.write(StoragePath.sessionNavIndex(Identifier.asScopeID(nextEntry.scopeID)), index) + return nextEntry + }) } export async function removeNavEntry(scopeID: string, sessionID: string): Promise { - using _ = await Lock.write(mutationKey(scopeID)) - const index = await readNavIndexUnlocked(scopeID) - index.entries = index.entries.filter((e) => e.id !== sessionID) - index.updatedAt = Date.now() - await Storage.write(StoragePath.sessionNavIndex(Identifier.asScopeID(scopeID)), index) + return Storage.transaction(async () => { + const index = await readNavIndexUnlocked(scopeID) + index.entries = index.entries.filter((e) => e.id !== sessionID) + index.updatedAt = Date.now() + await Storage.write(StoragePath.sessionNavIndex(Identifier.asScopeID(scopeID)), index) + }) } } diff --git a/packages/harness/src/session/part-write-buffer.ts b/packages/harness/src/session/part-write-buffer.ts index a0e891ac9..d11cb7776 100644 --- a/packages/harness/src/session/part-write-buffer.ts +++ b/packages/harness/src/session/part-write-buffer.ts @@ -1,63 +1,106 @@ -// Write-behind buffer for streaming part persistence (frontend sync redesign, -// perf hotspot S1). updatePart used to write the full part to disk on every -// text/reasoning delta — O(part²) disk I/O for a long streamed reply. Streaming -// increments are now coalesced through this buffer (at most one write per -// interval per part), while discrete/terminal updates (tool state changes, the -// final no-delta part write) go straight to disk so persistence is never lost -// at a meaningful boundary. +import { StorageBusyError } from "../storage/errors" +type Entry = { path: P; value: T; bytes: number } export class PartWriteBuffer { - private latest = new Map() - private timers = new Map>() + private bytes = 0 + private readonly latest = new Map>() + private readonly timers = new Map>() + private readonly running = new Map; promise: Promise }>() + private readonly failures = new Map; error: unknown }>() constructor( private readonly write: (path: P, value: T) => void | Promise, private readonly intervalMs = 500, ) {} - /** Coalesce a streaming increment: remember the latest value, flush on a timer. */ defer(key: string, path: P, value: T): void { - this.latest.set(key, { path, value }) - if (!this.timers.has(key)) { + const failure = this.failures.get(key) + if (failure) throw failure.error + const bytes = Buffer.byteLength(JSON.stringify(value)) + const previous = this.latest.get(key)?.bytes ?? 0 + if ((!this.latest.has(key) && this.latest.size >= 1024) || this.bytes - previous + bytes > 64 * 1024 * 1024) + throw new StorageBusyError("Streaming persistence buffer is full; drain before accepting more output") + this.latest.set(key, { path, value: structuredClone(value), bytes }) + this.bytes += bytes - previous + if (!this.timers.has(key)) this.timers.set( key, - setTimeout(() => void this.flush(key), this.intervalMs), + setTimeout(() => { + void this.flush(key).catch(() => {}) + }, this.intervalMs), ) - } } - /** Flush the buffered value for a key now (used by the timer and on shutdown). */ - flush(key: string): void | Promise { - const timer = this.timers.get(key) - if (timer) clearTimeout(timer) - this.timers.delete(key) + flush(key: string): Promise { const entry = this.latest.get(key) - this.latest.delete(key) - if (entry) return this.write(entry.path, entry.value) + this.cancel(key) + if (entry) return this.execute(key, entry) + return this.running.get(key)?.promise ?? Promise.resolve() + } + + private execute(key: string, entry: Entry, write = this.write): Promise { + if (this.bytes + entry.bytes > 64 * 1024 * 1024) + return Promise.reject(new StorageBusyError("Streaming persistence buffer is full")) + this.bytes += entry.bytes + const previous = this.running.get(key)?.promise + let writing: Promise + try { + writing = previous + ? previous.then(() => write(entry.path, entry.value)) + : Promise.resolve(write(entry.path, entry.value)) + } catch (error) { + writing = Promise.reject(error) + } + const promise = writing.then( + () => { + this.bytes -= entry.bytes + if (this.running.get(key)?.promise === promise) this.running.delete(key) + }, + (error: unknown) => { + this.bytes -= entry.bytes + this.failures.set(key, { entry, error }) + if (this.running.get(key)?.promise === promise) this.running.delete(key) + throw error + }, + ) + this.running.set(key, { entry, promise }) + // Timed writes have no awaiting caller; retain their errors for every drain boundary. + void promise.catch(() => {}) + return promise + } + + async writeNow(key: string, path: P, value: T, write = this.write): Promise { + this.cancel(key) + const failure = this.failures.get(key) + if (failure) throw failure.error + await this.execute( + key, + { path, value: structuredClone(value), bytes: Buffer.byteLength(JSON.stringify(value)) }, + write, + ) } - /** - * Flush every pending write and await them (e.g. before finalizing a turn so - * the persisted parts reflect all streamed content, even when a mid-stream - * interruption skipped the terminal write — issue #327). - */ - async flushAll(): Promise { - await Promise.all([...this.latest.keys()].map((key) => this.flush(key))) + flushAll(): Promise { + return this.flushWhere(() => true) } async flushWhere(predicate: (value: T, path: P) => boolean): Promise { - const keys = [...this.latest.entries()] - .filter(([, entry]) => predicate(entry.value, entry.path)) - .map(([key]) => key) - await Promise.all(keys.map((key) => this.flush(key))) + const keys = new Set() + for (const [key, entry] of this.latest) if (predicate(entry.value, entry.path)) keys.add(key) + for (const [key, { entry }] of this.running) if (predicate(entry.value, entry.path)) keys.add(key) + const results = await Promise.allSettled([...keys].map((key) => this.flush(key))) + const errors = results.flatMap((result) => (result.status === "rejected" ? [result.reason] : [])) + for (const { entry, error } of this.failures.values()) + if (predicate(entry.value, entry.path) && !errors.includes(error)) errors.push(error) + if (errors.length === 1) throw errors[0] + if (errors.length) throw new AggregateError(errors, "Part persistence failed") } - /** Drop any pending deferred write for a key without persisting it. Used when - * the caller is about to persist a superseding value itself. */ cancel(key: string): void { const timer = this.timers.get(key) if (timer) clearTimeout(timer) this.timers.delete(key) + this.bytes -= this.latest.get(key)?.bytes ?? 0 this.latest.delete(key) } } diff --git a/packages/harness/src/session/rollout/archive.ts b/packages/harness/src/session/rollout/archive.ts index 44d479705..8079df8b0 100644 --- a/packages/harness/src/session/rollout/archive.ts +++ b/packages/harness/src/session/rollout/archive.ts @@ -1,3 +1,4 @@ +import { SessionStaging } from "../staging" import { SessionSchemaRegistry } from "../schema-registry" import { SessionExecutionContributions } from "../execution-contributions" import { RolloutAttachment } from "./attachment" @@ -444,7 +445,7 @@ export namespace RolloutArchive { export async function restore(blob: Blob): Promise { const archive = await open(blob) - const created: RolloutSchema.Owner[] = [] + let stagingID: string | undefined try { const report = SessionExport.Report.parse(JSON.parse((await archive.bytes("transcript.json")).toString())) SessionImport.validateScope(report) @@ -468,6 +469,7 @@ export namespace RolloutArchive { ]) for (const record of records) if (!ids.has(record.id)) ids.set(record.id, crypto.randomUUID()) const scopeID = ScopeContext.current.scope.id + stagingID = await SessionStaging.begin(scopeID, [...sessionIDs.values()]) const ownerKey = (owner: RolloutSchema.Owner) => JSON.stringify(owner) const owners = new Map() for (const snapshot of archive.manifest.snapshots) { @@ -479,7 +481,6 @@ export namespace RolloutArchive { sessionID: sessionIDs.get(snapshot.owner.sessionID)!, } owners.set(ownerKey(snapshot.owner), owner) - created.push(owner) } const artifacts = new Map() for (const snapshot of archive.manifest.fileSnapshots) { @@ -598,18 +599,14 @@ export namespace RolloutArchive { "application/vnd.synergy.source-evidence+json", ), }) - await Storage.write( - [...root, "import"], - { - version: 1, - source: snapshot.owner, - revision: snapshot.revision, - integrity: archive.manifest.integrity, - evidence: sourceEvidence, - artifacts: [...artifacts.entries()].map(([source, ref]) => ({ source, ref })), - }, - { private: true, durable: true }, - ) + await Storage.write([...root, "import"], { + version: 1, + source: snapshot.owner, + revision: snapshot.revision, + integrity: archive.manifest.integrity, + evidence: sourceEvidence, + artifacts: [...artifacts.entries()].map(([source, ref]) => ({ source, ref })), + }) } for (const data of report.sessions) { const owner: RolloutSchema.Owner = { kind: "session", scopeID: data.info.scope.id, sessionID: data.info.id } @@ -648,19 +645,12 @@ export namespace RolloutArchive { } SessionSchemaRegistry.normalizeImport(data.info, "archive") } - const result = await SessionImport.fromReport(report, { sessionIDs, rollout: true }) + const result = await SessionImport.fromReport(report, { sessionIDs, rollout: true, stagingID }) result.warnings.push(...archive.manifest.integrity.missing) if (!archive.manifest.integrity.complete) result.warnings.push("Imported rollout is a partial evidence snapshot.") return result } catch (error) { - for (const owner of created) { - if (owner.kind === "session") { - await Session.remove(owner.sessionID).catch(() => {}) - await Storage.removeTree(RolloutArtifact.root(owner)) - await SnapshotLifecycle.beginDelete(owner.scopeID, owner.sessionID) - await SnapshotLifecycle.completeDelete(owner.scopeID, owner.sessionID) - } - } + if (stagingID) await SessionStaging.discard(stagingID) throw error } finally { await archive.reader.close() diff --git a/packages/harness/src/session/rollout/artifact.ts b/packages/harness/src/session/rollout/artifact.ts index 6443d697e..5503575ea 100644 --- a/packages/harness/src/session/rollout/artifact.ts +++ b/packages/harness/src/session/rollout/artifact.ts @@ -14,7 +14,6 @@ export namespace RolloutArtifact { const Chunk = z .object({ sha256: z.string().regex(/^[a-f0-9]{64}$/), bytes: z.number().int().positive().max(CHUNK_BYTES) }) .strict() - const options = { private: true, durable: true, compact: true } as const export function root(input: Owner) { const owner = Owner.parse(input) @@ -33,7 +32,7 @@ export namespace RolloutArtifact { } export async function list(owner: Owner): Promise { - const ids = await Storage.scan([...root(owner), "artifacts"], { strict: true }) + const ids = await Storage.scan([...root(owner), "artifacts"]) const result: Ref[] = [] for (const id of ids) result.push(await get(owner, id)) return result @@ -89,7 +88,7 @@ export namespace RolloutArtifact { status: "partial", } const key = artifactRoot(owner, ref.id) - await record(() => Storage.write([...key, "info"], ref, options)) + await record(() => Storage.write([...key, "info"], ref)) const buffer = new Uint8Array(CHUNK_BYTES) const hash = new Bun.CryptoHasher("sha256") let filled = 0 @@ -101,12 +100,10 @@ export namespace RolloutArtifact { const next = { ...ref, bytes: ref.bytes + filled, chunks: ref.chunks + 1 } await record(async () => { await Storage.writeBinary([...base, "blobs", sha256], data) - await Storage.write( - [...key, "chunks", String(ref.chunks).padStart(12, "0")], - { sha256, bytes: filled }, - options, - ) - await Storage.write([...key, "info"], next, options) + await Storage.transaction(async () => { + await Storage.write([...key, "chunks", String(ref.chunks).padStart(12, "0")], { sha256, bytes: filled }) + await Storage.write([...key, "info"], next) + }) }) hash.update(data) ref = next @@ -157,7 +154,7 @@ export namespace RolloutArtifact { if (finished) return ref await flush() const final: Ref = { ...ref, sha256: status === "complete" ? hash.digest("hex") : null, status } - await record(() => Storage.write([...key, "info"], final, options)) + await record(() => Storage.write([...key, "info"], final)) ref = final finished = true return ref diff --git a/packages/harness/src/session/rollout/continuation-migration.ts b/packages/harness/src/session/rollout/continuation-migration.ts index 03731295e..dcfc45b3e 100644 --- a/packages/harness/src/session/rollout/continuation-migration.ts +++ b/packages/harness/src/session/rollout/continuation-migration.ts @@ -15,7 +15,7 @@ export namespace RolloutContinuationMigration { if (owner.kind !== "session") return const root = [...RolloutArtifact.root(owner), "runs"] const candidates: RolloutSchema.RunRecord[] = [] - for (const id of await Storage.scan(root, { strict: true })) { + for (const id of await Storage.scan(root)) { const run = await RolloutLedger.getRun(owner, id) if (run.status === "completed" && run.recording !== "failed" && !run.cancelRequestedAt) candidates.push(run) } @@ -24,10 +24,7 @@ export namespace RolloutContinuationMigration { const scopeID = Identifier.asScopeID(owner.scopeID) const sessionID = Identifier.asSessionID(owner.sessionID) const history = StoragePath.sessionHistoryRoot(scopeID, sessionID) - const [infos, eventIDs] = await Promise.all([ - MessageV2.readInfoList({ scopeID, sessionID }), - Storage.scan(history, { strict: true }), - ]) + const [infos, eventIDs] = await Promise.all([MessageV2.readInfoList({ scopeID, sessionID }), Storage.scan(history)]) const events = await Promise.all(eventIDs.sort().map((id) => Storage.read([...history, id]))) const messages = SessionHistory.applyEvents( MessageV2.deriveSemantics(infos.map((info) => ({ info, parts: [] }))), @@ -47,8 +44,8 @@ export namespace RolloutContinuationMigration { async up(progress) { progress(0, 0) const owners: RolloutSchema.Owner[] = [] - for (const scopeID of await Storage.scan(["sessions"], { strict: true })) - for (const sessionID of await Storage.scan(["sessions", scopeID], { strict: true })) + for (const scopeID of await Storage.scan(["sessions"])) + for (const sessionID of await Storage.scan(["sessions", scopeID])) owners.push({ kind: "session", scopeID, sessionID }) progress(0, owners.length) for (const [index, owner] of owners.entries()) { diff --git a/packages/harness/src/session/rollout/continuation-recovery.ts b/packages/harness/src/session/rollout/continuation-recovery.ts index ea2b7608e..f1b3ac7c1 100644 --- a/packages/harness/src/session/rollout/continuation-recovery.ts +++ b/packages/harness/src/session/rollout/continuation-recovery.ts @@ -11,7 +11,7 @@ export namespace RolloutContinuationRecovery { const root = (owner: RolloutSchema.Owner) => [...RolloutArtifact.root(owner), "continuation-recovery"] export async function request(owner: RolloutSchema.Owner, runID: string) { - await Storage.write([...root(owner), runID], { runID }, { private: true, durable: true }) + await Storage.write([...root(owner), runID], { runID }) } export async function pending(sessionID: string): Promise { @@ -19,7 +19,7 @@ export namespace RolloutContinuationRecovery { const session = await SessionManager.getSession(sessionID) if (!session?.scope || session.time.archived) return false const owner = { kind: "session", scopeID: session.scope.id, sessionID } as const - const ids = await Storage.scan(root(owner), { strict: true }) + const ids = await Storage.scan(root(owner)) if (!ids.length) return false const [{ SessionHistory }, { SessionProgress }] = await Promise.all([import("../history"), import("../progress")]) if ((await SessionHistory.storedInfo(sessionID))?.rollback?.canUnrollback) return false @@ -45,14 +45,12 @@ export namespace RolloutContinuationRecovery { export async function list(scopeID?: string): Promise { const result: string[] = [] - const scopes = scopeID ? [scopeID] : await Storage.scan(["sessions"], { strict: true }) + const scopes = scopeID ? [scopeID] : await Storage.scan(["sessions"]) for (const scope of scopes) { - for (const sessionID of await Storage.scan(StoragePath.sessionsRoot(Identifier.asScopeID(scope)), { - strict: true, - })) { + for (const sessionID of await Storage.scan(StoragePath.sessionsRoot(Identifier.asScopeID(scope)))) { try { const owner = { kind: "session", scopeID: scope, sessionID } as const - if (!(await Storage.scan(root(owner), { strict: true })).length) continue + if (!(await Storage.scan(root(owner))).length) continue if (await pending(sessionID)) result.push(sessionID) } catch (error) { log.warn("continuation recovery discovery failed", { sessionID, error }) diff --git a/packages/harness/src/session/rollout/journal.ts b/packages/harness/src/session/rollout/journal.ts index eba8f0d26..9216d1fb3 100644 --- a/packages/harness/src/session/rollout/journal.ts +++ b/packages/harness/src/session/rollout/journal.ts @@ -7,7 +7,6 @@ import type { RolloutSchema } from "./schema" import { record } from "./error" export namespace RolloutJournal { - const options = { compact: true, durable: true, private: true } as const const Revision = z.number().int().nonnegative().safe() const Head = z .object({ allocated: Revision, committed: Revision }) @@ -50,23 +49,22 @@ export namespace RolloutJournal { const previous = await head(owner) const gaps: number[] = [] for (let seq = previous.committed + 1; seq <= previous.allocated; seq++) { - let event: Event - try { - event = Event.parse(await Storage.read(eventKey(owner, seq))) - } catch (error) { - if (!(error instanceof Storage.NotFoundError)) throw error - event = { version: 1, seq, time: Date.now(), kind: "gap" } - await Storage.write(eventKey(owner, seq), event, options) - } - if (event.seq !== seq) throw new Error("Rollout journal sequence mismatch") - if (event.kind === "record") { - await Storage.write([...RolloutArtifact.root(owner), ...event.key], event.value, options) - } else gaps.push(seq) + await Storage.transaction(async () => { + let event: Event + try { + event = Event.parse(await Storage.read(eventKey(owner, seq))) + } catch (error) { + if (!(error instanceof Storage.NotFoundError)) throw error + event = { version: 1, seq, time: Date.now(), kind: "gap" } + await Storage.write(eventKey(owner, seq), event) + } + if (event.seq !== seq) throw new Error("Rollout journal sequence mismatch") + if (event.kind === "record") await Storage.write([...RolloutArtifact.root(owner), ...event.key], event.value) + else gaps.push(seq) + await Storage.write([...root(owner), "head"], { ...previous, committed: seq }) + }) onProgress?.() } - if (previous.committed !== previous.allocated) { - await Storage.write([...root(owner), "head"], { ...previous, committed: previous.allocated }, options) - } return { recovered: previous.allocated - previous.committed, gaps } } @@ -80,8 +78,7 @@ export namespace RolloutJournal { const base = RolloutArtifact.root(owner) if (!base.every((segment, index) => key[index] === segment)) throw new Error("Rollout write escapes its owner") using lock = await Lock.write(lockKey(owner)) - // Record the owner before any mutation so a crash cannot leave mutated journal work unlisted. - await RolloutPending.track(owner) + if (Storage.inTransaction()) throw new Error("Rollout evidence requires its own commit boundary") await recoverPending(owner) const previous = await head(owner) const seq = Revision.parse(previous.allocated + 1) @@ -94,11 +91,15 @@ export namespace RolloutJournal { value: JSON.parse(JSON.stringify(value)), }) if (event.kind !== "record") throw new Error("Invalid rollout record") - // Reservation prevents reuse of a sequence whose evidence survived a failed commit. - await Storage.write([...root(owner), "head"], { ...previous, allocated: seq }, options) - await Storage.write(eventKey(owner, seq), event, options) - await Storage.write(key, event.value, options) - await Storage.write([...root(owner), "head"], { allocated: seq, committed: seq }, options) + await Storage.transaction(async () => { + await RolloutPending.track(owner) + await Storage.write([...root(owner), "head"], { ...previous, allocated: seq }) + await Storage.write(eventKey(owner, seq), event) + }) + await Storage.transaction(async () => { + await Storage.write(key, event.value) + await Storage.write([...root(owner), "head"], { allocated: seq, committed: seq }) + }) return seq }) } diff --git a/packages/harness/src/session/rollout/ledger.ts b/packages/harness/src/session/rollout/ledger.ts index b127b972c..8d1c6af11 100644 --- a/packages/harness/src/session/rollout/ledger.ts +++ b/packages/harness/src/session/rollout/ledger.ts @@ -151,7 +151,7 @@ export namespace RolloutLedger { export async function segments(owner: Owner, runID: string) { const base = [...root(owner, runID), "segments"] - const ids = await Storage.scan(base, { strict: true }) + const ids = await Storage.scan(base) return Promise.all(ids.map(async (id) => RolloutSchema.ExecutionSegment.parse(await Storage.read([...base, id])))) } @@ -172,7 +172,7 @@ export namespace RolloutLedger { } export async function calls(owner: Owner, runID: string) { - const ids = await Storage.scan([...root(owner, runID), "calls"], { strict: true }) + const ids = await Storage.scan([...root(owner, runID), "calls"]) const result: RolloutSchema.CallRecord[] = [] for (const id of ids) result.push(await getCall(owner, runID, id)) return result @@ -190,7 +190,7 @@ export namespace RolloutLedger { export async function attempts(owner: Owner, runID: string, callID: string) { const base = attemptRoot(owner, runID, callID) - const ids = await Storage.scan(base, { strict: true }) + const ids = await Storage.scan(base) const result: RolloutSchema.AttemptRecord[] = [] for (const id of ids) result.push(RolloutSchema.AttemptRecord.parse(await Storage.read([...base, id]))) return result.sort((a, b) => a.index - b.index) @@ -253,7 +253,7 @@ export namespace RolloutLedger { export async function tools(owner: Owner, runID: string) { const base = [...root(owner, runID), "tools"] - const ids = await Storage.scan(base, { strict: true }) + const ids = await Storage.scan(base) const result: RolloutSchema.ToolExecutionRecord[] = [] for (const id of ids) result.push(RolloutSchema.ToolExecutionRecord.parse(await Storage.read([...base, id]))) return result @@ -261,7 +261,7 @@ export namespace RolloutLedger { export async function processes(owner: Owner, runID: string) { const base = [...root(owner, runID), "processes"] - const ids = await Storage.scan(base, { strict: true }) + const ids = await Storage.scan(base) const result: RolloutSchema.ProcessRecord[] = [] for (const id of ids) result.push(RolloutSchema.ProcessRecord.parse(await Storage.read([...base, id]))) return result diff --git a/packages/harness/src/session/rollout/migration.ts b/packages/harness/src/session/rollout/migration.ts index 1057dabd3..ed9080128 100644 --- a/packages/harness/src/session/rollout/migration.ts +++ b/packages/harness/src/session/rollout/migration.ts @@ -31,7 +31,6 @@ export namespace RolloutMigration { requests: z.literal("historical_requests_not_recorded"), }) .strict() - const options = { private: true, durable: true, compact: true } as const async function retainedOutput(filepath: unknown) { if (typeof filepath !== "string") return undefined @@ -65,7 +64,7 @@ export namespace RolloutMigration { sessionID = Identifier.asSessionID(owner.sessionID) const infoKey = StoragePath.sessionInfo(scopeID, sessionID) const info = SettlementRecord.parse(await Storage.read(infoKey)) - for (const messageID of await Storage.scan(StoragePath.sessionMessagesRoot(scopeID, sessionID), { strict: true })) { + for (const messageID of await Storage.scan(StoragePath.sessionMessagesRoot(scopeID, sessionID))) { const mid = Identifier.asMessageID(messageID) const infoKey = StoragePath.messageInfo(scopeID, sessionID, mid) const raw = await Storage.read(infoKey) @@ -75,14 +74,10 @@ export namespace RolloutMigration { continue } if (parsed.data.role === "assistant" && !parsed.data.accounting) { - await Storage.write( - infoKey, - { ...parsed.data, accounting: { kind: "legacy", calculation: "session-v0" } }, - options, - ) + await Storage.write(infoKey, { ...parsed.data, accounting: { kind: "legacy", calculation: "session-v0" } }) audit.legacyMessages++ } else if (parsed.data.role === "assistant" && parsed.data.accounting?.kind === "legacy") audit.legacyMessages++ - for (const partID of await Storage.scan(StoragePath.messageParts(scopeID, sessionID, mid), { strict: true })) { + for (const partID of await Storage.scan(StoragePath.messageParts(scopeID, sessionID, mid))) { const partKey = StoragePath.messagePart(scopeID, sessionID, mid, Identifier.asPartID(partID)) const parsed = MessageV2.Part.safeParse(await Storage.read(partKey)) if (!parsed.success) { @@ -148,18 +143,18 @@ export namespace RolloutMigration { continue } Object.assign(attachment, captured) - await Storage.write(partKey, part, options) + await Storage.write(partKey, part) } else if (attachment.mime !== "application/x-directory") audit.missing.push(`attachment:${attachment.id}:original_not_recorded`) } - if (part !== parsed.data) await Storage.write(partKey, part, options) + if (part !== parsed.data) await Storage.write(partKey, part) } } if (info.cortex && !["queued", "running"].includes(info.cortex.status) && !info.cortex.settledAt) { - await Storage.write(infoKey, { ...info, cortex: { ...info.cortex, settledAt: audit.completedAt } }, options) + await Storage.write(infoKey, { ...info, cortex: { ...info.cortex, settledAt: audit.completedAt } }) audit.missing.push("cortex:historical_delivery_not_verified") } - await Storage.write(key, audit, options) + await Storage.write(key, audit) return audit } @@ -169,8 +164,8 @@ export namespace RolloutMigration { dependsOn: ["20260828-session-nav-timestamps"], async up(progress) { const owners: RolloutSchema.Owner[] = [] - for (const scopeID of await Storage.scan(["sessions"], { strict: true })) - for (const sessionID of await Storage.scan(["sessions", scopeID], { strict: true })) + for (const scopeID of await Storage.scan(["sessions"])) + for (const sessionID of await Storage.scan(["sessions", scopeID])) owners.push({ kind: "session", scopeID, sessionID }) for (let index = 0; index < owners.length; index++) { await session(owners[index]) diff --git a/packages/harness/src/session/rollout/pending.ts b/packages/harness/src/session/rollout/pending.ts index 5ee62e1dc..4179311c5 100644 --- a/packages/harness/src/session/rollout/pending.ts +++ b/packages/harness/src/session/rollout/pending.ts @@ -1,14 +1,11 @@ -import z from "zod" +import { z } from "zod" import { Storage } from "../../storage/storage" import { StoragePath } from "../../storage/path" -import { Lock } from "../../util/lock" import { RolloutSchema } from "./schema" import { record, RolloutRecordingError } from "./error" export namespace RolloutPending { type Document = { version: 1; owners: RolloutSchema.Owner[] } type Loaded = { kind: "absent" } | { kind: "ok"; owners: RolloutSchema.Owner[] } | { kind: "untrusted" } - const options = { compact: true, durable: true, private: true } as const - const lockKey = "rollout-recovery-pending" // Recovery-scoped suspension: while recovery settles owners, journal // writes must not consult the ledger. Listed owners are already listed, @@ -40,7 +37,7 @@ export namespace RolloutPending { raw = await Storage.read(key(), { silentNotFound: true }) } catch (error) { if (error instanceof Storage.NotFoundError) return { kind: "absent" } - return { kind: "untrusted" } + throw error } const full = z .object({ version: z.literal(1), owners: z.array(RolloutSchema.Owner) }) @@ -82,26 +79,28 @@ export namespace RolloutPending { // can establish that a missing ledger covers every historical owner. if (probe.kind === "absent") return if (probe.kind === "ok" && probe.owners.some((entry) => sameOwner(entry, identity))) return - await record(async () => { - using lock = await Lock.write(lockKey) - const current = await load() - // An unreadable ledger must not be silently reset: it may list other - // owners. Fail the recording so execution admission stops, and let the - // next exhaustive recovery re-arm the ledger. - if (current.kind === "untrusted") - throw new RolloutRecordingError({ message: "Rollout recovery pending set is unreadable" }) - if (current.kind === "ok" && current.owners.some((entry) => sameOwner(entry, identity))) return - if (current.kind === "absent") return - const owners = [...current.owners, identity] - await Storage.write(key(), { version: 1, owners }, options) - }) + await record(() => + Storage.transaction(async () => { + const current = await load() + // An unreadable ledger must not be silently reset: it may list other + // owners. Fail the recording so execution admission stops, and let the + // next exhaustive recovery re-arm the ledger. + if (current.kind === "untrusted") + throw new RolloutRecordingError({ message: "Rollout recovery pending set is unreadable" }) + if (current.kind === "ok" && current.owners.some((entry) => sameOwner(entry, identity))) return + if (current.kind === "absent") return + const owners = [...current.owners, identity] + await Storage.write(key(), { version: 1, owners }) + }), + ) } /** Re-arms the fast path after a verified recovery pass. */ export async function markClean(): Promise { - await record(async () => { - using lock = await Lock.write(lockKey) - await Storage.write(key(), { version: 1, owners: [] } satisfies Document, options) - }) + await record(() => + Storage.transaction(async () => { + await Storage.write(key(), { version: 1, owners: [] } satisfies Document) + }), + ) } } diff --git a/packages/harness/src/session/rollout/recovery.ts b/packages/harness/src/session/rollout/recovery.ts index 35a7d4908..fa6ee1493 100644 --- a/packages/harness/src/session/rollout/recovery.ts +++ b/packages/harness/src/session/rollout/recovery.ts @@ -81,8 +81,8 @@ export namespace RolloutRecovery { export async function* owners(onProgress?: () => void): AsyncGenerator { for (const category of ["sessions", "operations"] as const) { - for (const scopeID of await Storage.scan([category], { strict: true })) { - for (const id of await Storage.scan([category, scopeID], { strict: true })) { + for (const scopeID of await Storage.scan([category])) { + for (const id of await Storage.scan([category, scopeID])) { const identity: RolloutSchema.Owner = category === "sessions" ? { kind: "session", scopeID, sessionID: id } diff --git a/packages/harness/src/session/search-index.ts b/packages/harness/src/session/search-index.ts index f24dd8b8a..4a5848f2f 100644 --- a/packages/harness/src/session/search-index.ts +++ b/packages/harness/src/session/search-index.ts @@ -2,7 +2,6 @@ import { Identifier } from "../id/id" import { MessageV2 } from "./message-v2" import { Storage } from "../storage/storage" import { StoragePath } from "../storage/path" -import { Lock } from "../util/lock" import { Log } from "../util/log" /** @@ -229,72 +228,17 @@ export namespace SessionSearchIndex { return marker !== undefined } - /** - * Record that the session's searchable content changed. Callers invoke this - * BEFORE the content mutation persists (a crash between invalidation and the - * write costs only one extra scan) and again AFTER it (a scan that started - * before the mutation must see a marker newer than its scan start; see - * commitRebuild). When the marker write itself fails, the index record is - * dropped instead — under the same lock, so a concurrent commitRebuild - * cannot republish a clean record between the failed marker write and this - * removal. Never throws. - */ export async function markDirty(scopeID: Identifier.ScopeID, sessionID: Identifier.SessionID): Promise { - try { - using _ = await Lock.write(sessionLockKey(scopeID, sessionID)) - try { - await Storage.write(dirtyKey(scopeID, sessionID), { dirtyAt: Date.now() } satisfies DirtyMarker, { - compact: true, - }) - } catch (error) { - // The marker could not be persisted. A stale-but-clean record must not - // survive, so remove it while still holding the lock; if that also - // fails, the record is only ever read as absent on a later query when - // the session's messages are scanned anyway. - try { - await Storage.remove(recordKey(scopeID, sessionID)) - log.warn("marking session search dirty failed; dropped index record", { - scopeID, - sessionID, - error: String(error), - }) - } catch (recordError) { - log.warn("failed to mark session search dirty and drop stale record", { - scopeID, - sessionID, - error: String(error), - recordError: String(recordError), - }) - } - } - } catch (error) { - log.warn("failed to acquire session search index lock for markDirty", { - scopeID, - sessionID, - error: String(error), - }) - } + await Storage.write(dirtyKey(scopeID, sessionID), { dirtyAt: Date.now() } satisfies DirtyMarker) } - async function clearDirtyLocked(scopeID: Identifier.ScopeID, sessionID: Identifier.SessionID): Promise { - await Storage.remove(dirtyKey(scopeID, sessionID)) - } - - /** - * Persist a rebuilt record and clear the dirty marker — but only when no - * write landed after the rebuild started. `sinceMs` is the timestamp the - * caller began collecting; a marker newer than it means a mutation raced in - * during the rebuild, so the marker must survive to trigger another pass. - * Never throws. - */ export async function commitRebuild( scopeID: Identifier.ScopeID, sessionID: Identifier.SessionID, messages: IndexedMessage[], opts?: { sinceMs?: number }, ): Promise { - try { - using _ = await Lock.write(sessionLockKey(scopeID, sessionID)) + await Storage.transaction(async (tx) => { const record: SearchIndexRecord = { version: VERSION, tokenizerVersion: TOKENIZER_VERSION, @@ -303,29 +247,18 @@ export namespace SessionSearchIndex { updatedAt: Date.now(), messages, } - await Storage.write(recordKey(scopeID, sessionID), record, { compact: true }) - if (opts?.sinceMs === undefined) { - await clearDirtyLocked(scopeID, sessionID) - return - } - const marker = await Storage.read(dirtyKey(scopeID, sessionID), { - silentNotFound: true, - }).catch(() => undefined) - if (!marker || marker.dirtyAt <= opts.sinceMs) await clearDirtyLocked(scopeID, sessionID) - } catch (error) { - log.warn("failed to commit session search index", { scopeID, sessionID, error: String(error) }) - } + await tx.write(recordKey(scopeID, sessionID), record) + const [marker] = await tx.readMany([dirtyKey(scopeID, sessionID)]) + if (!marker || (opts?.sinceMs !== undefined && marker.dirtyAt < opts.sinceMs)) + await tx.remove(dirtyKey(scopeID, sessionID)) + }) } - /** Delete a session's index record and dirty marker. Never throws. */ export async function removeRecords(scopeID: Identifier.ScopeID, sessionID: Identifier.SessionID): Promise { - try { - using _ = await Lock.write(sessionLockKey(scopeID, sessionID)) - await Storage.remove(recordKey(scopeID, sessionID)) - await Storage.remove(dirtyKey(scopeID, sessionID)) - } catch (error) { - log.warn("failed to remove session search index", { scopeID, sessionID, error: String(error) }) - } + await Storage.transaction(async (tx) => { + await tx.remove(recordKey(scopeID, sessionID)) + await tx.remove(dirtyKey(scopeID, sessionID)) + }) } /** diff --git a/packages/harness/src/session/session-import.ts b/packages/harness/src/session/session-import.ts index f9b0c288d..5c244376c 100644 --- a/packages/harness/src/session/session-import.ts +++ b/packages/harness/src/session/session-import.ts @@ -1,3 +1,4 @@ +import { SessionStaging } from "./staging" import { SessionSchemaRegistry } from "./schema-registry" import { SnapshotLifecycle } from "./snapshot-lifecycle" import { SnapshotRecords } from "./snapshot-records" @@ -152,7 +153,7 @@ export namespace SessionImport { export async function fromReport( report: SessionExport.Report, - options: { sessionIDs?: Map; rollout?: boolean } = {}, + options: { sessionIDs?: Map; rollout?: boolean; stagingID?: string } = {}, ): Promise { if (report.sessions.length === 0) throw new Error("Session import report does not contain any sessions") @@ -186,83 +187,110 @@ export namespace SessionImport { const imported: ImportedSession[] = [] let messageCount = 0 - for (const data of ordered) { - const sessionID = idMap.get(data.info.id)! - const parentID = data.info.parentID ? idMap.get(data.info.parentID) : undefined - const info = normalizeSessionInfo({ info: data.info, sessionID, parentID, scope, idMap }) - - await Session.create({ - scope, - id: sessionID, - parentID, - title: info.title, - permission: info.permission, - controlProfile: info.controlProfile, - preAuthorizedActions: info.preAuthorizedActions, - interaction: info.interaction, - cortex: info.cortex, - workspace: info.workspace, - forkedFrom: info.forkedFrom, - completionNotice: info.completionNotice, - }) - await writeSessionInfo(scopeID, info) - const snapshots = await SnapshotLifecycle.adopt({ - scopeID: scope.id, - sourceSessionID: data.info.id, - targetSessionID: sessionID, - workspace: info.workspace?.path ?? ScopeContext.current.directory, - hashes: data.messages.flatMap((message) => message.parts.flatMap(SnapshotRecords.partRoots)), - allowMissing: true, - }) - if (snapshots.missing.length) - warnings.push( - `Imported session has ${snapshots.missing.length} unavailable file snapshots; JSON exports do not contain file objects.`, - ) - - for (const message of data.messages) { - const nextMessage = await remapMessage(message.info, sessionID, idMap) - await Session.updateMessage(nextMessage) - messageCount++ - - for (const part of message.parts) { - await Session.updatePart(remapPart(part, sessionID, message.info.id, idMap, options.rollout)) + const stagingID = options.stagingID ?? (await SessionStaging.begin(scope.id, [...idMap.values()])) + try { + const prepared: Array<{ + data: SessionExport.SessionData + sessionID: string + parentID: string | undefined + info: Session.Info + messages: MessageV2.WithParts[] + }> = [] + for (const data of ordered) { + const sessionID = idMap.get(data.info.id)! + const parentID = data.info.parentID ? idMap.get(data.info.parentID) : undefined + const info = normalizeSessionInfo({ info: data.info, sessionID, parentID, scope, idMap }) + if (!options.rollout) { + const snapshots = await SnapshotLifecycle.adopt({ + scopeID: scope.id, + sourceSessionID: data.info.id, + targetSessionID: sessionID, + workspace: info.workspace?.path ?? ScopeContext.current.directory, + hashes: data.messages.flatMap((message) => message.parts.flatMap(SnapshotRecords.partRoots)), + allowMissing: true, + }) + if (snapshots.missing.length) + warnings.push( + `Imported session has ${snapshots.missing.length} unavailable file snapshots; JSON exports do not contain file objects.`, + ) } + const messages = [] + for (const message of data.messages) { + const parts = [] + for (const part of message.parts) + parts.push( + await Session.preparePart(remapPart(part, sessionID, message.info.id, idMap, options.rollout), scope.id), + ) + messages.push({ info: await remapMessage(message.info, sessionID, idMap), parts }) + } + prepared.push({ data, sessionID, parentID, info, messages }) } + return await Storage.transaction(async () => { + for (const { data, sessionID, parentID, info, messages } of prepared) { + await Session.create({ + scope, + id: sessionID, + parentID, + title: info.title, + permission: info.permission, + controlProfile: info.controlProfile, + preAuthorizedActions: info.preAuthorizedActions, + interaction: info.interaction, + cortex: info.cortex, + workspace: info.workspace, + forkedFrom: info.forkedFrom, + completionNotice: info.completionNotice, + }) + await writeSessionInfo(scopeID, info) + for (const message of messages) { + await Session.updateMessage(message.info) + messageCount++ + + for (const part of message.parts) { + await Session.updatePart(part) + } + } - const dag = Dag.normalizeRetiredAssignments( - data.dag.map((node) => ({ - ...node, - session_id: node.session_id ? (idMap.get(node.session_id) ?? node.session_id) : undefined, - })), - ) - if (dag.length > 0) await Dag.update({ sessionID, nodes: dag }) - if (data.todos.length > 0) await Todo.update({ sessionID, todos: data.todos }) - if (data.diffs.length > 0) { - await Storage.write( - StoragePath.sessionSummary(scopeID, Identifier.asSessionID(sessionID)), - SnapshotSchema.boundArray(data.diffs), - ) - } + const dag = Dag.normalizeRetiredAssignments( + data.dag.map((node) => ({ + ...node, + session_id: node.session_id ? (idMap.get(node.session_id) ?? node.session_id) : undefined, + })), + ) + if (dag.length > 0) await Dag.update({ sessionID, nodes: dag }) + if (data.todos.length > 0) await Todo.update({ sessionID, todos: data.todos }) + if (data.diffs.length > 0) { + await Storage.write( + StoragePath.sessionSummary(scopeID, Identifier.asSessionID(sessionID)), + SnapshotSchema.boundArray(data.diffs), + ) + } - imported.push({ sourceSessionID: data.info.id, session: info }) - } + imported.push({ sourceSessionID: data.info.id, session: info }) + } - const navIndex = await SessionNav.buildNavIndex(scope.id) - for (const item of imported) { - Bus.publish(SessionEvent.Updated, { - info: await Session.withRuntimeInfo(item.session), - navEntry: navIndex.entries.find((entry) => entry.id === item.session.id), - }) - } + const navIndex = await SessionNav.buildNavIndex(scope.id) + for (const item of imported) { + Bus.publish(SessionEvent.Updated, { + info: await Session.withRuntimeInfo(item.session), + navEntry: navIndex.entries.find((entry) => entry.id === item.session.id), + }) + } - const rootSessionID = idMap.get(report.rootSessionID) ?? imported[0]?.session.id - if (!rootSessionID) throw new Error("Session import did not create a root session") - return { - rootSessionID, - sessions: imported, - sessionCount: imported.length, - messageCount, - warnings, + const rootSessionID = idMap.get(report.rootSessionID) ?? imported[0]?.session.id + if (!rootSessionID) throw new Error("Session import did not create a root session") + await SessionStaging.finish(stagingID) + return { + rootSessionID, + sessions: imported, + sessionCount: imported.length, + messageCount, + warnings, + } + }) + } catch (error) { + await SessionStaging.discard(stagingID) + throw error } } diff --git a/packages/harness/src/session/snapshot-archive.ts b/packages/harness/src/session/snapshot-archive.ts index 55ae497db..b3c00939b 100644 --- a/packages/harness/src/session/snapshot-archive.ts +++ b/packages/harness/src/session/snapshot-archive.ts @@ -143,13 +143,17 @@ export namespace SnapshotArchive { return refs } - async function mergeRepository(source: string, target: string) { + async function mergeRepository(source: string, target: string, skipped: ReadonlySet = new Set()) { await SnapshotGit.checked(source, ["fsck", "--full"]) await SnapshotStore.initializeBareRepository(target) if (await Bun.file(path.join(target, "objects", "info", "alternates")).exists()) throw new SnapshotStore.StorageError("Destination snapshot store is not self-contained") await SnapshotGit.checked(target, ["fsck", "--full"]) - const incoming = await references(source) + const incoming = new Map( + [...(await references(source))].filter( + ([ref]) => !skipped.has(/^refs\/synergy\/snapshots\/([^/]+)\//.exec(ref)?.[1] ?? ""), + ), + ) const existing = await references(target) for (const [ref, oid] of incoming) { if (existing.has(ref) && existing.get(ref) !== oid) @@ -176,7 +180,11 @@ export namespace SnapshotArchive { // An archive is independent of the source home, including legacy alternates. // Provenance: https://git-scm.com/docs/git-index-pack (--strict and --keep). - export async function merge(sourceData: string, targetData: string) { + export async function merge( + sourceData: string, + targetData: string, + options: { metadata?: boolean; skipped?: ReadonlySet } = {}, + ) { if (path.resolve(sourceData) === path.resolve(targetData)) throw new SnapshotStore.StorageError("Snapshot merge source equals destination") const sourceV2 = path.join(sourceData, "snapshot-v2") @@ -192,7 +200,7 @@ export namespace SnapshotArchive { const to = path.join(targetV2, scope.name) const owners = new Map>() for (const entry of await SnapshotRecords.entries(path.join(from, "owners"))) { - if (!entry.isFile() || !entry.name.endsWith(".json")) continue + if (!entry.isFile() || !entry.name.endsWith(".json") || options.skipped?.has(entry.name.slice(0, -5))) continue SnapshotStore.component(entry.name.slice(0, -5)) const incoming = SnapshotStore.Owner.parse(await read(path.join(from, "owners", entry.name))) const value = await read(path.join(to, "owners", entry.name)) @@ -206,28 +214,31 @@ export namespace SnapshotArchive { const marker = await read(path.join(from, "repository.json")) if (marker !== undefined) z.object({ version: z.literal(2), objectFormat: z.literal("sha1") }).parse(marker) if (await Bun.file(path.join(from, "store.git", "HEAD")).exists()) { - await mergeRepository(path.join(from, "store.git"), path.join(to, "store.git")) - await write(path.join(to, "repository.json"), { version: 2, objectFormat: "sha1" }) + await mergeRepository(path.join(from, "store.git"), path.join(to, "store.git"), options.skipped) + if (options.metadata !== false) + await write(path.join(to, "repository.json"), { version: 2, objectFormat: "sha1" }) } else if (marker !== undefined || [...owners.values()].some((owner) => owner.backend === "shared")) throw new SnapshotStore.StorageError("Archive is missing its shared snapshot object store") - for (const directory of ["migrations", "deletions"]) - await copyMetadata(path.join(from, directory), path.join(to, directory)) - for (const [name, owner] of owners) await write(path.join(to, "owners", name), owner) + if (options.metadata !== false) { + for (const directory of ["migrations", "deletions"]) + await copyMetadata(path.join(from, directory), path.join(to, directory)) + for (const [name, owner] of owners) await write(path.join(to, "owners", name), owner) + } } for (const scope of await SnapshotRecords.entries(path.join(sourceData, "snapshot"))) { if (!scope.isDirectory()) continue const sourceScope = path.join(sourceData, "snapshot", scope.name) if (await Bun.file(path.join(sourceScope, "HEAD")).exists()) { - await mergeRepository(sourceScope, path.join(targetData, "snapshot", scope.name)) + await mergeRepository(sourceScope, path.join(targetData, "snapshot", scope.name), options.skipped) continue } for (const entry of await SnapshotRecords.entries(sourceScope)) { - if (!entry.isDirectory()) continue + if (!entry.isDirectory() || options.skipped?.has(entry.name)) continue const from = path.join(sourceScope, entry.name) if (!(await Bun.file(path.join(from, "HEAD")).exists())) continue const to = path.join(targetData, "snapshot", scope.name, entry.name) await mergeRepository(from, to) - if (!entry.name.startsWith(".") && !scope.name.startsWith(".")) { + if (options.metadata !== false && !entry.name.startsWith(".") && !scope.name.startsWith(".")) { SnapshotStore.component(entry.name) const owner = path.join(targetV2, scope.name, "owners", entry.name + ".json") if ((await read(owner)) === undefined) await write(owner, { version: 2, backend: "legacy" }) diff --git a/packages/harness/src/session/snapshot-lease.ts b/packages/harness/src/session/snapshot-lease.ts index d2cc1519f..66ee59b47 100644 --- a/packages/harness/src/session/snapshot-lease.ts +++ b/packages/harness/src/session/snapshot-lease.ts @@ -5,7 +5,6 @@ import { withFileLock } from "@ericsanchezok/synergy-util/fs-lock" import { processStartIdentity } from "@ericsanchezok/synergy-util/process-identity" import { Storage } from "../storage/storage" import { StoragePath } from "../storage/path" -import { Global } from "../global" export namespace SnapshotLease { const Owner = z.object({ @@ -24,7 +23,7 @@ export namespace SnapshotLease { } } - export function directory(dataRoot = Global.Path.data) { + export function directory(dataRoot = Storage.current().artifactDirectory) { return path.join(dataRoot, "snapshot-v2", ".locks") } @@ -84,7 +83,7 @@ export namespace SnapshotLease { // process start identities, not lease age, determine abandoned ownership. export async function acquire(scopeID: string, exclusive: boolean, options: Options = {}) { if (!/^[a-zA-Z0-9_-]+$/.test(scopeID)) throw new Error("Invalid snapshot lease Scope") - const dataRoot = options.dataRoot ?? Global.Path.data + const dataRoot = options.dataRoot ?? Storage.current().artifactDirectory const home = await admit("", false, { ...options, dataRoot }) try { const scope = await admit(scopeID, exclusive, { ...options, dataRoot }) @@ -108,7 +107,7 @@ export namespace SnapshotLease { } async function admit(scopeID: string, exclusive: boolean, options: Options) { - const dataRoot = options.dataRoot ?? Global.Path.data + const dataRoot = options.dataRoot ?? Storage.current().artifactDirectory const owner: Owner = { token: randomUUID(), pid: process.pid, diff --git a/packages/harness/src/session/snapshot-lifecycle.ts b/packages/harness/src/session/snapshot-lifecycle.ts index c08b7d2a2..5f4a3241a 100644 --- a/packages/harness/src/session/snapshot-lifecycle.ts +++ b/packages/harness/src/session/snapshot-lifecycle.ts @@ -2,7 +2,6 @@ import fs from "node:fs/promises" import path from "node:path" import { z } from "zod" import { withFileLock } from "@ericsanchezok/synergy-util/fs-lock" -import { Global } from "../global" import { Identifier } from "../id/id" import { Storage } from "../storage/storage" import { StoragePath } from "../storage/path" @@ -10,7 +9,6 @@ import { SnapshotStore } from "./snapshot-store" import { SnapshotLease } from "./snapshot-lease" import { SnapshotTransfer } from "./snapshot-transfer" import { SnapshotGit } from "./snapshot-git" -import { SnapshotRecords } from "./snapshot-records" export namespace SnapshotLifecycle { const Deletion = z.object({ version: z.literal(2), backend: z.enum(["legacy", "shared"]) }) @@ -82,6 +80,12 @@ export namespace SnapshotLifecycle { export async function beginDelete(scopeID: string, sessionID: string) { return locked(scopeID, [sessionID], async () => { + await scheduleDelete(scopeID, sessionID) + }) + } + + export async function scheduleDelete(scopeID: string, sessionID: string) { + return Storage.transaction(async () => { const key = StoragePath.snapshotDeletion(scopeID, sessionID) const previous = await SnapshotStore.optional(key) if (previous !== undefined) Deletion.parse(previous) @@ -106,17 +110,14 @@ export namespace SnapshotLifecycle { if (stored === undefined) return const job = Deletion.parse(stored) const canonical = path.join( - Global.Path.data, + Storage.current().artifactDirectory, ...StoragePath.sessionRoot(Identifier.asScopeID(scopeID), Identifier.asSessionID(sessionID)), ) - const exists = await fs.lstat(canonical).then( - () => true, - (error: NodeJS.ErrnoException) => { - if (error.code === "ENOENT") return false - throw error - }, + const remaining = await Storage.list( + StoragePath.sessionRoot(Identifier.asScopeID(scopeID), Identifier.asSessionID(sessionID)), ) - if (exists) throw new SnapshotStore.StorageError("Cannot release snapshots before permanent session deletion") + if (remaining.length) + throw new SnapshotStore.StorageError("Cannot release snapshots before permanent session deletion") const repo = SnapshotStore.repository(scopeID) if (await Bun.file(path.join(repo, "HEAD")).exists()) { const prefix = `refs/synergy/snapshots/${SnapshotStore.component(sessionID)}/` @@ -140,18 +141,17 @@ export namespace SnapshotLifecycle { if (job.backend === "legacy") await fs.rm(SnapshotStore.legacyRepository(scopeID, sessionID), { recursive: true, force: true }) await fs.rm(SnapshotStore.cache(scopeID, sessionID), { recursive: true, force: true }) + await fs.rm(canonical, { recursive: true, force: true }) await Storage.remove(StoragePath.snapshotMigration(scopeID, sessionID)) await Storage.remove(key) }) } export async function recover(scopeID: string) { - const jobs = await SnapshotRecords.entries(path.join(SnapshotStore.root(scopeID), "deletions")) - if (!jobs.length) return + const jobs = await Storage.scan(["snapshot-v2", scopeID, "deletions"]) const { SessionRecovery } = await import("./recovery") - for (const entry of jobs) { - if (!entry.isFile() || !entry.name.endsWith(".json")) continue - const report = await SessionRecovery.remove({ scopeID, sessionID: entry.name.slice(0, -5) }) + for (const sessionID of jobs) { + const report = await SessionRecovery.remove({ scopeID, sessionID }) if (report.errors.length) throw new SnapshotStore.StorageError("Snapshot deletion recovery is incomplete") } } diff --git a/packages/harness/src/session/snapshot-maintenance.ts b/packages/harness/src/session/snapshot-maintenance.ts index ecbcbf4f5..a1af1fc86 100644 --- a/packages/harness/src/session/snapshot-maintenance.ts +++ b/packages/harness/src/session/snapshot-maintenance.ts @@ -1,7 +1,6 @@ import fs from "node:fs/promises" import path from "node:path" import { z } from "zod" -import { Global } from "../global" import { Storage } from "../storage/storage" import { StoragePath } from "../storage/path" import { Identifier } from "../id/id" @@ -35,8 +34,13 @@ export namespace SnapshotMaintenance { const { entries, historicalRoots } = SnapshotRecords export async function scopes() { - const result = new Set() - for (const dir of [Global.Path.snapshot, path.join(Global.Path.data, "snapshot-v2")]) { + const result = new Set( + (await Storage.scan(["snapshot-v2"])).filter((id) => id !== "format" && id !== "leases"), + ) + for (const dir of [ + path.join(Storage.current().artifactDirectory, "snapshot"), + path.join(Storage.current().artifactDirectory, "snapshot-v2"), + ]) { for (const entry of await entries(dir)) if (entry.isDirectory() && /^[a-zA-Z0-9_-]+$/.test(entry.name)) result.add(entry.name) } @@ -48,9 +52,16 @@ export namespace SnapshotMaintenance { let done = 0 for (const scopeID of ids) { await SnapshotLease.use(scopeID, true, async () => { - for (const entry of await entries(path.join(Global.Path.snapshot, scopeID))) { + for (const entry of await entries( + path.join(path.join(Storage.current().artifactDirectory, "snapshot"), scopeID), + )) { if (!entry.isDirectory() || !/^[a-zA-Z0-9_-]+$/.test(entry.name)) continue - if (!(await Bun.file(path.join(Global.Path.snapshot, scopeID, entry.name, "HEAD")).exists())) continue + if ( + !(await Bun.file( + path.join(path.join(Storage.current().artifactDirectory, "snapshot"), scopeID, entry.name, "HEAD"), + ).exists()) + ) + continue if (await SnapshotStore.owner(scopeID, entry.name)) continue await SnapshotStore.write(StoragePath.snapshotOwner(scopeID, entry.name), { version: 2, @@ -80,7 +91,12 @@ export namespace SnapshotMaintenance { for (const sessionID of await ownerIDs(scopeID)) { const owner = await SnapshotStore.owner(scopeID, sessionID) if (owner?.backend !== "legacy") continue - if (!(await Bun.file(path.join(Global.Path.snapshot, scopeID, sessionID, "HEAD")).exists())) continue + if ( + !(await Bun.file( + path.join(path.join(Storage.current().artifactDirectory, "snapshot"), scopeID, sessionID, "HEAD"), + ).exists()) + ) + continue if (await SnapshotStore.optional(StoragePath.snapshotMigration(scopeID, sessionID))) continue const info = await SnapshotStore.optional( StoragePath.sessionInfo(Identifier.asScopeID(scopeID), Identifier.asSessionID(sessionID)), @@ -112,10 +128,7 @@ export namespace SnapshotMaintenance { } async function ownerIDs(scopeID: string) { - return (await entries(path.join(Global.Path.data, ...StoragePath.snapshotOwners(scopeID)))) - .filter((entry) => entry.isFile() && entry.name.endsWith(".json")) - .map((entry) => entry.name.slice(0, -5)) - .sort() + return Storage.scan(StoragePath.snapshotOwners(scopeID)) } export async function inspect(scopeID?: string) { @@ -126,21 +139,26 @@ export namespace SnapshotMaintenance { const owner = await SnapshotStore.owner(id, sessionID) if (owner) owners[owner.backend]++ } - const legacy = await statistics(path.join(Global.Path.snapshot, id)) + const legacy = await statistics(path.join(path.join(Storage.current().artifactDirectory, "snapshot"), id)) const shared = await statistics(SnapshotStore.repository(id)) const indexes = await statistics(SnapshotStore.cache(id)) const retainedLegacy = { unowned: 0, reclaimed: 0, sharedBaselines: 0, unregistered: 0 } - for (const entry of await entries(path.join(Global.Path.snapshot, id))) { + for (const entry of await entries(path.join(path.join(Storage.current().artifactDirectory, "snapshot"), id))) { if (!entry.isDirectory()) continue if (entry.name === ".shared.old") { retainedLegacy.sharedBaselines++ continue } if (!/^[a-zA-Z0-9_-]+$/.test(entry.name)) continue - if (!(await Bun.file(path.join(Global.Path.snapshot, id, entry.name, "HEAD")).exists())) continue + if ( + !(await Bun.file( + path.join(path.join(Storage.current().artifactDirectory, "snapshot"), id, entry.name, "HEAD"), + ).exists()) + ) + continue if (!(await SnapshotStore.owner(id, entry.name))) retainedLegacy.unregistered++ if (id === "__reclaimed__") retainedLegacy.reclaimed++ - else if (!(await Bun.file(path.join(Global.Path.data, "sessions", id, entry.name, "info.json")).exists())) + else if ((await SnapshotStore.optional(["sessions", id, entry.name, "info"])) === undefined) retainedLegacy.unowned++ } result.push({ scopeID: id, owners, retainedLegacy, legacy, shared, indexes }) @@ -165,9 +183,7 @@ export namespace SnapshotMaintenance { issues.push(error instanceof Error ? error.message : String(error)) } } - const sessions = (await entries(path.join(Global.Path.data, "sessions", scopeID))) - .filter((entry) => entry.isDirectory()) - .map((entry) => entry.name) + const sessions = await Storage.scan(["sessions", scopeID]) for (const sessionID of new Set([...(await ownerIDs(scopeID)), ...sessions])) { signal?.throwIfAborted() const owner = await SnapshotStore.owner(scopeID, sessionID) @@ -331,9 +347,8 @@ export namespace SnapshotMaintenance { if (packs.some((entry) => entry.name.endsWith(".keep"))) throw new SnapshotStore.StorageError("Snapshot packs have unresolved import protection") for (const dir of ["migrations", "deletions"]) { - for (const entry of await entries(path.join(SnapshotStore.root(scopeID), dir))) { - if (!entry.isFile() || !entry.name.endsWith(".json")) continue - const record = await Storage.read(["snapshot-v2", scopeID, dir, entry.name.slice(0, -5)]) + for (const id of await Storage.scan(["snapshot-v2", scopeID, dir])) { + const record = await Storage.read(["snapshot-v2", scopeID, dir, id]) if (dir === "deletions" || Journal.parse(record).phase !== "cleaned") throw new SnapshotStore.StorageError("Snapshot maintenance has unfinished recovery work") } @@ -391,7 +406,9 @@ export namespace SnapshotMaintenance { if (info !== undefined) return undefined return { sessionID, - bytes: (await statistics(path.join(Global.Path.snapshot, scopeID, sessionID))).bytes, + bytes: ( + await statistics(path.join(path.join(Storage.current().artifactDirectory, "snapshot"), scopeID, sessionID)) + ).bytes, reason: scopeID === "__reclaimed__" ? "reclaimed" : "unowned", } } @@ -420,9 +437,16 @@ export namespace SnapshotMaintenance { skippedProtected: 0, errors: [], } - for (const entry of await entries(path.join(Global.Path.snapshot, scopeID))) { + for (const entry of await entries( + path.join(path.join(Storage.current().artifactDirectory, "snapshot"), scopeID), + )) { if (!entry.isDirectory() || !/^[a-zA-Z0-9_-]+$/.test(entry.name)) continue - if (!(await Bun.file(path.join(Global.Path.snapshot, scopeID, entry.name, "HEAD")).exists())) continue + if ( + !(await Bun.file( + path.join(path.join(Storage.current().artifactDirectory, "snapshot"), scopeID, entry.name, "HEAD"), + ).exists()) + ) + continue const candidate = await cleanCandidate(scopeID, entry.name) if (candidate) result.candidates.push(candidate) else result.skippedProtected++ @@ -434,10 +458,13 @@ export namespace SnapshotMaintenance { for (const candidate of result.candidates) { options.signal?.throwIfAborted() try { - await fs.rm(path.join(Global.Path.snapshot, scopeID, candidate.sessionID), { - recursive: true, - force: true, - }) + await fs.rm( + path.join(path.join(Storage.current().artifactDirectory, "snapshot"), scopeID, candidate.sessionID), + { + recursive: true, + force: true, + }, + ) await fs.rm(SnapshotStore.cache(scopeID, candidate.sessionID), { recursive: true, force: true }) result.removed++ result.bytes += candidate.bytes diff --git a/packages/harness/src/session/snapshot-records.ts b/packages/harness/src/session/snapshot-records.ts index fb2896152..df887f628 100644 --- a/packages/harness/src/session/snapshot-records.ts +++ b/packages/harness/src/session/snapshot-records.ts @@ -1,9 +1,5 @@ import fs from "node:fs/promises" -import path from "node:path" -import { Global } from "../global" import { Storage } from "../storage/storage" -import { StoragePath } from "../storage/path" -import { Identifier } from "../id/id" import { SnapshotStore } from "./snapshot-store" export namespace SnapshotRecords { @@ -31,17 +27,8 @@ export namespace SnapshotRecords { export async function historicalRoots(scopeID: string, sessionID: string) { const roots = new Set() - const sid = Identifier.asSessionID(sessionID) - const scope = Identifier.asScopeID(scopeID) - const messages = StoragePath.sessionMessagesRoot(scope, sid) - for (const message of await entries(path.join(Global.Path.data, ...messages))) { - if (!message.isDirectory()) continue - const parts = StoragePath.messageParts(scope, sid, Identifier.asMessageID(message.name)) - for (const part of await entries(path.join(Global.Path.data, ...parts))) { - if (!part.isFile() || !part.name.endsWith(".json")) continue - const value = await Storage.read([...parts, part.name.slice(0, -5)]) - for (const hash of partRoots(value)) roots.add(hash) - } + for await (const record of Storage.records({ kind: "part", scopeID, sessionID })) { + for (const hash of partRoots(record.value)) roots.add(hash) } return [...roots] } diff --git a/packages/harness/src/session/snapshot-store.ts b/packages/harness/src/session/snapshot-store.ts index 8dc47934a..46309d5ae 100644 --- a/packages/harness/src/session/snapshot-store.ts +++ b/packages/harness/src/session/snapshot-store.ts @@ -39,7 +39,7 @@ export namespace SnapshotStore { } export function root(scopeID: string) { - return path.join(Global.Path.data, "snapshot-v2", component(scopeID)) + return path.join(Storage.current().artifactDirectory, "snapshot-v2", component(scopeID)) } export function repository(scopeID: string) { @@ -47,13 +47,18 @@ export namespace SnapshotStore { } export function legacyRepository(scopeID: string, sessionID: string) { - return path.join(Global.Path.snapshot, component(scopeID), component(sessionID)) + return path.join( + path.join(Storage.current().artifactDirectory, "snapshot"), + component(scopeID), + component(sessionID), + ) } export function cache(scopeID: string, sessionID?: string) { return path.join( Global.Path.cache, "snapshot-index", + createHash("sha256").update(Storage.current().artifactDirectory).digest("hex").slice(0, 16), component(scopeID), ...(sessionID ? [component(sessionID)] : []), ) @@ -67,7 +72,7 @@ export namespace SnapshotStore { } export function write(key: string[], value: T) { - return Storage.write(key, value, { durable: true }) + return Storage.write(key, value) } export async function owner(scopeID: string, sessionID: string) { @@ -139,12 +144,15 @@ export namespace SnapshotStore { return } await initializeRepository(operation.scopeID) - if (!(await owner(operation.scopeID, operation.sessionID))) { - await write(StoragePath.snapshotOwner(operation.scopeID, operation.sessionID), { - version: 2, - backend: "shared", - } satisfies Owner) - } + await Storage.transaction(async () => { + const current = await owner(operation.scopeID, operation.sessionID) + if (current?.backend === "deleted") throw new StorageError("Session snapshot ownership was deleted") + if (!current) + await write(StoragePath.snapshotOwner(operation.scopeID, operation.sessionID), { + version: 2, + backend: "shared", + } satisfies Owner) + }) }, ) } diff --git a/packages/harness/src/session/snapshot-transfer.ts b/packages/harness/src/session/snapshot-transfer.ts index f1e00f698..cfba484bb 100644 --- a/packages/harness/src/session/snapshot-transfer.ts +++ b/packages/harness/src/session/snapshot-transfer.ts @@ -1,3 +1,4 @@ +import { initializeSqliteEngine } from "../storage/sqlite-engine" import path from "node:path" import fs from "node:fs/promises" import { Database } from "bun:sqlite" @@ -5,6 +6,8 @@ import { Global } from "../global" import { SnapshotGit } from "./snapshot-git" import { SnapshotStore } from "./snapshot-store" +initializeSqliteEngine() + export namespace SnapshotTransfer { export async function recoverImports(target: string, signal?: AbortSignal) { const packs = path.join(target, "objects", "pack") diff --git a/packages/harness/src/session/staging.ts b/packages/harness/src/session/staging.ts new file mode 100644 index 000000000..b4c5fef09 --- /dev/null +++ b/packages/harness/src/session/staging.ts @@ -0,0 +1,63 @@ +import { randomUUID } from "node:crypto" +import { z } from "zod" +import { Storage } from "../storage/storage" +import { SnapshotLifecycle } from "./snapshot-lifecycle" +import { StorageIntegrityError } from "../storage/errors" + +export namespace SessionStaging { + const Job = z + .object({ version: z.literal(1), scopeID: z.string(), sessionIDs: z.array(z.string()).min(1), created: z.number() }) + .strict() + export async function begin(scopeID: string, sessionIDs: string[]) { + const id = randomUUID() + const job = Job.parse({ version: 1, scopeID, sessionIDs: [...new Set(sessionIDs)], created: Date.now() }) + await Storage.transaction(async () => { + for (const key of await Storage.list(["storage_staging"])) { + const pending = Job.parse(await Storage.read(key)) + if (pending.sessionIDs.some((sessionID) => job.sessionIDs.includes(sessionID))) + throw new StorageIntegrityError("A staged session ID is reserved by another import") + } + for (const sessionID of job.sessionIDs) { + if ( + ( + await Storage.readMany([ + ["session_index", sessionID], + ["sessions", scopeID, sessionID, "info"], + ]) + ).some((record) => record !== undefined) + ) + throw new StorageIntegrityError("A staged session ID already belongs to existing data") + } + await Storage.write(["storage_staging", id], job) + }) + return id + } + + export async function finish(id: string) { + if (!Storage.inTransaction()) + throw new StorageIntegrityError("Staged sessions must become visible in their final business transaction") + const job = Job.parse(await Storage.read(["storage_staging", id])) + for (const sessionID of job.sessionIDs) await Storage.read(["sessions", job.scopeID, sessionID, "info"]) + await Storage.remove(["storage_staging", id]) + } + + export async function discard(id: string) { + const [raw] = await Storage.readMany([["storage_staging", id]]) + if (!raw) return + const job = Job.parse(raw) + await Storage.transaction(async () => { + for (const sessionID of job.sessionIDs) { + if ((await Storage.readMany([["sessions", job.scopeID, sessionID, "info"]]))[0] !== undefined) + throw new StorageIntegrityError("Cannot discard a staging job containing a published session") + await SnapshotLifecycle.scheduleDelete(job.scopeID, sessionID) + await Storage.removeTree(["sessions", job.scopeID, sessionID]) + } + }) + for (const sessionID of job.sessionIDs) await SnapshotLifecycle.completeDelete(job.scopeID, sessionID) + await Storage.remove(["storage_staging", id]) + } + + export async function recover() { + for (const id of await Storage.scan(["storage_staging"])) await discard(id) + } +} diff --git a/packages/harness/src/session/todo.ts b/packages/harness/src/session/todo.ts index f26a89203..6b8e5909a 100644 --- a/packages/harness/src/session/todo.ts +++ b/packages/harness/src/session/todo.ts @@ -36,9 +36,11 @@ export namespace Todo { } export async function update(input: { sessionID: string; todos: Info[] }) { - const scopeID = await resolveScopeID(input.sessionID) - await Storage.write(StoragePath.sessionTodo(scopeID, asSessionID(input.sessionID)), input.todos) - Bus.publish(Event.Updated, input) + return Storage.transaction(async () => { + const scopeID = await resolveScopeID(input.sessionID) + await Storage.write(StoragePath.sessionTodo(scopeID, asSessionID(input.sessionID)), input.todos) + Bus.publish(Event.Updated, input) + }) } export async function get(sessionID: string) { diff --git a/packages/harness/src/session/user-message-materialization.ts b/packages/harness/src/session/user-message-materialization.ts index 3a0d1bb9d..2a7338d4a 100644 --- a/packages/harness/src/session/user-message-materialization.ts +++ b/packages/harness/src/session/user-message-materialization.ts @@ -1,3 +1,4 @@ +import { Storage } from "../storage/storage" import { SessionPluginHooks } from "./plugin-hooks" import { Log } from "../util/log" import { MessageV2 } from "./message-v2" @@ -26,16 +27,39 @@ function observerInput(message: MessageV2.WithParts) { } export namespace SessionUserMessageMaterialization { + export interface CommitOptions { + commit?(message: MessageV2.WithParts): Promise + } export const input = observerInput - export async function write(message: { - info: Info - parts: MessageV2.Part[] - }): Promise<{ info: Info; parts: MessageV2.Part[] }> { + export async function write( + message: { + info: Info + parts: MessageV2.Part[] + }, + options: CommitOptions = {}, + ): Promise<{ info: Info; parts: MessageV2.Part[] }> { const { Session } = await import(".") - const info = (await Session.updateMessage(message.info)) as Info - for (const part of message.parts) await Session.updatePart(part) - after({ info, parts: message.parts }) - return { info, parts: message.parts } + const prepared: MessageV2.Part[] = [] + for (const part of message.parts) prepared.push(await Session.preparePart(part)) + return Storage.transaction(async () => { + const existing = await MessageV2.get({ sessionID: message.info.sessionID, messageID: message.info.id }).catch( + (error) => { + if (error instanceof Storage.NotFoundError) return + throw error + }, + ) + if (existing) { + await options.commit?.(existing) + return existing as { info: Info; parts: MessageV2.Part[] } + } + const info = (await Session.updateMessage(message.info)) as Info + const parts = [] + for (const part of prepared) parts.push(await Session.updatePart(part)) + const result = { info, parts } + await options.commit?.(result) + Storage.afterCommit(() => after(result)) + return result + }) } export function after(message: MessageV2.WithParts) { diff --git a/packages/harness/src/storage/atomic-file.ts b/packages/harness/src/storage/atomic-file.ts new file mode 100644 index 000000000..c2c777f6d --- /dev/null +++ b/packages/harness/src/storage/atomic-file.ts @@ -0,0 +1,81 @@ +import path from "node:path" +import fs from "node:fs/promises" +import { isRetryableIOError } from "../util/io-retry" + +export namespace AtomicFile { + export interface WriteOptions { + durable?: boolean + private?: boolean + } + // Windows maps rename onto MoveFileEx: when another process (antivirus, + // OneDrive, a cross-process reader of these JSON files) briefly holds a + // handle on the source or target without FILE_SHARE_DELETE, the rename + // fails with EPERM/EACCES. Sharing violations clear within milliseconds, + // so retry the whole write+rename sequence with short backoff instead of + // failing session persistence and terminating the owning session (#1247). + const ATOMIC_WRITE_ATTEMPTS = 4 + const ATOMIC_WRITE_RETRY_BASE_MS = 50 + const ATOMIC_WRITE_RETRY_MAX_MS = 200 + + export async function writeJsonAtomic(target: string, serialized: string, options?: WriteOptions) { + return writeFileAtomic(target, serialized, options) + } + + export async function writeFileAtomic(target: string, content: string | Uint8Array, options?: WriteOptions) { + await fs.mkdir(path.dirname(target), { recursive: true, ...(options?.private ? { mode: 0o700 } : {}) }) + const tmp = path.join( + path.dirname(target), + `.tmp-${process.pid}-${Date.now().toString(36)}-${Math.random().toString(36).slice(2)}`, + ) + for (let attempt = 1; ; attempt++) { + try { + if (options?.private || options?.durable) { + const file = await fs.open(tmp, "w", options.private ? 0o600 : 0o666) + try { + await file.writeFile(content) + if (options.durable) await file.sync() + } finally { + await file.close() + } + } else { + await Bun.write(tmp, content) + } + await fs.rename(tmp, target) + if (options?.durable && process.platform !== "win32") { + const directory = await fs.open(path.dirname(target), "r") + try { + await directory.sync() + } finally { + await directory.close() + } + } + return + } catch (error) { + if (!isRetryableIOError(error) || attempt >= ATOMIC_WRITE_ATTEMPTS) { + await removeTempFile(tmp) + throw error + } + await new Promise((resolve) => setTimeout(resolve, atomicRetryDelayMs(attempt))) + } + } + } + + // The terminal-failure cleanup can hit the same Windows sharing violation + // that failed the rename (antivirus holding the temp handle), so transient + // unlink errors retry with the same backoff before being suppressed (#1247). + async function removeTempFile(tmp: string) { + for (let attempt = 1; attempt <= ATOMIC_WRITE_ATTEMPTS; attempt++) { + try { + await fs.unlink(tmp) + return + } catch (error) { + if (!isRetryableIOError(error) || attempt >= ATOMIC_WRITE_ATTEMPTS) return + await new Promise((resolve) => setTimeout(resolve, atomicRetryDelayMs(attempt))) + } + } + } + + function atomicRetryDelayMs(attempt: number) { + return Math.min(ATOMIC_WRITE_RETRY_MAX_MS, ATOMIC_WRITE_RETRY_BASE_MS * 2 ** (attempt - 1)) + } +} diff --git a/packages/harness/src/storage/bootstrap.ts b/packages/harness/src/storage/bootstrap.ts new file mode 100644 index 000000000..6d72b3a42 --- /dev/null +++ b/packages/harness/src/storage/bootstrap.ts @@ -0,0 +1,282 @@ +import { createHash, randomUUID } from "node:crypto" +import fs from "node:fs/promises" +import { createReadStream } from "node:fs" +import path from "node:path" +import { z } from "zod" +import { withFileLock } from "@ericsanchezok/synergy-util/fs-lock" +import { AtomicFile } from "./atomic-file" +import { StoragePortable } from "./portable" +import { StorageConfiguration, readStorageConfiguration, resolveStoreOptions } from "./config" +import { StorageIntegrityError } from "./errors" +import { LegacyJsonImporter, legacySources, legacyRecordKey, type ImportProgress } from "./legacy-import" +import { TransactionalStore } from "./transactional-store" +import type { StoreOptions } from "./sql-contract" + +const Manifest = z + .object({ + version: z.literal(1), + namespace: z.string(), + backend: z.enum(["sqlite", "postgres"]), + target: z.string(), + artifactStoreID: z.uuid(), + storeID: z.uuid().optional(), + backupID: z.uuid(), + phase: z.enum(["importing", "validating", "activating", "active"]), + }) + .strict() + +const Switch = z + .object({ + version: z.literal(1), + id: z.uuid(), + configuration: StorageConfiguration, + manifest: Manifest, + archive: z.string(), + sha256: z.string().regex(/^[a-f0-9]{64}$/), + }) + .strict() + +type Manifest = z.infer + +function targetIdentity(options: StoreOptions) { + let target: string + if (options.backend === "sqlite") target = path.resolve(options.filename) + else { + const url = new URL(options.url) + target = JSON.stringify([url.hostname, url.port || "5432", url.pathname]) + } + return createHash("sha256") + .update(JSON.stringify([options.backend, options.namespace, target])) + .digest("hex") +} + +async function optionalJson(filename: string): Promise { + try { + return JSON.parse(await fs.readFile(filename, "utf8")) as unknown + } catch (error) { + if (error && typeof error === "object" && "code" in error && error.code === "ENOENT") return + throw error + } +} + +async function rejectLegacyWriters(dataRoot: string) { + for await (const entry of legacySources(dataRoot)) { + if (legacyRecordKey(entry.relative)) + throw new StorageIntegrityError( + "Legacy JSON records appeared after database activation; preserve both datasets and resolve the old writer before starting", + ) + } +} + +export namespace StorageBootstrap { + export async function status(root: string) { + const saved = await optionalJson(path.join(root, "data", "storage", "manifest.json")) + return saved === undefined ? undefined : Manifest.parse(saved) + } + + export type Prepared = Awaited> + + export async function inspect(root: string) { + if (await optionalJson(path.join(root, "data", "storage", "switch.json"))) + throw new StorageIntegrityError("An interrupted storage switch requires data storage resume") + const saved = await optionalJson(path.join(root, "data", "storage", "manifest.json")) + if (saved === undefined) return + const manifest = Manifest.parse(saved) + if (manifest.phase !== "active") + throw new StorageIntegrityError("Authoritative storage migration must finish before opening a read-only Handle") + const options = resolveStoreOptions(root, await readStorageConfiguration(root), manifest.namespace) + if (targetIdentity(options) !== manifest.target) + throw new StorageIntegrityError("Storage configuration does not identify the active dataset") + const store = await TransactionalStore.open({ ...options, readonly: true, mustExist: true }) + try { + const identity = await store.read<{ storeID: string; artifactStoreID: string }>(["storage_meta", "identity"]) + if (identity.storeID !== manifest.storeID || identity.artifactStoreID !== manifest.artifactStoreID) + throw new StorageIntegrityError("Database and artifacts belong to different datasets") + return { store, artifactDirectory: path.join(root, "data"), manifest } + } catch (error) { + await store.close() + throw error + } + } + + export async function migrateTarget(input: { + root: string + store: TransactionalStore + configuration: StorageConfiguration + }) { + const directory = path.join(input.root, "data", "storage") + if (await optionalJson(path.join(directory, "switch.json"))) + throw new StorageIntegrityError("Resume the interrupted storage switch before starting another") + const manifest = Manifest.parse(await optionalJson(path.join(directory, "manifest.json"))) + if (manifest.phase !== "active" || manifest.namespace !== input.store.options.namespace) + throw new StorageIntegrityError("Storage must be active before migrating its target") + const configuration = StorageConfiguration.parse(input.configuration) + const namespace = configuration.namespace ?? manifest.namespace + const options = resolveStoreOptions(input.root, configuration, namespace) + const target = targetIdentity(options) + if (target === manifest.target) throw new StorageIntegrityError("The requested target is already active") + const id = randomUUID() + const archive = path.join(directory, "transfers", id, "source.ndjson") + await StoragePortable.exportFile(input.store, archive) + const sha256 = await fileDigest(archive) + const next = { ...manifest, namespace, backend: options.backend, target } + await AtomicFile.writeJsonAtomic( + path.join(directory, "switch.json"), + JSON.stringify({ + version: 1, + id, + configuration, + manifest: next, + archive: path.relative(directory, archive), + sha256, + }), + { private: true, durable: true }, + ) + await resumeTargetSwitch(input.root) + } + + export async function resumeTargetSwitch(root: string) { + const directory = path.join(root, "data", "storage") + const filename = path.join(directory, "switch.json") + const raw = await optionalJson(filename) + if (!raw) return false + const intent = Switch.parse(raw) + const archive = path.resolve(directory, intent.archive) + if (!archive.startsWith(directory + path.sep) || (await fileDigest(archive)) !== intent.sha256) + throw new StorageIntegrityError("Storage transfer backup checksum failed") + const options = resolveStoreOptions(root, intent.configuration, intent.manifest.namespace) + if (targetIdentity(options) !== intent.manifest.target) + throw new StorageIntegrityError("Storage transfer target identity changed") + const store = await TransactionalStore.open({ ...options, recover: true }) + try { + const completed = await store.operationReceipt(intent.id) + if (!completed && (await store.query({ limit: 1 })).length) + throw new StorageIntegrityError("Storage target is not empty; choose a new namespace or database") + await StoragePortable.importFile(store, archive, { operationID: intent.id }) + const report = await store.verify() + if (report.issues.length) throw new StorageIntegrityError("Transferred data failed relationship verification") + const identity = await store.read<{ storeID: string; artifactStoreID: string }>(["storage_meta", "identity"]) + if (identity.storeID !== intent.manifest.storeID || identity.artifactStoreID !== intent.manifest.artifactStoreID) + throw new StorageIntegrityError("Transferred database identity differs from its artifacts") + } finally { + await store.close() + } + await AtomicFile.writeJsonAtomic( + path.join(root, "config", "synergy.d", "130-storage.jsonc"), + JSON.stringify({ storage: intent.configuration }), + { private: true, durable: true }, + ) + await AtomicFile.writeJsonAtomic(path.join(directory, "manifest.json"), JSON.stringify(intent.manifest), { + private: true, + durable: true, + }) + await fs.unlink(filename) + return true + } + + export async function prepare(options: { + root: string + recover?: boolean + progress?: (progress: ImportProgress) => void + }) { + const root = path.resolve(options.root) + if (await optionalJson(path.join(root, "data", "storage", "switch.json"))) + throw new StorageIntegrityError("An interrupted storage switch requires data storage resume") + const directory = path.join(root, "data", "storage") + await fs.mkdir(directory, { recursive: true, mode: 0o700 }) + return withFileLock({ directory: path.join(directory, ".locks"), key: "bootstrap" }, async () => { + const filename = path.join(directory, "manifest.json") + const saved = await optionalJson(filename) + const configuration = await readStorageConfiguration(root) + const namespace = saved ? Manifest.parse(saved).namespace : (configuration.namespace ?? randomUUID()) + const storeOptions = resolveStoreOptions(root, configuration, namespace) + const target = targetIdentity(storeOptions) + const manifest: Manifest = saved + ? Manifest.parse(saved) + : { + version: 1, + namespace, + backend: storeOptions.backend, + target, + artifactStoreID: randomUUID(), + backupID: randomUUID(), + phase: "importing", + } + if (manifest.target !== target || manifest.backend !== storeOptions.backend) + throw new StorageIntegrityError( + "Storage configuration points away from the active dataset; use an explicit storage migration", + ) + const persist = () => + AtomicFile.writeJsonAtomic(filename, JSON.stringify(manifest), { private: true, durable: true }) + if (!saved) await persist() + const store = await TransactionalStore.open({ + ...storeOptions, + recover: options.recover, + mustExist: manifest.phase !== "importing", + }) + try { + const identityKey = ["storage_meta", "identity"] + const [identity] = await store.readMany<{ storeID: string; artifactStoreID: string }>([identityKey]) + if (identity) { + if ( + identity.artifactStoreID !== manifest.artifactStoreID || + (manifest.storeID && manifest.storeID !== identity.storeID) + ) + throw new StorageIntegrityError("Database and local artifact storage belong to different datasets") + manifest.storeID = identity.storeID + } else { + if (manifest.phase !== "importing" || manifest.storeID) + throw new StorageIntegrityError("Active storage identity is missing") + manifest.storeID = randomUUID() + await store.write(identityKey, { storeID: manifest.storeID, artifactStoreID: manifest.artifactStoreID }) + } + const importer = new LegacyJsonImporter({ + dataRoot: path.join(root, "data"), + backupRoot: path.join(directory, "backups", manifest.backupID), + store, + progress: options.progress, + }) + if (manifest.phase === "importing") { + await importer.run() + const archive = path.join(root, "data", "agent-records.ndjson") + if (await Bun.file(archive).exists()) + await StoragePortable.importFile(store, archive, { + operationID: `portable-${manifest.backupID}`, + accept: (entry) => + entry.type !== "record" || + ![ + "storage_meta", + "storage_import", + "storage_import_files", + "storage_staging", + "storage_transfer", + ].includes(entry.key[0]), + }) + manifest.phase = "validating" + await persist() + } + if (manifest.phase === "active") await rejectLegacyWriters(path.join(root, "data")) + const activate = async () => { + if (manifest.phase === "active") return + manifest.phase = "activating" + await persist() + await importer.retire() + await rejectLegacyWriters(path.join(root, "data")) + manifest.phase = "active" + await persist() + } + if (manifest.phase === "activating") await activate() + return { store, manifest, activate } + } catch (error) { + await store.close() + throw error + } + }) + } +} + +async function fileDigest(filename: string) { + const hash = createHash("sha256") + for await (const chunk of createReadStream(filename)) hash.update(chunk) + return hash.digest("hex") +} diff --git a/packages/harness/src/storage/config.ts b/packages/harness/src/storage/config.ts new file mode 100644 index 000000000..39098de4e --- /dev/null +++ b/packages/harness/src/storage/config.ts @@ -0,0 +1,66 @@ +import fs from "node:fs/promises" +import path from "node:path" +import { parse, type ParseError } from "jsonc-parser" +import { z } from "zod" +import { StorageIntegrityError } from "./errors" +import type { StoreOptions } from "./sql-contract" + +const Namespace = z.string().regex(/^[a-zA-Z0-9_-]{1,128}$/) +export const StorageConfiguration = z.discriminatedUnion("backend", [ + z + .object({ backend: z.literal("sqlite"), namespace: Namespace.optional(), filename: z.string().min(1).optional() }) + .strict(), + z + .object({ + backend: z.literal("postgres"), + namespace: Namespace, + connectionEnv: z.string().regex(/^[a-zA-Z_][a-zA-Z0-9_]*$/), + maxConnections: z.number().int().min(2).max(64).optional(), + }) + .strict(), +]) +export type StorageConfiguration = z.infer + +export async function readStorageConfiguration(root: string): Promise { + let source: string + try { + source = await fs.readFile(path.join(root, "config", "synergy.d", "130-storage.jsonc"), "utf8") + } catch (error) { + if (error && typeof error === "object" && "code" in error && error.code === "ENOENT") return { backend: "sqlite" } + throw error + } + return parseStorageConfiguration(source) +} + +export function parseStorageConfiguration(source: string): StorageConfiguration { + const errors: ParseError[] = [] + const value: unknown = parse(source, errors, { allowTrailingComma: true }) + if (errors.length) throw new StorageIntegrityError("Storage bootstrap configuration is not valid JSONC") + return z.object({ storage: StorageConfiguration, $schema: z.string().optional() }).strict().parse(value).storage +} + +export function resolveStoreOptions( + root: string, + configuration: StorageConfiguration, + namespace: string, +): StoreOptions { + if (configuration.namespace && configuration.namespace !== namespace) + throw new StorageIntegrityError("Configured namespace differs from the active data namespace") + if (configuration.backend === "sqlite") + return { + backend: "sqlite", + namespace, + filename: configuration.filename + ? path.resolve(root, configuration.filename) + : path.join(root, "data", "storage", "agent.sqlite"), + } + const url = process.env[configuration.connectionEnv] + if (!url) + throw new StorageIntegrityError( + `Storage connection environment variable ${configuration.connectionEnv} is unavailable`, + ) + const parsed = new URL(url) + if (parsed.protocol !== "postgres:" && parsed.protocol !== "postgresql:") + throw new StorageIntegrityError("PostgreSQL storage requires a PostgreSQL connection URL") + return { backend: "postgres", namespace, url, maxConnections: configuration.maxConnections } +} diff --git a/packages/harness/src/storage/errors.ts b/packages/harness/src/storage/errors.ts new file mode 100644 index 000000000..167ec3583 --- /dev/null +++ b/packages/harness/src/storage/errors.ts @@ -0,0 +1,43 @@ +import { NamedError } from "@ericsanchezok/synergy-util/error" +import { z } from "zod" + +export const NotFoundError = NamedError.create("NotFoundError", z.object({ message: z.string() })) + +export class StorageConflictError extends Error { + override readonly name = "StorageConflictError" +} + +export class StorageClosedError extends Error { + override readonly name = "StorageClosedError" + constructor() { + super("The authoritative store is closed") + } +} + +export class StorageIntegrityError extends Error { + override readonly name = "StorageIntegrityError" +} + +export class StorageOwnershipError extends Error { + override readonly name = "StorageOwnershipError" +} + +export class StorageCommitUnknownError extends Error { + override readonly name = "StorageCommitUnknownError" + constructor( + readonly operationID: string | undefined, + cause: unknown, + ) { + super("The database did not confirm the commit; reconcile the operation receipt before retrying", { cause }) + } +} + +export class StorageBusyError extends Error { + override readonly name = "StorageBusyError" +} + +export function databaseErrorCode(error: unknown): string | undefined { + if (!error || typeof error !== "object") return + if ("errno" in error && typeof error.errno === "string" && /^[A-Z0-9]{5}$/.test(error.errno)) return error.errno + return "code" in error ? String(error.code) : undefined +} diff --git a/packages/harness/src/storage/legacy-import.ts b/packages/harness/src/storage/legacy-import.ts new file mode 100644 index 000000000..89c7a8de5 --- /dev/null +++ b/packages/harness/src/storage/legacy-import.ts @@ -0,0 +1,444 @@ +import { createHash, randomUUID } from "node:crypto" +import fs from "node:fs/promises" +import { createReadStream } from "node:fs" +import path from "node:path" +import { AtomicFile } from "./atomic-file" +import { StorageIntegrityError } from "./errors" +import { TransactionalStore } from "./transactional-store" + +const recordRoots = new Set([ + "projects", + "sessions", + "operations", + "session_index", + "endpoint_session", + "sessions_page_index", + "session_child_index", + "session_nav_v2", + "session_search_v1", + "session_search_dirty_v1", + "session_message_order_v1", + "permissions", + "permission-rules", + "shares", + "meta", + "agenda", + "notes", + "blueprint_loops", + "superplan", + "lattice", + "holos", + "synergy_link", + "stats", + "snapshot-v2", + "plugin-approvals", + "plugin-audit", + "plugin-runtime-state", + "plugin-incompatible", + "registry", +]) + +export function legacyRecordKey(relative: string): string[] | undefined { + if (relative === "@home/plugin.lock") return ["plugin-lock"] + if (!relative.endsWith(".json")) return + const key = relative.slice(0, -5).split("/") + if (key.some((segment) => !segment || segment === "." || segment === "..")) + throw new StorageIntegrityError("Invalid legacy record path") + if (key[0] === "channel") { + if (key[1] === "workspaces") return + return key + } + if (key[0] === "browser" && /^sessions(?:-v\d+)?$/.test(key[1] ?? "")) return key + if (key[0] === "push" && key[1] === "subscriptions") return key + if (key[0] === "library" && key[1] === "stats") return key + if (key[0] === "snapshot-v2" && (key.includes(".locks") || key.includes("leases"))) return + return recordRoots.has(key[0]) ? key : undefined +} + +export interface ImportProgress { + stage: "backup" | "import" | "verify" + current: number + total: number + bytes: number +} + +interface ImportFile { + relative: string + hash: string + size: number + key?: string[] + disposition: "backed-up" | "imported" | "retained" | "quarantined" + error?: string + linkTarget?: string + retired?: boolean +} + +interface ImportState { + source: string + backup: string + backedUp: boolean + complete: boolean + files: number +} + +export interface ImportResult { + files: number + imported: number + quarantined: number + retained: number + bytes: number +} + +const stateKey = ["storage_import", "info"] + +function entryKey(relative: string) { + return ["storage_import_files", createHash("sha256").update(relative).digest("hex")] +} + +async function digest(filename: string) { + const hash = createHash("sha256") + for await (const chunk of createReadStream(filename)) hash.update(chunk) + return hash.digest("hex") +} + +export async function* legacyFiles( + root: string, + segments: string[] = [], +): AsyncGenerator<{ relative: string; size: number; linkTarget?: string }> { + const directory = await fs.opendir(path.join(root, ...segments)) + for await (const entry of directory) { + if (entry.name === ".locks" || entry.name.startsWith(".tmp-") || entry.name.endsWith(".tmp")) continue + if (segments.length === 0 && entry.name === "storage") continue + const child = [...segments, entry.name] + if (entry.isSymbolicLink()) { + const relative = child.join("/") + if (recordRoots.has(child[0]) || legacyRecordKey(relative)) + throw new StorageIntegrityError("Authoritative legacy records cannot be symbolic links") + const linkTarget = await fs.readlink(path.join(root, ...child)) + yield { relative, size: Buffer.byteLength(linkTarget), linkTarget } + continue + } + if (entry.isDirectory()) yield* legacyFiles(root, child) + else if (entry.isFile()) { + const stat = await fs.stat(path.join(root, ...child)) + yield { relative: child.join("/"), size: stat.size } + } else throw new StorageIntegrityError("Legacy storage contains an unsupported file type") + } +} + +export async function* legacySources( + dataRoot: string, +): AsyncGenerator<{ relative: string; size: number; linkTarget?: string }> { + yield* legacyFiles(dataRoot) + const config = path.join(dataRoot, "..", "config") + try { + for await (const entry of legacyFiles(config)) yield { ...entry, relative: `@home/config/${entry.relative}` } + } catch (error) { + if (!(error && typeof error === "object" && "code" in error && error.code === "ENOENT")) throw error + } + const lock = path.join(dataRoot, "..", "plugin.lock") + try { + const stat = await fs.lstat(lock) + if (!stat.isFile() || stat.isSymbolicLink()) + throw new StorageIntegrityError("Plugin installation metadata is not a regular file") + yield { relative: "@home/plugin.lock", size: stat.size } + } catch (error) { + if (!(error && typeof error === "object" && "code" in error && error.code === "ENOENT")) throw error + } +} + +function sourcePath(dataRoot: string, relative: string) { + if (relative === "@home/plugin.lock") return path.join(dataRoot, "..", "plugin.lock") + const segments = relative.split("/") + if (segments.some((segment) => !segment || segment === "." || segment === ".." || segment.includes("\\"))) + throw new StorageIntegrityError("Unsafe migration source identity") + return segments[0] === "@home" ? path.join(dataRoot, "..", ...segments.slice(1)) : path.join(dataRoot, ...segments) +} + +async function backupFile(source: string, target: string, expectedHash: string) { + await fs.mkdir(path.dirname(target), { recursive: true, mode: 0o700 }) + const temporary = `${target}.tmp-${randomUUID()}` + try { + await fs.copyFile(source, temporary) + if ((await digest(temporary)) !== expectedHash) + throw new StorageIntegrityError("Legacy data changed while its backup was being copied") + await fs.chmod(temporary, 0o600) + const file = await fs.open(temporary, "r+") + try { + await file.sync() + } finally { + await file.close() + } + await fs.rename(temporary, target) + if (process.platform !== "win32") { + const directory = await fs.open(path.dirname(target), "r") + try { + await directory.sync() + } finally { + await directory.close() + } + } + } finally { + await fs.rm(temporary, { force: true }) + } +} + +export class LegacyJsonImporter { + constructor( + private readonly options: { + dataRoot: string + backupRoot: string + store: TransactionalStore + progress?: (progress: ImportProgress) => void + }, + ) {} + + async run(): Promise { + const { store, dataRoot, backupRoot, progress } = this.options + const identity = createHash("sha256") + .update(await fs.realpath(dataRoot)) + .digest("hex") + const [saved] = await store.readMany([stateKey]) + if (saved && (saved.source !== identity || saved.backup !== path.resolve(backupRoot))) + throw new StorageIntegrityError("Migration source or backup identity changed") + if (!saved) { + let backupBytes = 0n + let recordBytes = 0n + for await (const entry of legacySources(dataRoot)) { + backupBytes += BigInt(entry.size) + if (legacyRecordKey(entry.relative)) recordBytes += BigInt(entry.size) + 4096n + } + const disk = await fs.statfs(dataRoot, { bigint: true }) + const required = backupBytes + recordBytes * 3n + 32n * 1024n * 1024n + if (disk.bavail * disk.bsize < required) + throw new StorageIntegrityError( + "Insufficient free space for the immutable backup, database and migration journal; free space and resume", + ) + } + const state: ImportState = saved ?? { + source: identity, + backup: path.resolve(backupRoot), + backedUp: false, + complete: false, + files: 0, + } + await store.write(stateKey, state) + let count = 0 + let bytes = 0 + progress?.({ stage: "backup", current: 0, total: state.files, bytes: 0 }) + for await (const source of legacySources(dataRoot)) { + const filename = sourcePath(dataRoot, source.relative) + const key = entryKey(source.relative) + const [previous] = await store.readMany([key]) + const hash = + source.linkTarget === undefined + ? await digest(filename) + : createHash("sha256").update(source.linkTarget).digest("hex") + if (previous) { + if (previous.relative !== source.relative || previous.hash !== hash) + throw new StorageIntegrityError("Legacy data changed after the migration snapshot was recorded") + const backup = path.join(backupRoot, "data", source.relative) + const backupHash = + previous.linkTarget === undefined + ? await digest(backup) + : createHash("sha256") + .update(await fs.readlink(backup)) + .digest("hex") + if (backupHash !== previous.hash) throw new StorageIntegrityError("Migration backup failed its integrity check") + } else { + if (state.backedUp) + throw new StorageIntegrityError("New legacy data appeared after the migration snapshot was sealed") + const backup = path.join(backupRoot, "data", source.relative) + if (source.linkTarget === undefined) await backupFile(filename, backup, hash) + else { + await fs.mkdir(path.dirname(backup), { recursive: true, mode: 0o700 }) + try { + await fs.symlink(source.linkTarget, backup) + } catch (error) { + if ( + !(error && typeof error === "object" && "code" in error && error.code === "EEXIST") || + (await fs.readlink(backup)) !== source.linkTarget + ) + throw error + } + } + const entry: ImportFile = { ...source, hash, key: legacyRecordKey(source.relative), disposition: "backed-up" } + await store.write(key, entry) + } + bytes += source.size + count++ + progress?.({ stage: "backup", current: count, total: state.files, bytes }) + } + if (state.backedUp && count !== state.files) + throw new StorageIntegrityError("Legacy files disappeared after the migration snapshot was sealed") + const inventoryPath = path.join(backupRoot, "inventory.ndjson") + await fs.mkdir(backupRoot, { recursive: true, mode: 0o700 }) + const inventory = await fs.open(inventoryPath + ".tmp", "w", 0o600) + const inventoryHash = createHash("sha256") + try { + let cursor: string[] | undefined + for (;;) { + const records = await store.query({ kind: "storage_import_files", after: cursor, limit: 128 }) + if (!records.length) break + for (const { value } of records) { + const line = + JSON.stringify({ + relative: value.relative, + hash: value.hash, + size: value.size, + linkTarget: value.linkTarget, + key: value.key, + }) + "\n" + inventoryHash.update(line) + await inventory.writeFile(line) + } + cursor = records.at(-1)!.key + } + await inventory.sync() + } finally { + await inventory.close() + } + await fs.rename(inventoryPath + ".tmp", inventoryPath) + state.backedUp = true + state.files = count + await store.write(stateKey, state) + await AtomicFile.writeJsonAtomic( + path.join(backupRoot, "manifest.json"), + JSON.stringify({ + version: 1, + source: identity, + files: count, + bytes, + inventorySHA256: inventoryHash.digest("hex"), + }), + { private: true, durable: true }, + ) + + const result: ImportResult = { files: count, bytes, imported: 0, quarantined: 0, retained: 0 } + let after: string[] | undefined + progress?.({ stage: "import", current: 0, total: count, bytes: 0 }) + let importedBytes = 0 + for (;;) { + const batch = await store.query({ kind: "storage_import_files", after, limit: 128 }) + if (!batch.length) break + for (const record of batch) { + const entry = record.value + if (entry.disposition === "backed-up") await this.import(record.key, entry) + if (entry.disposition === "quarantined" && ["projects", "meta"].includes(entry.key?.[0] ?? "")) + throw new StorageIntegrityError( + "A global identity or migration ledger is corrupt; restore and repair a copy of the pre-upgrade backup before retrying", + ) + if (entry.disposition === "imported") result.imported++ + else if (entry.disposition === "quarantined") result.quarantined++ + else result.retained++ + importedBytes += entry.size + progress?.({ + stage: "import", + current: result.imported + result.quarantined + result.retained, + total: count, + bytes: importedBytes, + }) + } + after = batch.at(-1)!.key + } + progress?.({ stage: "verify", current: 0, total: count, bytes: 0 }) + if (result.imported + result.quarantined + result.retained !== count) + throw new StorageIntegrityError("Migration did not account for every source file") + state.complete = true + await store.write(stateKey, state) + progress?.({ stage: "verify", current: count, total: count, bytes }) + return result + } + + async retire(): Promise { + const { store, dataRoot, backupRoot } = this.options + let after: string[] | undefined + for (;;) { + const batch = await store.query({ kind: "storage_import_files", after, limit: 128 }) + if (!batch.length) return + for (const record of batch) { + const entry = record.value + if (!entry.key || entry.retired) continue + if (entry.disposition !== "imported" && entry.disposition !== "quarantined") + throw new StorageIntegrityError("Cannot retire an unaccounted legacy record") + if ((await digest(path.join(backupRoot, "data", entry.relative))) !== entry.hash) + throw new StorageIntegrityError("Cannot retire a legacy record whose backup is invalid") + const source = sourcePath(dataRoot, entry.relative) + try { + if ((await digest(source)) !== entry.hash) + throw new StorageIntegrityError("A legacy writer changed data during activation") + await fs.unlink(source) + } catch (error) { + if (!(error && typeof error === "object" && "code" in error && error.code === "ENOENT")) throw error + } + entry.retired = true + await store.write(record.key, entry) + } + after = batch.at(-1)!.key + } + } + + private async import(recordKey: string[], entry: ImportFile) { + const { store, backupRoot } = this.options + if (!entry.key) { + entry.disposition = "retained" + await store.write(recordKey, entry) + return + } + let value: unknown + try { + value = JSON.parse(await fs.readFile(path.join(backupRoot, "data", entry.relative), "utf8")) + if ( + entry.key[0] === "projects" && + (!value || typeof value !== "object" || Array.isArray(value) || !("id" in value) || value.id !== entry.key[1]) + ) + throw new StorageIntegrityError("Legacy Scope identity does not match its key") + if ( + entry.key[0] === "meta" && + entry.key[1] === "migration" && + (!value || + typeof value !== "object" || + Array.isArray(value) || + Object.values(value).some((timestamp) => typeof timestamp !== "number" || !Number.isFinite(timestamp))) + ) + throw new StorageIntegrityError("Legacy migration ledger is malformed") + if (entry.key[0] === "sessions" && (entry.key[3] === "info" || entry.key[3] === "messages")) { + const expectedID = entry.key.at(-1) === "info" ? entry.key.at(-2) : entry.key.at(-1) + if (!value || typeof value !== "object" || Array.isArray(value) || !("id" in value) || value.id !== expectedID) + throw new StorageIntegrityError("Legacy record identity does not match its owner") + } + } catch (error) { + if (!(error instanceof SyntaxError) && !(error instanceof StorageIntegrityError)) throw error + entry.disposition = "quarantined" + entry.error = error instanceof SyntaxError ? "Invalid JSON" : "Invalid record identity" + await store.transaction(async (tx) => { + await tx.write(recordKey, entry) + if (entry.key?.[0] === "sessions" && entry.key[2]) { + await tx.write(["storage_recovery", "sessions", entry.key[2], "info"], { + blocked: true, + reason: "historical_data_gap", + scopeID: entry.key[1], + }) + await tx.write(["storage_recovery", "sessions", entry.key[2], "issues", recordKey.at(-1)!], { + source: entry.relative, + error: entry.error, + }) + } else + await tx.write(["storage_recovery", "records", recordKey.at(-1)!], { + key: entry.key, + source: entry.relative, + error: entry.error, + }) + }) + return + } + await store.transaction(async (tx) => { + const [existing] = await tx.readMany([entry.key!]) + if (existing !== undefined) + throw new StorageIntegrityError("A legacy record conflicts with an existing target record") + await tx.write(entry.key!, value) + entry.disposition = "imported" + await tx.write(recordKey, entry) + }) + } +} diff --git a/packages/harness/src/storage/maintenance.ts b/packages/harness/src/storage/maintenance.ts new file mode 100644 index 000000000..1eb7b786c --- /dev/null +++ b/packages/harness/src/storage/maintenance.ts @@ -0,0 +1,59 @@ +import { SessionStaging } from "../session/staging" +import { Global } from "../global" +import { ensureMigrations } from "../migration" +import { ServerProcessLock } from "../util/server-process-lock" +import { Storage } from "./storage" +import { StorageBootstrap } from "./bootstrap" +import { StorageRecovery } from "./recovery" +import { StorageIntegrityError } from "./errors" + +export namespace StorageMaintenance { + export async function open(options: { readonly?: boolean; migrate?: boolean; recover?: boolean } = {}) { + if (Storage.available()) throw new StorageIntegrityError("Maintenance cannot replace an installed Runtime Handle") + if (options.readonly) { + const handle = await StorageBootstrap.inspect(Global.Path.root) + if (!handle) + throw new StorageIntegrityError("Storage has not been initialized; run data storage resume before inspection") + const uninstall = Storage.install(handle) + const close = async () => { + try { + await handle.store.close() + } finally { + uninstall() + } + } + return { ...handle, close, [Symbol.asyncDispose]: close } + } + const ownership = await ServerProcessLock.acquire(undefined, "oneshot") + let prepared: StorageBootstrap.Prepared | undefined + let uninstall: (() => void) | undefined + let closing: Promise | undefined + const close = () => + (closing ??= (async () => { + try { + await prepared?.store.close() + } finally { + uninstall?.() + await ownership.release() + } + })()) + try { + await Global.initialize({ cache: false }) + if (options.recover) await StorageBootstrap.resumeTargetSwitch(Global.Path.root) + prepared = await StorageBootstrap.prepare({ root: Global.Path.root, recover: options.recover }) + const handle = { store: prepared.store, artifactDirectory: Global.Path.data } + uninstall = Storage.install(handle) + await SessionStaging.recover() + if (options.migrate !== false) { + await ensureMigrations({ output: "silent" }) + if (prepared.manifest.phase !== "active") await StorageRecovery.validate() + await prepared.activate() + await StorageRecovery.recoverOwners() + } + return { ...handle, manifest: prepared.manifest, close, [Symbol.asyncDispose]: close } + } catch (error) { + await close() + throw error + } + } +} diff --git a/packages/harness/src/storage/portable.ts b/packages/harness/src/storage/portable.ts new file mode 100644 index 000000000..56603d85b --- /dev/null +++ b/packages/harness/src/storage/portable.ts @@ -0,0 +1,155 @@ +import { createHash, randomUUID } from "node:crypto" +import { createReadStream } from "node:fs" +import fs from "node:fs/promises" +import path from "node:path" +import { z } from "zod" +import type { StoreTransaction, TransactionalStore } from "./transactional-store" +import { StorageIntegrityError } from "./errors" + +export const StorageEntry = z.discriminatedUnion("type", [ + z + .object({ + type: z.literal("record"), + key: z.array(z.string().min(1)).min(1), + value: z.unknown(), + revision: z.string().regex(/^[1-9][0-9]*$/), + }) + .strict(), + z + .object({ + type: z.literal("receipt"), + operationID: z.string(), + requestHash: z.string(), + result: z.string(), + created: z.number().int(), + }) + .strict(), + z + .object({ + type: z.literal("event"), + id: z.string(), + scopeID: z.string(), + eventType: z.string(), + payload: z.unknown(), + }) + .strict(), +]) +export type StorageEntry = z.infer +const Header = z.object({ format: z.literal("synergy-agent-data"), version: z.literal(1) }).strict() +const Footer = z + .object({ end: z.literal(true), count: z.number().int().nonnegative(), sha256: z.string().regex(/^[a-f0-9]{64}$/) }) + .strict() +const MAX_LINE_BYTES = 32 * 1024 * 1024 + +export namespace StoragePortable { + export async function exportFile(store: TransactionalStore, filename: string) { + await fs.mkdir(path.dirname(filename), { recursive: true, mode: 0o700 }) + const temporary = `${filename}.tmp-${randomUUID()}` + const file = await fs.open(temporary, "wx", 0o600) + let count = 0 + const hash = createHash("sha256") + try { + await file.writeFile(JSON.stringify({ format: "synergy-agent-data", version: 1 }) + "\n") + await store.snapshot(async (tx) => { + for await (const entry of tx.exportEntries()) { + const line = JSON.stringify(entry) + "\n" + if (Buffer.byteLength(line) > MAX_LINE_BYTES) + throw new StorageIntegrityError("Portable record exceeds the supported byte limit") + hash.update(line) + await file.writeFile(line) + count++ + } + }) + const sha256 = hash.digest("hex") + await file.writeFile(JSON.stringify({ end: true, count, sha256 }) + "\n") + await file.sync() + await file.close() + await fs.rename(temporary, filename) + if (process.platform !== "win32") { + const directory = await fs.open(path.dirname(filename), "r") + try { + await directory.sync() + } finally { + await directory.close() + } + } + return { count, sha256 } + } catch (error) { + await file.close().catch(() => {}) + await fs.rm(temporary, { force: true }) + throw error + } + } + + export async function importFile( + store: TransactionalStore, + filename: string, + options: { + accept?: (entry: StorageEntry, tx: StoreTransaction) => boolean | Promise + operationID?: string + afterImport?: (tx: StoreTransaction) => Promise + } = {}, + ) { + const fileHash = createHash("sha256") + for await (const chunk of createReadStream(filename)) fileHash.update(chunk) + const requestHash = fileHash.digest("hex") + return store.transaction( + async (tx) => { + let count = 0 + let accepted = 0 + let header = false + let footer = false + const hash = createHash("sha256") + const consumed = createHash("sha256") + for await (const line of lines(filename)) { + consumed.update(line + "\n") + if (!header) { + Header.parse(JSON.parse(line)) + header = true + continue + } + if (footer) throw new StorageIntegrityError("Portable archive contains trailing data") + const parsed: unknown = JSON.parse(line) + if (parsed && typeof parsed === "object" && "end" in parsed) { + const last = Footer.parse(parsed) + if (last.count !== count || last.sha256 !== hash.digest("hex")) + throw new StorageIntegrityError("Portable archive checksum mismatch") + footer = true + continue + } + hash.update(line + "\n") + count++ + const entry = StorageEntry.parse(parsed) + if (options.accept && !(await options.accept(entry, tx))) continue + await tx.restoreEntry(entry) + accepted++ + } + if (!header || !footer) + throw new StorageIntegrityError("Portable archive is truncated; checksum footer is missing") + if (consumed.digest("hex") !== requestHash) + throw new StorageIntegrityError("Portable archive changed during import") + await options.afterImport?.(tx) + return { count, accepted, sha256: requestHash } + }, + options.operationID ? { operationID: options.operationID, requestHash } : undefined, + ) + } +} + +async function* lines(filename: string) { + let remainder = "" + for await (const chunk of createReadStream(filename, { encoding: "utf8", highWaterMark: 65536 })) { + remainder += chunk + let index: number + while ((index = remainder.indexOf("\n")) !== -1) { + const line = remainder.slice(0, index) + if (Buffer.byteLength(line) > MAX_LINE_BYTES) + throw new StorageIntegrityError("Portable record exceeds the supported byte limit") + yield line + remainder = remainder.slice(index + 1) + } + if (Buffer.byteLength(remainder) > MAX_LINE_BYTES) + throw new StorageIntegrityError("Portable record exceeds the supported byte limit") + } + if (remainder) throw new StorageIntegrityError("Portable archive is truncated; final newline is missing") +} diff --git a/packages/harness/src/storage/postgres-driver.ts b/packages/harness/src/storage/postgres-driver.ts new file mode 100644 index 000000000..c8870affa --- /dev/null +++ b/packages/harness/src/storage/postgres-driver.ts @@ -0,0 +1,154 @@ +import { SQL } from "bun" +import { createHash } from "node:crypto" +import { + databaseErrorCode, + StorageClosedError, + StorageCommitUnknownError, + StorageOwnershipError, + StorageIntegrityError, +} from "./errors" +import type { SqlConnection, SqlDriver, SqlRow, SqlValue } from "./sql-contract" + +function statement(sql: string) { + let index = 0 + return sql.replace(/\?/g, () => `$${++index}`) +} + +// Bun 1.3.14 uses errno for SQLSTATE and an asynchronous reserved-connection release. +// https://github.com/oven-sh/bun/blob/bun-v1.3.14/src/js/bun/sql.ts +export class PostgresDriver implements SqlDriver { + readonly backend = "postgres" as const + private closed = false + private ownershipLost = false + private closing?: Promise + private constructor( + private readonly pool: SQL, + private readonly owner?: SQL, + private readonly ownerIdentity?: { pid: number; classID: number; objectID: number }, + ) {} + + static async open(url: string, namespace: string, max = 8, readonly = false) { + const options = { + max: readonly ? Math.max(2, max) : Math.max(1, max - 1), + bigint: true, + connectionTimeout: 10, + connection: { + synchronous_commit: "on", + statement_timeout: 30000, + lock_timeout: 5000, + idle_in_transaction_session_timeout: 30000, + }, + } + const pool = new SQL(url, options) + let owner: SQL | undefined + try { + const [version] = await pool`SELECT current_setting('server_version_num')::integer AS version` + if (Number(version.version) < 160000 || Number(version.version) >= 190000) + throw new StorageIntegrityError("Authoritative storage requires PostgreSQL 16, 17, or 18") + if (readonly) return new PostgresDriver(pool) + owner = new SQL(url, { ...options, max: 1, idleTimeout: 0, maxLifetime: 0 }) + const digest = createHash("sha256").update(namespace).digest() + const [lock] = + await owner`SELECT pg_try_advisory_lock(${digest.readInt32BE(0)}, ${digest.readInt32BE(4)}) AS acquired` + if (!lock.acquired) throw new StorageOwnershipError("Another Runtime owns this PostgreSQL namespace") + const [identity] = await owner`SELECT pg_backend_pid() AS pid` + return new PostgresDriver(pool, owner, { + pid: Number(identity.pid), + classID: digest.readUInt32BE(0), + objectID: digest.readUInt32BE(4), + }) + } catch (error) { + if (owner) await owner.close({ timeout: 0 }) + await pool.close() + throw error + } + } + + async query(sql: string, values: SqlValue[] = []): Promise { + if (this.closed) throw new StorageClosedError() + return this.pool.unsafe(statement(sql), values) as unknown as Promise + } + + async transaction( + body: (connection: SqlConnection) => Promise, + options: { readOnly?: boolean; operationID?: string } = {}, + ): Promise { + for (let attempt = 0; ; attempt++) { + try { + return await this.attempt(body, options) + } catch (error) { + if (attempt >= 2 || !["40001", "40P01"].includes(databaseErrorCode(error) ?? "")) throw error + await Bun.sleep(10 * 2 ** attempt + Math.floor(Math.random() * 10)) + } + } + } + + private async attempt( + body: (connection: SqlConnection) => Promise, + options: { readOnly?: boolean; operationID?: string } = {}, + ): Promise { + if (this.closed) throw new StorageClosedError() + if (!options.readOnly) await this.assertOwnership() + const connection = await this.pool.reserve() + const query = (sql: string, values: SqlValue[] = []) => + connection.unsafe(statement(sql), values) as unknown as Promise + let committing = false + try { + await query( + options.readOnly ? "BEGIN ISOLATION LEVEL REPEATABLE READ READ ONLY" : "BEGIN ISOLATION LEVEL SERIALIZABLE", + ) + if (!options.readOnly) await query("SET LOCAL synchronous_commit = on") + const result = await body({ query }) + if (!options.readOnly) await this.assertOwnership() + committing = true + await query("COMMIT") + return result + } catch (error) { + const code = databaseErrorCode(error) + try { + await query("ROLLBACK") + } catch { + /* A broken connection is closed below. */ + } + if (committing && code !== "40001" && code !== "40P01") + throw new StorageCommitUnknownError(options.operationID, error) + throw error + } finally { + await connection.release() + } + } + + private async assertOwnership() { + if (!this.owner || !this.ownerIdentity) throw new StorageOwnershipError("This PostgreSQL Handle is read-only") + if (this.ownershipLost) + throw new StorageOwnershipError("PostgreSQL Runtime ownership was lost; reopen through explicit recovery") + const expected = this.ownerIdentity + try { + const [identity] = await this + .owner`SELECT pg_backend_pid() AS pid, EXISTS(SELECT 1 FROM pg_locks WHERE locktype = 'advisory' AND pid = pg_backend_pid() AND classid::bigint = ${expected.classID} AND objid::bigint = ${expected.objectID} AND objsubid = 2 AND granted) AS held` + if (Number(identity.pid) !== expected.pid || !identity.held) + throw new StorageOwnershipError("PostgreSQL Runtime ownership was lost; reopen through explicit recovery") + } catch (error) { + this.ownershipLost = true + throw new StorageOwnershipError("PostgreSQL Runtime ownership was lost; reopen through explicit recovery", { + cause: error, + }) + } + } + + close(): Promise { + this.closing ??= (async () => { + this.closed = true + try { + if (this.owner && !this.ownershipLost) await this.owner`SELECT pg_advisory_unlock_all()` + } finally { + try { + await this.owner?.close({ timeout: 0 }) + } finally { + await this.pool.close() + } + } + })() + return this.closing + } +} diff --git a/packages/harness/src/storage/queue.ts b/packages/harness/src/storage/queue.ts new file mode 100644 index 000000000..53f3f6023 --- /dev/null +++ b/packages/harness/src/storage/queue.ts @@ -0,0 +1,30 @@ +import { StorageBusyError, StorageClosedError } from "./errors" + +export class StorageQueue { + private tail = Promise.resolve() + private pending = 0 + private closed = false + + async run(body: () => Promise): Promise { + if (this.closed) throw new StorageClosedError() + if (this.pending >= 1024) throw new StorageBusyError("Authoritative storage queue is full") + this.pending++ + const deadline = Date.now() + 30_000 + const previous = this.tail + const next = Promise.withResolvers() + this.tail = next.promise + try { + await previous + if (Date.now() > deadline) throw new StorageBusyError("Authoritative storage admission deadline exceeded") + return await body() + } finally { + this.pending-- + next.resolve() + } + } + + async close() { + this.closed = true + await this.tail + } +} diff --git a/packages/harness/src/storage/recovery.ts b/packages/harness/src/storage/recovery.ts new file mode 100644 index 000000000..ef5637851 --- /dev/null +++ b/packages/harness/src/storage/recovery.ts @@ -0,0 +1,75 @@ +import { createHash } from "node:crypto" +import { Storage } from "./storage" +import { StorageIntegrityError } from "./errors" + +export namespace StorageRecovery { + const owners = new Map Promise>() + let sealed = false + export function register(name: string, recover: () => Promise) { + if (owners.get(name) === recover) return + if (sealed) throw new StorageIntegrityError("Register storage recovery owners before opening the Runtime") + owners.set(name, recover) + } + export async function recoverOwners() { + sealed = true + for (const recover of owners.values()) await recover() + } + + const blocked = new WeakMap>() + + export async function validate() { + const report = await Storage.current().store.verify() + for (const issue of report.issues) { + if (issue.key[0] !== "sessions" || !issue.key[2]) + throw new StorageIntegrityError("Authoritative data failed validation") + await Storage.transaction(async () => { + const sessionID = issue.key[2] + await Storage.write(["storage_recovery", "sessions", sessionID, "info"], { + blocked: true, + reason: "historical_data_gap", + scopeID: issue.key[1], + }) + const id = createHash("sha256").update(JSON.stringify(issue)).digest("hex") + await Storage.write(["storage_recovery", "sessions", sessionID, "issues", id], issue) + }) + } + return report + } + + export async function load() { + const sessions = new Set() + for (const sessionID of await Storage.scan(["storage_recovery", "sessions"])) { + const [state] = await Storage.readMany<{ blocked: boolean }>([ + ["storage_recovery", "sessions", sessionID, "info"], + ]) + if (state?.blocked) sessions.add(sessionID) + } + blocked.set(Storage.current().store, sessions) + } + + export function assertRunnable(sessionID: string) { + if (Storage.available() && blocked.get(Storage.current().store)?.has(sessionID)) + throw new StorageIntegrityError( + "This session contains quarantined historical data; inspect and repair it before continuing execution", + ) + } + + export async function reconcileNotifications() { + const store = Storage.current().store + let count = 0 + for (;;) { + const events = await store.pendingEvents(100) + if (!events.length) break + count += events.length + await store.write(["storage_meta", "notification-reconciliation"], { + reason: "runtime_epoch_changed", + count, + at: Date.now(), + }) + // A new Runtime has a new event epoch and clients must reload snapshots. + // Replaying arbitrary Bus subscribers could repeat an external effect. + await store.acknowledgeEvents(events.map((event) => event.id)) + } + return count + } +} diff --git a/packages/harness/src/storage/sql-contract.ts b/packages/harness/src/storage/sql-contract.ts new file mode 100644 index 000000000..5bcb548fc --- /dev/null +++ b/packages/harness/src/storage/sql-contract.ts @@ -0,0 +1,38 @@ +export type SqlValue = string | number | bigint | Uint8Array | null +export type SqlRow = Record + +export interface SqlConnection { + query(statement: string, values?: SqlValue[]): Promise +} + +export interface SqlDriver extends SqlConnection { + readonly backend: "sqlite" | "postgres" + transaction( + body: (connection: SqlConnection) => Promise, + options?: { readOnly?: boolean; operationID?: string }, + ): Promise + close(): Promise +} + +export type StoreOptions = { + namespace: string + readonly?: boolean + recover?: boolean + mustExist?: boolean +} & ({ backend: "sqlite"; filename: string } | { backend: "postgres"; url: string; maxConnections?: number }) + +export type SqliteRequest = { + id: number + action: "open" | "query" | "close" + filename?: string + readonly?: boolean + reader?: boolean + statement?: string + values?: SqlValue[] +} + +export type SqliteResponse = { + id: number + rows?: SqlRow[] + error?: { name: string; message: string; code?: string } +} diff --git a/packages/harness/src/storage/sqlite-driver.ts b/packages/harness/src/storage/sqlite-driver.ts new file mode 100644 index 000000000..80d6d787a --- /dev/null +++ b/packages/harness/src/storage/sqlite-driver.ts @@ -0,0 +1,176 @@ +import fs from "node:fs/promises" +import path from "node:path" +import { existsSync } from "node:fs" +import { fileURLToPath } from "node:url" +import { + StorageBusyError, + StorageClosedError, + StorageCommitUnknownError, + StorageIntegrityError, + StorageOwnershipError, +} from "./errors" +import { ServerProcessLock } from "../util/server-process-lock" +import { StorageQueue } from "./queue" +import type { SqlConnection, SqlDriver, SqliteRequest, SqliteResponse, SqlRow, SqlValue } from "./sql-contract" + +export class SqliteDriver implements SqlDriver { + readonly backend = "sqlite" as const + private readonly worker: Bun.Subprocess + private readonly writerQueue = new StorageQueue() + private readonly readerQueue = new StorageQueue() + private readonly pending = new Map< + number, + { + resolve(rows: SqlRow[]): void + reject(error: unknown): void + bytes: number + timeout: ReturnType + } + >() + private sequence = 0 + private queuedBytes = 0 + private closed = false + private closing?: Promise + + private constructor(private readonly ownership?: { release(): Promise }) { + const entry = fileURLToPath(new URL("./sqlite-worker.ts", import.meta.url)) + this.worker = Bun.spawn({ + cmd: existsSync(entry) ? [process.execPath, "run", entry] : [process.execPath, "__storage-worker-runner"], + env: { ...process.env, SYNERGY_STORAGE_PARENT_PID: String(process.pid) }, + serialization: "advanced", + stdout: "ignore", + stderr: "inherit", + ipc: (message: SqliteResponse) => { + const pending = this.pending.get(message.id) + if (!pending) return + clearTimeout(pending.timeout) + this.pending.delete(message.id) + this.queuedBytes -= pending.bytes + if (message.error) pending.reject(Object.assign(new Error(message.error.message), message.error)) + else pending.resolve(message.rows ?? []) + }, + onExit: (_child, code) => { + this.closed = true + for (const pending of this.pending.values()) { + clearTimeout(pending.timeout) + pending.reject(new Error(`SQLite worker exited with code ${code}`)) + } + this.pending.clear() + this.queuedBytes = 0 + }, + }) + } + + static async open(filename: string, readonly = false, mustExist = false) { + if (!readonly) await fs.mkdir(path.dirname(filename), { recursive: true, mode: 0o700 }) + let ownership: { release(): Promise } | undefined + if (!readonly) { + try { + ownership = await ServerProcessLock.acquire(`${filename}.owner`, "oneshot") + } catch (error) { + if (error instanceof ServerProcessLock.AlreadyRunningError) + throw new StorageOwnershipError("Another Runtime owns this SQLite database", { cause: error }) + throw error + } + } + const driver = new SqliteDriver(ownership) + try { + if (mustExist) { + try { + await fs.access(filename) + } catch (error) { + if (error && typeof error === "object" && "code" in error && error.code === "ENOENT") + throw new StorageIntegrityError("The active SQLite database is missing") + throw error + } + } + await driver.request({ action: "open", filename, readonly }) + if (!readonly && process.platform !== "win32") await fs.chmod(filename, 0o600) + return driver + } catch (error) { + driver.worker.kill() + await driver.worker.exited + await ownership?.release() + throw error + } + } + + private request(request: Omit): Promise { + if (this.closed) return Promise.reject(new StorageClosedError()) + const bytes = (request.values ?? []).reduce( + (total, value) => + total + + (typeof value === "string" ? Buffer.byteLength(value) : value instanceof Uint8Array ? value.byteLength : 8), + 0, + ) + if (this.queuedBytes + bytes > 32 * 1024 * 1024) + return Promise.reject(new StorageBusyError("Authoritative storage byte queue is full")) + const id = ++this.sequence + const promise = new Promise((resolve, reject) => { + const timeout = setTimeout(() => { + this.closed = true + this.worker.kill() + reject(new StorageBusyError("SQLite worker exceeded its request deadline")) + }, 30_000) + this.pending.set(id, { resolve, reject, bytes, timeout }) + }) + this.queuedBytes += bytes + try { + this.worker.send({ ...request, id }) + } catch (error) { + const pending = this.pending.get(id)! + clearTimeout(pending.timeout) + this.pending.delete(id) + this.queuedBytes -= bytes + pending.reject(error) + } + return promise + } + + query(statement: string, values: SqlValue[] = []): Promise { + return this.readerQueue.run(() => this.request({ action: "query", reader: true, statement, values })) as Promise< + Row[] + > + } + + transaction( + body: (connection: SqlConnection) => Promise, + options: { readOnly?: boolean; operationID?: string } = {}, + ): Promise { + const queue = options.readOnly ? this.readerQueue : this.writerQueue + return queue.run(async () => { + const query = (statement: string, values: SqlValue[] = []) => + this.request({ action: "query", reader: options.readOnly, statement, values }) as Promise + await query(options.readOnly ? "BEGIN" : "BEGIN IMMEDIATE") + let committing = false + try { + const value = await body({ query }) + committing = true + await query("COMMIT") + return value + } catch (error) { + try { + await query("ROLLBACK") + } catch { + if (committing) throw new StorageCommitUnknownError(options.operationID, error) + } + throw error + } + }) + } + + close(): Promise { + this.closing ??= (async () => { + try { + await Promise.all([this.writerQueue.close(), this.readerQueue.close()]) + if (!this.closed) await this.request({ action: "close" }) + } finally { + this.closed = true + this.worker.kill() + await this.worker.exited + await this.ownership?.release() + } + })() + return this.closing + } +} diff --git a/packages/harness/src/storage/sqlite-engine.ts b/packages/harness/src/storage/sqlite-engine.ts new file mode 100644 index 000000000..bae1525ca --- /dev/null +++ b/packages/harness/src/storage/sqlite-engine.ts @@ -0,0 +1,36 @@ +import { Database } from "bun:sqlite" +import { existsSync } from "node:fs" +import path from "node:path" + +let initialized = false +export function initializeSqliteEngine() { + if (initialized) return + if (process.platform === "darwin") { + const packaged = path.resolve(path.dirname(process.execPath), "../libsqlite3.dylib") + const source = path.resolve(import.meta.dirname, "../../.artifacts/sqlite/libsqlite3.dylib") + const candidates = [packaged, source] + // Source development can use a verified Homebrew engine; packaged builds + // include their own engine and never depend on machine-wide libraries. + if (existsSync(path.resolve(import.meta.dirname, "../../package.json"))) + candidates.push("/opt/homebrew/opt/sqlite/lib/libsqlite3.dylib", "/usr/local/opt/sqlite/lib/libsqlite3.dylib") + const selected = candidates.find(existsSync) + if (!selected) + throw new Error( + "A patched SQLite engine is required. Run bun packages/harness/script/build-sqlite.ts for source development or reinstall the complete Synergy runtime.", + ) + Database.setCustomSQLite(selected) + } + const probe = new Database(":memory:") + try { + const row = probe.query<{ version: string }, []>("SELECT sqlite_version() AS version").get()! + const [major, minor, patch] = row.version.split(".").map(Number) + const supported = + major === 3 && + (minor > 51 || (minor === 51 && patch >= 3) || (minor === 50 && patch >= 7) || (minor === 44 && patch >= 6)) + if (!supported) + throw new Error(`SQLite ${row.version} lacks the required WAL reset fix; use a supported Synergy runtime`) + } finally { + probe.close() + } + initialized = true +} diff --git a/packages/harness/src/storage/sqlite-worker.ts b/packages/harness/src/storage/sqlite-worker.ts new file mode 100644 index 000000000..bd2b4a769 --- /dev/null +++ b/packages/harness/src/storage/sqlite-worker.ts @@ -0,0 +1,55 @@ +import { initializeSqliteEngine } from "./sqlite-engine" +import { Database } from "bun:sqlite" +import { watchManagedParent } from "../util/managed-parent" +import type { SqliteRequest, SqliteResponse } from "./sql-contract" + +let writer: Database | undefined +let reader: Database | undefined + +if (!process.send) throw new Error("SQLite worker requires a parent IPC channel") + +process.on("message", (request: SqliteRequest) => { + const response: SqliteResponse = { id: request.id } + try { + if (request.action === "open") { + writer = new Database(request.filename!, { + create: !request.readonly, + readonly: request.readonly, + strict: true, + safeIntegers: true, + }) + writer.run("PRAGMA busy_timeout = 5000") + writer.run("PRAGMA foreign_keys = ON") + if (!request.readonly) { + writer.run("PRAGMA journal_mode = WAL") + writer.run("PRAGMA synchronous = FULL") + } + reader = new Database(request.filename!, { readonly: true, strict: true, safeIntegers: true }) + reader.run("PRAGMA busy_timeout = 5000") + reader.run("PRAGMA query_only = ON") + } else if (request.action === "close") { + reader?.close() + writer?.close() + reader = undefined + writer = undefined + } else { + const connection = request.reader ? reader : writer + if (!connection) throw new Error("SQLite connection is not open") + response.rows = connection.query(request.statement!).all(...(request.values ?? [])) as NonNullable< + SqliteResponse["rows"] + > + } + } catch (error) { + response.error = { + name: error instanceof Error ? error.name : "Error", + message: error instanceof Error ? error.message : String(error), + code: error && typeof error === "object" && "code" in error ? String(error.code) : undefined, + } + } + process.send?.(response) +}) + +process.on("disconnect", () => process.exit(0)) +watchManagedParent({ expectedParentPid: process.env.SYNERGY_STORAGE_PARENT_PID, onParentExit: () => process.exit(0) }) + +initializeSqliteEngine() diff --git a/packages/harness/src/storage/storage.ts b/packages/harness/src/storage/storage.ts index 6613faaab..e23c45a74 100644 --- a/packages/harness/src/storage/storage.ts +++ b/packages/harness/src/storage/storage.ts @@ -1,186 +1,213 @@ -import path from "path" -import fs from "fs/promises" -import { Global } from "../global" -import { Lock } from "../util/lock" -import { isRetryableIOError } from "../util/io-retry" -import { NamedError } from "@ericsanchezok/synergy-util/error" -import z from "zod" +import path from "node:path" +import { AsyncLocalStorage } from "node:async_hooks" +import { AtomicFile } from "./atomic-file" +import { NotFoundError as MissingRecord, StorageClosedError, StorageConflictError } from "./errors" +import { + TransactionalStore, + type StoreTransaction, + type TransactionOptions, + type RecordQuery, + type StoredEvent, +} from "./transactional-store" import { ObservabilityIssues } from "../observability/issues" import { ObservabilityMetrics } from "../observability/metrics" import { ObservabilityResources } from "../observability/resources" export namespace Storage { - const READ_MANY_CONCURRENCY = 32 - // Successful duration samples are high-cardinality and previously amplified - // telemetry write load under UI polling (#343). Keep errors at 100%. const STORAGE_DURATION_SAMPLE_RATE = 0.02 + export const NotFoundError = MissingRecord + export const writeJsonAtomic = AtomicFile.writeJsonAtomic + export interface Handle { + store: TransactionalStore + artifactDirectory: string + } + interface Context extends Handle { + transaction?: StoreTransaction + effects?: Array<() => Promise | void> + pending?: Promise[] + } + const context = new AsyncLocalStorage() + let installed: Handle | undefined - export const NotFoundError = NamedError.create( - "NotFoundError", - z.object({ - message: z.string(), - }), - ) + export function state(create: () => T): () => T { + const values = new WeakMap() + return () => { + const store = current().store + let value = values.get(store) + if (value === undefined) { + value = create() + values.set(store, value) + } + return value + } + } - function resolveDir() { - return Global.Path.data + export function install(handle: Handle) { + if (installed === handle) return () => {} + if (installed && installed !== handle) throw new StorageConflictError("A storage Handle is already installed") + installed = handle + return () => { + if (installed === handle) installed = undefined + } } - export async function remove(key: string[]) { - const dir = resolveDir() - const target = path.join(dir, ...key) + ".json" - return measureStorage("remove", key, async () => { - await fs.unlink(target).catch(() => {}) - await pruneEmptyParents(path.dirname(target), dir) - }) + export function current(): Context { + const value = context.getStore() ?? installed + if (!value) throw new StorageClosedError() + return value + } + + export function available() { + return Boolean(context.getStore() ?? installed) } - export async function read(key: string[], options: { silentNotFound?: boolean } = {}) { - const dir = resolveDir() - const target = path.join(dir, ...key) + ".json" - return measureStorage( - "read", - key, - async () => - withErrorHandling(async () => { - using _ = await Lock.read(target) - const file = Bun.file(target) - const result = await file.json() - const size = file.size - ObservabilityResources.addRead(size) - return result as T - }), - options, - ) + export function provide(handle: Handle, body: () => T): T { + return context.run(handle, body) } - export async function readMany(keys: string[][]): Promise<(T | undefined)[]> { - const dir = resolveDir() - return measureStorage("readMany", [keys[0]?.[0] ?? "root"], async () => { - const result: (T | undefined)[] = new Array(keys.length) - let next = 0 - let readBytes = 0 - const workers = Array.from({ length: Math.min(READ_MANY_CONCURRENCY, keys.length) }, async () => { - while (next < keys.length) { - const index = next++ - const key = keys[index] - const target = path.join(dir, ...key) + ".json" - try { - using _ = await Lock.read(target) - const file = Bun.file(target) - result[index] = (await file.json()) as T - readBytes += file.size - } catch { - result[index] = undefined - } + export async function transaction( + body: (tx: StoreTransaction) => Promise, + options?: TransactionOptions, + ): Promise { + const parent = current() + if (parent.transaction) { + if (options?.operationID) throw new StorageConflictError("Idempotency belongs to the outer business transaction") + return body(parent.transaction) + } + let effects: Array<() => Promise | void> = [] + const result = await parent.store.transaction(async (tx) => { + effects = [] + const pending: Promise[] = [] + return context.run({ ...parent, transaction: tx, effects, pending }, async () => { + const result = await body(tx) + for (let offset = 0; offset < pending.length; ) { + const batch = pending.slice(offset) + offset += batch.length + await Promise.all(batch) } + return result }) - await Promise.all(workers) - if (readBytes) ObservabilityResources.addRead(readBytes) - return result + }, options) + for (const effect of effects) { + try { + await effect() + } catch (error) { + ObservabilityIssues.raise({ + code: "STORAGE_POST_COMMIT_FAILED", + severity: "error", + module: "storage", + title: "A committed change could not publish its notification", + message: "The database commit succeeded. Pending events remain available for reconciliation.", + evidence: { errorName: error instanceof Error ? error.name : "unknown" }, + }) + } + } + return result + } + + export function snapshot(body: (tx: StoreTransaction) => Promise): Promise { + const parent = current() + if (parent.transaction) return body(parent.transaction) + return parent.store.snapshot((tx) => context.run({ ...parent, transaction: tx }, () => body(tx))) + } + + export function inTransaction() { + return Boolean(context.getStore()?.transaction) + } + + export function afterCommit(effect: () => Promise | void): void { + const active = context.getStore() + if (!active?.effects) throw new StorageConflictError("Post-commit effects require a write transaction") + active.effects.push(effect) + } + + export function enqueue(event: StoredEvent, effect: () => Promise): Promise { + const active = context.getStore() + if (!active?.transaction || !active.effects || !active.pending) + throw new StorageConflictError("An event requires a write transaction") + const pending = active.transaction.enqueue(event) + active.pending.push(pending) + void pending.catch(() => {}) + active.effects.push(async () => { + await effect() + await active.store.acknowledgeEvents([event.id]) }) + return pending } - export interface WriteOptions { - compact?: boolean - durable?: boolean - private?: boolean + export function read(key: string[], options: { silentNotFound?: boolean } = {}): Promise { + return measureStorage("read", key, () => snapshot((tx) => tx.read(key)), options) } - function serialize(content: unknown, options?: WriteOptions) { - return options?.compact ? JSON.stringify(content) : JSON.stringify(content, null, 2) + export function readMany(keys: string[][]): Promise<(T | undefined)[]> { + return measureStorage("readMany", [keys[0]?.[0] ?? "root"], () => snapshot((tx) => tx.readMany(keys))) } - export async function update(key: string[], fn: (draft: T) => void, options?: WriteOptions) { - const dir = resolveDir() - const target = path.join(dir, ...key) + ".json" - return measureStorage("update", key, async () => - withErrorHandling(async () => { - using _ = await Lock.write(target) - const content = await Bun.file(target).json() - fn(content) - const serialized = serialize(content, options) - await writeJsonAtomic(target, serialized, options) - ObservabilityResources.addWrite(Buffer.byteLength(serialized, "utf8")) - return content as T - }), - ) + export function versioned(key: string[]) { + return snapshot((tx) => tx.versioned(key)) } - export async function write(key: string[], content: T, options?: WriteOptions) { - const dir = resolveDir() - const target = path.join(dir, ...key) + ".json" - return measureStorage("write", key, async () => - withErrorHandling(async () => { - using _ = await Lock.write(target) - const serialized = serialize(content, options) - await writeJsonAtomic(target, serialized, options) - ObservabilityResources.addWrite(Buffer.byteLength(serialized, "utf8")) - }), - ) + export function write(key: string[], value: T) { + return measureStorage("write", key, () => transaction((tx) => tx.write(key, value))) } - export async function scan(prefix: string[], options?: { strict?: boolean }): Promise { - const dir = resolveDir() - const target = path.join(dir, ...prefix) - return measureStorage("scan", prefix, async () => { - try { - const entries = await fs.readdir(target) - return entries - .filter((e) => !isTempFile(e)) - .map((e) => (e.endsWith(".json") ? e.slice(0, -5) : e)) - .sort() - } catch (error) { - if (options?.strict && !(error instanceof Error && "code" in error && error.code === "ENOENT")) throw error - return [] - } - }) + export function update(key: string[], change: (value: T) => void): Promise { + return measureStorage("update", key, () => transaction((tx) => tx.update(key, change))) } - export async function writeBinary(key: string[], content: Uint8Array) { - const target = path.join(resolveDir(), ...key) + ".bin" - return measureStorage("write", key, async () => - withErrorHandling(async () => { - using _ = await Lock.write(target) - await writeFileAtomic(target, content, { private: true, durable: true }) - ObservabilityResources.addWrite(content.byteLength) - }), - ) + export function remove(key: string[]) { + return measureStorage("remove", key, () => transaction((tx) => tx.remove(key))) } - export async function readBinary(key: string[], options?: { maxBytes?: number }): Promise { - const target = path.join(resolveDir(), ...key) + ".bin" - return measureStorage("read", key, async () => - withErrorHandling(async () => { - using _ = await Lock.read(target) - const file = Bun.file(target) - if (options?.maxBytes !== undefined && file.size > options.maxBytes) { - throw new Error("Binary record exceeds its byte limit") - } - const content = await file.bytes() - ObservabilityResources.addRead(content.byteLength) - return content - }), - ) + export function removeTree(prefix: string[]) { + return measureStorage("removeTree", prefix, () => transaction((tx) => tx.removeTree(prefix))) } - export async function removeTree(prefix: string[]) { - const dir = resolveDir() - const target = path.join(dir, ...prefix) - await fs.rm(target, { recursive: true, force: true }) - await pruneEmptyParents(path.dirname(target), dir) + export function scan(prefix: string[]) { + return measureStorage("scan", prefix, () => snapshot((tx) => tx.scan(prefix))) } - async function pruneEmptyParents(current: string, root: string) { - while (current !== root && current.startsWith(root)) { - try { - const entries = await fs.readdir(current) - if (entries.length > 0) break - await fs.rmdir(current) - current = path.dirname(current) - } catch { - break - } + export function list(prefix: string[]) { + return measureStorage("list", prefix, () => snapshot((tx) => tx.list(prefix))) + } + + export function query(input: RecordQuery) { + return snapshot((tx) => tx.query(input)) + } + + export async function* records(input: Omit = {}) { + let after: string[] | undefined + for (;;) { + const page = await query({ ...input, after, limit: input.limit ?? 256 }) + if (!page.length) return + yield* page + after = page.at(-1)!.key + } + } + + function binaryPath(key: string[]) { + if (!key.length || key.some((part) => !part || part === "." || part === ".." || /[\\/\0]/.test(part))) + throw new StorageConflictError("Invalid artifact key") + return path.join(current().artifactDirectory, ...key) + ".bin" + } + + export async function writeBinary(key: string[], content: Uint8Array) { + await AtomicFile.writeFileAtomic(binaryPath(key), content, { private: true, durable: true }) + ObservabilityResources.addWrite(content.byteLength) + } + + export async function readBinary(key: string[], options?: { maxBytes?: number }): Promise { + const file = Bun.file(binaryPath(key)) + if (options?.maxBytes !== undefined && file.size > options.maxBytes) + throw new StorageConflictError("Binary record exceeds its byte limit") + try { + const content = await file.bytes() + ObservabilityResources.addRead(content.byteLength) + return content + } catch (error) { + if (error && typeof error === "object" && "code" in error && error.code === "ENOENT") + throw new NotFoundError({ message: "Artifact does not exist" }) + throw error } } @@ -245,114 +272,4 @@ export namespace Storage { } } } - - async function withErrorHandling(body: () => Promise) { - return body().catch((e) => { - if (!(e instanceof Error)) throw e - const errnoException = e as NodeJS.ErrnoException - if (errnoException.code === "ENOENT") { - throw new NotFoundError({ message: `Resource not found: ${errnoException.path}` }) - } - throw e - }) - } - - const glob = new Bun.Glob("**/*") - export async function list(prefix: string[]) { - const dir = resolveDir() - return measureStorage("list", prefix, async () => { - try { - const result = await Array.fromAsync( - glob.scan({ - cwd: path.join(dir, ...prefix), - onlyFiles: true, - }), - ).then((results) => - results - .filter((x) => x.endsWith(".json") && !isTempFile(path.basename(x))) - .map((x) => [...prefix, ...x.slice(0, -5).split(path.sep)]), - ) - result.sort() - return result - } catch { - return [] - } - }) - } - - // Windows maps rename onto MoveFileEx: when another process (antivirus, - // OneDrive, a cross-process reader of these JSON files) briefly holds a - // handle on the source or target without FILE_SHARE_DELETE, the rename - // fails with EPERM/EACCES. Sharing violations clear within milliseconds, - // so retry the whole write+rename sequence with short backoff instead of - // failing session persistence and terminating the owning session (#1247). - const ATOMIC_WRITE_ATTEMPTS = 4 - const ATOMIC_WRITE_RETRY_BASE_MS = 50 - const ATOMIC_WRITE_RETRY_MAX_MS = 200 - - export async function writeJsonAtomic(target: string, serialized: string, options?: WriteOptions) { - return writeFileAtomic(target, serialized, options) - } - - async function writeFileAtomic(target: string, content: string | Uint8Array, options?: WriteOptions) { - await fs.mkdir(path.dirname(target), { recursive: true, ...(options?.private ? { mode: 0o700 } : {}) }) - const tmp = path.join( - path.dirname(target), - `.tmp-${process.pid}-${Date.now().toString(36)}-${Math.random().toString(36).slice(2)}`, - ) - for (let attempt = 1; ; attempt++) { - try { - if (options?.private || options?.durable) { - const file = await fs.open(tmp, "w", options.private ? 0o600 : 0o666) - try { - await file.writeFile(content) - if (options.durable) await file.sync() - } finally { - await file.close() - } - } else { - await Bun.write(tmp, content) - } - await fs.rename(tmp, target) - if (options?.durable && process.platform !== "win32") { - const directory = await fs.open(path.dirname(target), "r") - try { - await directory.sync() - } finally { - await directory.close() - } - } - return - } catch (error) { - if (!isRetryableIOError(error) || attempt >= ATOMIC_WRITE_ATTEMPTS) { - await removeTempFile(tmp) - throw error - } - await new Promise((resolve) => setTimeout(resolve, atomicRetryDelayMs(attempt))) - } - } - } - - // The terminal-failure cleanup can hit the same Windows sharing violation - // that failed the rename (antivirus holding the temp handle), so transient - // unlink errors retry with the same backoff before being suppressed (#1247). - async function removeTempFile(tmp: string) { - for (let attempt = 1; attempt <= ATOMIC_WRITE_ATTEMPTS; attempt++) { - try { - await fs.unlink(tmp) - return - } catch (error) { - if (!isRetryableIOError(error) || attempt >= ATOMIC_WRITE_ATTEMPTS) return - await new Promise((resolve) => setTimeout(resolve, atomicRetryDelayMs(attempt))) - } - } - } - - function atomicRetryDelayMs(attempt: number) { - return Math.min(ATOMIC_WRITE_RETRY_MAX_MS, ATOMIC_WRITE_RETRY_BASE_MS * 2 ** (attempt - 1)) - } - - function isTempFile(name: string) { - return name.includes(".tmp-") || name.endsWith(".tmp") - } } diff --git a/packages/harness/src/storage/transactional-store.ts b/packages/harness/src/storage/transactional-store.ts new file mode 100644 index 000000000..694d23808 --- /dev/null +++ b/packages/harness/src/storage/transactional-store.ts @@ -0,0 +1,683 @@ +import type { StorageEntry } from "./portable" +import { createHash, randomUUID } from "node:crypto" +import { + NotFoundError, + StorageClosedError, + StorageConflictError, + StorageIntegrityError, + StorageOwnershipError, +} from "./errors" +import { StorageQueue } from "./queue" +import { SqliteDriver } from "./sqlite-driver" +import { PostgresDriver } from "./postgres-driver" +import type { SqlConnection, SqlDriver, SqlRow, SqlValue, StoreOptions } from "./sql-contract" + +export type { StoreOptions } from "./sql-contract" +export interface StoredEvent { + id: string + scopeID: string + type: string + payload: unknown +} +export interface TransactionOptions { + operationID?: string + requestHash?: string +} +export interface RecordQuery { + kind?: string + scopeID?: string + sessionID?: string + messageID?: string + after?: string[] + limit?: number + descending?: boolean +} +export interface StoredRecord { + key: string[] + value: T + revision: bigint +} + +type RecordRow = SqlRow & { key_text: string; body: string | null; revision: bigint | number | string } + +function keyID(key: readonly string[]) { + if (!Array.isArray(key) || key.some((value) => typeof value !== "string" || value.length === 0)) + throw new StorageIntegrityError("A storage key must contain nonempty string segments") + return createHash("sha256").update(JSON.stringify(key)).digest("hex") +} + +function encode(value: unknown) { + const result = JSON.stringify(value) + if (result === undefined) throw new StorageIntegrityError("A storage record must be JSON serializable") + return result +} + +function metadata(key: string[]) { + if (key[0] === "sessions") { + const message = key[3] === "messages" + return { + kind: message + ? key[5] === "parts" + ? "part" + : "message" + : key[3] === "info" + ? "session" + : key[3] === "inbox" + ? "inbox" + : (key[3] ?? "session-record"), + scope: key[1] ?? "", + session: key[2] ?? "", + message: message ? (key[4] ?? "") : "", + order: key.at(-1) === "info" ? key.at(-2)! : key.at(-1)!, + } + } + return { + kind: key[0], + scope: key[0] === "projects" ? (key[1] ?? "") : "", + session: "", + message: "", + order: key.at(-1)!, + } +} + +const schema = [ + "CREATE TABLE IF NOT EXISTS storage_namespaces (namespace TEXT PRIMARY KEY, version INTEGER NOT NULL, owner TEXT NOT NULL, state TEXT NOT NULL, next_event BIGINT NOT NULL DEFAULT 0)", + "CREATE TABLE IF NOT EXISTS storage_nodes (namespace TEXT NOT NULL, key_id TEXT NOT NULL, parent_id TEXT NOT NULL, key_text TEXT NOT NULL, segment TEXT NOT NULL, PRIMARY KEY(namespace, key_id))", + "CREATE INDEX IF NOT EXISTS storage_nodes_parent ON storage_nodes(namespace, parent_id)", + "CREATE TABLE IF NOT EXISTS storage_records (namespace TEXT NOT NULL, key_id TEXT NOT NULL, key_text TEXT NOT NULL, body TEXT, revision BIGINT NOT NULL, kind TEXT NOT NULL, scope_id TEXT NOT NULL, session_id TEXT NOT NULL, message_id TEXT NOT NULL, order_key TEXT NOT NULL, updated BIGINT NOT NULL, PRIMARY KEY(namespace, key_id))", + "CREATE INDEX IF NOT EXISTS storage_records_session ON storage_records(namespace, session_id, kind, order_key, key_id)", + "CREATE INDEX IF NOT EXISTS storage_records_scope ON storage_records(namespace, scope_id, kind, updated, key_id)", + "CREATE INDEX IF NOT EXISTS storage_records_message ON storage_records(namespace, message_id, kind, order_key, key_id)", + "CREATE INDEX IF NOT EXISTS storage_records_kind ON storage_records(namespace, kind, order_key, key_id)", + "CREATE TABLE IF NOT EXISTS storage_receipts (namespace TEXT NOT NULL, operation_id TEXT NOT NULL, request_hash TEXT NOT NULL, result TEXT NOT NULL, created BIGINT NOT NULL, PRIMARY KEY(namespace, operation_id))", + "CREATE TABLE IF NOT EXISTS storage_events (namespace TEXT NOT NULL, id TEXT NOT NULL, scope_id TEXT NOT NULL, type TEXT NOT NULL, payload TEXT NOT NULL, position BIGINT NOT NULL, PRIMARY KEY(namespace, id))", + "CREATE INDEX IF NOT EXISTS storage_events_pending ON storage_events(namespace, position)", +] + +export class StoreTransaction { + private active = true + private failure?: unknown + private readonly connection: SqlConnection + constructor( + connection: SqlConnection, + readonly namespace: string, + private readonly readonly = false, + ) { + this.connection = { + query: async (statement: string, values?: SqlValue[]) => { + this.check() + try { + return await connection.query(statement, values) + } catch (error) { + this.failure = error + throw error + } + }, + } + } + + assertHealthy() { + if (this.failure) throw this.failure + } + + finish() { + this.active = false + } + + private check(write = false) { + if (!this.active) throw new StorageClosedError() + this.assertHealthy() + if (write && this.readonly) throw new StorageConflictError("Cannot write from a read-only snapshot") + } + + private async row(key: string[]) { + this.check() + const [row] = await this.connection.query( + "SELECT key_text, body, revision FROM storage_records WHERE namespace = ? AND key_id = ?", + [this.namespace, keyID(key)], + ) + if (row && row.key_text !== JSON.stringify(key)) throw new StorageIntegrityError("Logical key identity collision") + return row + } + + async versioned(key: string[]): Promise> { + const row = await this.row(key) + if (!row || row.body === null) throw new NotFoundError({ message: "Storage record does not exist" }) + return { key, value: JSON.parse(row.body) as T, revision: BigInt(row.revision) } + } + + async read(key: string[]): Promise { + return (await this.versioned(key)).value + } + + async readMany(keys: string[][]): Promise<(T | undefined)[]> { + this.check() + const result: (T | undefined)[] = [] + for (let offset = 0; offset < keys.length; offset += 128) { + const batch = keys.slice(offset, offset + 128) + const rows = await this.connection.query( + `SELECT key_id, key_text, body, revision FROM storage_records WHERE namespace = ? AND key_id IN (${batch.map(() => "?").join(",")})`, + [this.namespace, ...batch.map(keyID)], + ) + const index = new Map(rows.map((row) => [row.key_id, row])) + for (const key of batch) { + const row = index.get(keyID(key)) + if (row && row.key_text !== JSON.stringify(key)) + throw new StorageIntegrityError("Logical key identity collision") + result.push(row?.body ? (JSON.parse(row.body) as T) : undefined) + } + } + return result + } + + async write(key: string[], value: T, options: { expectedRevision?: bigint } = {}): Promise { + this.check(true) + if (!key.length) throw new StorageIntegrityError("Cannot write the storage root") + const previous = await this.row(key) + const revision = previous ? BigInt(previous.revision) : 0n + if (options.expectedRevision !== undefined && options.expectedRevision !== revision) + throw new StorageConflictError("Storage revision changed") + const nodes: SqlValue[] = [] + for (let depth = 1; depth <= key.length; depth++) { + const prefix = key.slice(0, depth) + nodes.push(this.namespace, keyID(prefix), keyID(prefix.slice(0, -1)), JSON.stringify(prefix), prefix.at(-1)!) + } + await this.connection.query( + `INSERT INTO storage_nodes(namespace, key_id, parent_id, key_text, segment) VALUES ${key.map(() => "(?, ?, ?, ?, ?)").join(",")} ON CONFLICT(namespace, key_id) DO NOTHING`, + nodes, + ) + const meta = metadata(key) + await this.connection.query( + "INSERT INTO storage_records(namespace, key_id, key_text, body, revision, kind, scope_id, session_id, message_id, order_key, updated) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) ON CONFLICT(namespace, key_id) DO UPDATE SET body = excluded.body, revision = excluded.revision, kind = excluded.kind, scope_id = excluded.scope_id, session_id = excluded.session_id, message_id = excluded.message_id, order_key = excluded.order_key, updated = excluded.updated", + [ + this.namespace, + keyID(key), + JSON.stringify(key), + encode(value), + revision + 1n, + meta.kind, + meta.scope, + meta.session, + meta.message, + meta.order, + Date.now(), + ], + ) + } + + async update(key: string[], change: (value: T) => void): Promise { + this.check(true) + const before = await this.versioned(key) + change(before.value) + await this.write(key, before.value, { expectedRevision: before.revision }) + return before.value + } + + async remove(key: string[]): Promise { + this.check(true) + await this.connection.query( + "UPDATE storage_records SET body = NULL, revision = revision + 1, updated = ? WHERE namespace = ? AND key_id = ? AND body IS NOT NULL", + [Date.now(), this.namespace, keyID(key)], + ) + } + + async scan(prefix: string[]): Promise { + this.check() + const rows = await this.connection.query( + "WITH RECURSIVE tree(key_id, child) AS (SELECT key_id, segment FROM storage_nodes WHERE namespace = ? AND parent_id = ? UNION ALL SELECT node.key_id, tree.child FROM storage_nodes node JOIN tree ON node.parent_id = tree.key_id WHERE node.namespace = ?) SELECT DISTINCT tree.child FROM tree JOIN storage_records record ON record.key_id = tree.key_id AND record.namespace = ? WHERE record.body IS NOT NULL", + [this.namespace, keyID(prefix), this.namespace, this.namespace], + ) + return rows.map((row) => row.child).sort() + } + + async list(prefix: string[]): Promise { + this.check() + const rows = await this.connection.query( + "WITH RECURSIVE tree(key_id) AS (SELECT key_id FROM storage_nodes WHERE namespace = ? AND parent_id = ? UNION ALL SELECT node.key_id FROM storage_nodes node JOIN tree ON node.parent_id = tree.key_id WHERE node.namespace = ?) SELECT record.key_text FROM tree JOIN storage_records record ON record.key_id = tree.key_id AND record.namespace = ? WHERE record.body IS NOT NULL", + [this.namespace, keyID(prefix), this.namespace, this.namespace], + ) + return rows.map((row) => JSON.parse(row.key_text) as string[]).sort() + } + + async removeTree(prefix: string[]): Promise { + this.check(true) + if (!prefix.length) { + await this.connection.query( + "UPDATE storage_records SET body = NULL, revision = revision + 1, updated = ? WHERE namespace = ? AND body IS NOT NULL", + [Date.now(), this.namespace], + ) + return + } + await this.connection.query( + "WITH RECURSIVE tree(key_id) AS (SELECT key_id FROM storage_nodes WHERE namespace = ? AND key_id = ? UNION ALL SELECT node.key_id FROM storage_nodes node JOIN tree ON node.parent_id = tree.key_id WHERE node.namespace = ?) UPDATE storage_records SET body = NULL, revision = revision + 1, updated = ? WHERE namespace = ? AND key_id IN (SELECT key_id FROM tree) AND body IS NOT NULL", + [this.namespace, keyID(prefix), this.namespace, Date.now(), this.namespace], + ) + } + + async query(input: RecordQuery): Promise[]> { + this.check() + const conditions = ["namespace = ?", "body IS NOT NULL"] + const values: SqlValue[] = [this.namespace] + for (const [field, value] of [ + ["kind", input.kind], + ["scope_id", input.scopeID], + ["session_id", input.sessionID], + ["message_id", input.messageID], + ] as const) { + if (value === undefined) continue + conditions.push(`${field} = ?`) + values.push(value) + } + if (input.after !== undefined) { + const comparison = input.descending ? "<" : ">" + conditions.push(`(order_key ${comparison} ? OR (order_key = ? AND key_id ${comparison} ?))`) + const order = metadata(input.after).order + values.push(order, order, keyID(input.after)) + } + const limit = input.limit ?? 100 + if (!Number.isSafeInteger(limit) || limit < 1 || limit > 10000) + throw new StorageIntegrityError("Invalid storage page limit") + values.push(limit) + const direction = input.descending ? "DESC" : "ASC" + const rows = await this.connection.query( + `SELECT key_text, body, revision FROM storage_records WHERE ${conditions.join(" AND ")} ORDER BY order_key ${direction}, key_id ${direction} LIMIT ?`, + values, + ) + return rows.map((row) => ({ + key: JSON.parse(row.key_text) as string[], + value: JSON.parse(row.body!) as T, + revision: BigInt(row.revision), + })) + } + + async *exportEntries(): AsyncGenerator { + let after: string[] | undefined + for (;;) { + const page = await this.query({ after, limit: 256 }) + if (!page.length) break + for (const record of page) + yield { type: "record", key: record.key, value: record.value, revision: record.revision.toString() } + after = page.at(-1)!.key + } + let operationID = "" + for (;;) { + const page = await this.connection.query( + "SELECT operation_id, request_hash, result, created FROM storage_receipts WHERE namespace = ? AND operation_id > ? ORDER BY operation_id LIMIT 256", + [this.namespace, operationID], + ) + if (!page.length) break + for (const row of page) + yield { + type: "receipt", + operationID: String(row.operation_id), + requestHash: String(row.request_hash), + result: String(row.result), + created: Number(row.created), + } + operationID = String(page.at(-1)!.operation_id) + } + let position = 0n + for (;;) { + const page = await this.connection.query( + "SELECT id, scope_id, type, payload, position FROM storage_events WHERE namespace = ? AND position > ? ORDER BY position LIMIT 256", + [this.namespace, position], + ) + if (!page.length) break + for (const row of page) + yield { + type: "event", + id: String(row.id), + scopeID: String(row.scope_id), + eventType: String(row.type), + payload: JSON.parse(String(row.payload)) as unknown, + } + position = BigInt(page.at(-1)!.position as bigint) + } + } + + async assertNotDeleted(key: string[]) { + this.check() + const [record] = await this.connection.query( + "SELECT body FROM storage_records WHERE namespace = ? AND key_id = ?", + [this.namespace, keyID(key)], + ) + if (record && record.body === null) + throw new StorageConflictError("A deleted record cannot be revived by a delayed writer") + } + + async restoreEntry(entry: StorageEntry) { + this.check(true) + if (entry.type === "record") { + if (BigInt(entry.revision) > 9223372036854775807n) throw new StorageIntegrityError("Unsupported record revision") + if ((await this.readMany([entry.key]))[0] !== undefined) + throw new StorageConflictError("Portable record conflicts with existing target data") + await this.write(entry.key, entry.value) + await this.connection.query( + "UPDATE storage_records SET revision = CASE WHEN revision > ? THEN revision ELSE ? END WHERE namespace = ? AND key_id = ?", + [BigInt(entry.revision), BigInt(entry.revision), this.namespace, keyID(entry.key)], + ) + return + } + if (entry.type === "event") { + await this.enqueue({ id: entry.id, scopeID: entry.scopeID, type: entry.eventType, payload: entry.payload }) + return + } + const [existing] = await this.connection.query( + "SELECT request_hash, result FROM storage_receipts WHERE namespace = ? AND operation_id = ?", + [this.namespace, entry.operationID], + ) + if (existing) { + if (existing.request_hash !== entry.requestHash || existing.result !== entry.result) + throw new StorageConflictError("Command receipt conflicts with existing target data") + return + } + JSON.parse(entry.result) + await this.connection.query( + "INSERT INTO storage_receipts(namespace, operation_id, request_hash, result, created) VALUES (?, ?, ?, ?, ?)", + [this.namespace, entry.operationID, entry.requestHash, entry.result, entry.created], + ) + } + + async enqueue(event: StoredEvent): Promise { + this.check(true) + const [counter] = await this.connection.query( + "UPDATE storage_namespaces SET next_event = next_event + 1 WHERE namespace = ? RETURNING next_event", + [this.namespace], + ) + await this.connection.query( + "INSERT INTO storage_events(namespace, id, scope_id, type, payload, position) VALUES (?, ?, ?, ?, ?, ?) ON CONFLICT(namespace, id) DO NOTHING", + [this.namespace, event.id, event.scopeID, event.type, encode(event.payload), counter.next_event], + ) + } +} + +export class TransactionalStore { + private readonly writes = new StorageQueue() + private readonly owner = randomUUID() + private closing?: Promise + private constructor( + private readonly driver: SqlDriver, + readonly options: StoreOptions, + ) {} + + static async open(options: StoreOptions) { + if (!/^[a-zA-Z0-9_-]{1,128}$/.test(options.namespace)) throw new StorageIntegrityError("Invalid storage namespace") + const driver = + options.backend === "sqlite" + ? await SqliteDriver.open(options.filename, options.readonly, options.mustExist) + : await PostgresDriver.open(options.url, options.namespace, options.maxConnections, options.readonly) + const store = new TransactionalStore(driver, options) + try { + await driver.transaction( + async (connection) => { + if (!options.readonly) for (const statement of schema) await connection.query(statement) + const [existing] = await connection.query( + "SELECT version, owner, state FROM storage_namespaces WHERE namespace = ?", + [options.namespace], + ) + if (options.mustExist && !existing) throw new StorageIntegrityError("The active storage namespace is missing") + if (existing && Number(existing.version) !== 1) + throw new StorageIntegrityError("Unsupported authoritative storage version") + if (options.readonly) { + if (!existing) throw new StorageIntegrityError("Storage namespace does not exist") + return + } + if (existing?.state === "active" && options.backend === "postgres" && !options.recover) + throw new StorageOwnershipError( + "The previous PostgreSQL Runtime did not release ownership; verify it has stopped before recovering", + ) + await connection.query( + "INSERT INTO storage_namespaces(namespace, version, owner, state) VALUES (?, 1, ?, 'active') ON CONFLICT(namespace) DO UPDATE SET owner = excluded.owner, state = excluded.state", + [options.namespace, store.owner], + ) + }, + { readOnly: options.readonly }, + ) + return store + } catch (error) { + await driver.close() + throw error + } + } + + private check() { + if (this.closing) throw new StorageClosedError() + } + + async snapshot(body: (snapshot: StoreTransaction) => Promise): Promise { + this.check() + return this.driver.transaction( + async (connection) => { + const snapshot = new StoreTransaction(connection, this.options.namespace, true) + try { + const result = await body(snapshot) + snapshot.assertHealthy() + return result + } finally { + snapshot.finish() + } + }, + { readOnly: true }, + ) + } + + transaction(body: (tx: StoreTransaction) => Promise, options: TransactionOptions = {}): Promise { + this.check() + if (this.options.readonly) return Promise.reject(new StorageConflictError("The store is read-only")) + if (options.operationID && !options.requestHash) + return Promise.reject(new StorageIntegrityError("Idempotent commands require a request hash")) + return this.writes.run(async () => { + for (let attempt = 0; ; attempt++) { + try { + return await this.driver.transaction(async (connection) => { + const [identity] = await connection.query( + "SELECT owner, state FROM storage_namespaces WHERE namespace = ?" + + (this.driver.backend === "postgres" ? " FOR SHARE" : ""), + [this.options.namespace], + ) + if (!identity || identity.owner !== this.owner || identity.state !== "active") + throw new StorageOwnershipError("Runtime no longer owns this data namespace") + if (options.operationID) { + const [receipt] = await connection.query( + "SELECT request_hash, result FROM storage_receipts WHERE namespace = ? AND operation_id = ?", + [this.options.namespace, options.operationID], + ) + if (receipt) { + if (receipt.request_hash !== options.requestHash) + throw new StorageConflictError("Operation ID was already used for different input") + return (JSON.parse(String(receipt.result)) as { value: T }).value + } + } + const tx = new StoreTransaction(connection, this.options.namespace) + try { + const result = await body(tx) + tx.assertHealthy() + if (options.operationID) + await connection.query( + "INSERT INTO storage_receipts(namespace, operation_id, request_hash, result, created) VALUES (?, ?, ?, ?, ?)", + [ + this.options.namespace, + options.operationID, + options.requestHash!, + encode({ value: result }), + Date.now(), + ], + ) + return result + } finally { + tx.finish() + } + }, options) + } catch (error) { + const code = error && typeof error === "object" && "code" in error ? String(error.code) : undefined + if ( + this.driver.backend === "postgres" || + attempt >= 2 || + !["SQLITE_BUSY", "SQLITE_BUSY_SNAPSHOT"].includes(code ?? "") + ) + throw error + await Bun.sleep(10 * 2 ** attempt + Math.floor(Math.random() * 10)) + } + } + }) + } + + read(key: string[]) { + return this.snapshot((tx) => tx.read(key)) + } + versioned(key: string[]) { + return this.snapshot((tx) => tx.versioned(key)) + } + readMany(keys: string[][]) { + return this.snapshot((tx) => tx.readMany(keys)) + } + write(key: string[], value: T) { + return this.transaction((tx) => tx.write(key, value)) + } + update(key: string[], change: (value: T) => void) { + return this.transaction((tx) => tx.update(key, change)) + } + remove(key: string[]) { + return this.transaction((tx) => tx.remove(key)) + } + removeTree(prefix: string[]) { + return this.transaction((tx) => tx.removeTree(prefix)) + } + scan(prefix: string[]) { + return this.snapshot((tx) => tx.scan(prefix)) + } + list(prefix: string[]) { + return this.snapshot((tx) => tx.list(prefix)) + } + query(input: RecordQuery) { + return this.snapshot((tx) => tx.query(input)) + } + + async operationReceipt(operationID: string) { + this.check() + const [receipt] = await this.driver.query( + "SELECT request_hash, result FROM storage_receipts WHERE namespace = ? AND operation_id = ?", + [this.options.namespace, operationID], + ) + return receipt + ? { requestHash: String(receipt.request_hash), result: JSON.parse(String(receipt.result)) as unknown } + : undefined + } + + async pendingEventCount(): Promise { + this.check() + const [row] = await this.driver.query("SELECT COUNT(*) AS count FROM storage_events WHERE namespace = ?", [ + this.options.namespace, + ]) + return Number(row.count) + } + + async pendingEvents(limit = 100): Promise { + this.check() + const rows = await this.driver.query( + "SELECT id, scope_id, type, payload FROM storage_events WHERE namespace = ? ORDER BY position LIMIT ?", + [this.options.namespace, limit], + ) + return rows.map((row) => ({ + id: String(row.id), + scopeID: String(row.scope_id), + type: String(row.type), + payload: JSON.parse(String(row.payload)) as unknown, + })) + } + + async acknowledgeEvents(ids: string[]): Promise { + this.check() + await this.writes.run(() => + this.driver.transaction(async (connection) => { + const [owner] = await connection.query("SELECT owner FROM storage_namespaces WHERE namespace = ?", [ + this.options.namespace, + ]) + if (this.options.readonly || owner?.owner !== this.owner) + throw new StorageOwnershipError("Cannot acknowledge events without Runtime ownership") + for (const id of ids) + await connection.query("DELETE FROM storage_events WHERE namespace = ? AND id = ?", [ + this.options.namespace, + id, + ]) + }), + ) + } + + async verify() { + this.check() + return this.driver.transaction( + async (connection) => { + if (this.driver.backend === "sqlite") { + const rows = await connection.query("PRAGMA integrity_check") + if (rows.length !== 1 || rows[0].integrity_check !== "ok") + throw new StorageIntegrityError("SQLite integrity verification failed") + } + const tx = new StoreTransaction(connection, this.options.namespace, true) + const issues: Array<{ key: string[]; reason: string }> = [] + const kinds: Record = {} + let records = 0 + try { + let after: string[] | undefined + for (;;) { + const batch = await tx.query>({ after, limit: 256 }) + if (!batch.length) break + for (const record of batch) { + records++ + const meta = metadata(record.key) + kinds[meta.kind] = (kinds[meta.kind] ?? 0) + 1 + const key = record.key + if (key[0] !== "sessions") continue + const parents: string[][] = [] + if (key[3] !== "info") parents.push([...key.slice(0, 3), "info"]) + if (meta.kind === "part") parents.push([...key.slice(0, 5), "info"]) + const values = await tx.readMany(parents) + for (const [index, parent] of values.entries()) { + if (parent === undefined) + issues.push({ key, reason: index === 0 ? "missing_session" : "missing_message" }) + } + if (["session", "message", "part"].includes(meta.kind)) { + const expectedID = meta.kind === "part" ? key.at(-1) : key.at(-2) + if (!record.value || typeof record.value !== "object" || record.value.id !== expectedID) + issues.push({ key, reason: "identity_mismatch" }) + } + } + after = batch.at(-1)!.key + } + const [invalid] = await connection.query( + "SELECT COUNT(*) AS count FROM storage_records r LEFT JOIN storage_nodes n ON r.namespace = n.namespace AND r.key_id = n.key_id WHERE r.namespace = ? AND r.body IS NOT NULL AND (n.key_id IS NULL OR n.key_text <> r.key_text OR r.revision < 1)", + [this.options.namespace], + ) + if (Number(invalid.count)) + throw new StorageIntegrityError("Logical storage index integrity verification failed") + return { backend: this.driver.backend, namespace: this.options.namespace, records, kinds, issues } + } finally { + tx.finish() + } + }, + { readOnly: true }, + ) + } + + close(): Promise { + this.closing ??= (async () => { + await this.writes.close() + try { + if (!this.options.readonly) { + const release = (connection: SqlConnection) => + connection.query( + "UPDATE storage_namespaces SET state = 'idle', owner = '' WHERE namespace = ? AND owner = ?", + [this.options.namespace, this.owner], + ) + if (this.driver.backend === "postgres") await release(this.driver) + else await this.driver.transaction(release) + } + } catch (error) { + if (!(error instanceof StorageOwnershipError)) throw error + } finally { + await this.driver.close() + } + })() + return this.closing + } +} diff --git a/packages/harness/test/cortex/cancel-queued-followups.test.ts b/packages/harness/test/cortex/cancel-queued-followups.test.ts index 9f40903e7..220a6f0c2 100644 --- a/packages/harness/test/cortex/cancel-queued-followups.test.ts +++ b/packages/harness/test/cortex/cancel-queued-followups.test.ts @@ -87,12 +87,12 @@ describe("Cortex cancellation fences queued follow-ups", () => { const held = Promise.withResolvers() const release = Promise.withResolvers() const realWrite = Storage.write - using write = spyOn(Storage, "write").mockImplementation(async (key, value, options) => { + using write = spyOn(Storage, "write").mockImplementation(async (key, value) => { if (key.includes(task.sessionID) && key.some((part) => part.startsWith("inb_"))) { held.resolve() await release.promise } - return realWrite(key, value, options) + return realWrite(key, value) }) const delivery = SessionInbox.enqueueMail({ sessionID: task.sessionID, diff --git a/packages/harness/test/lifecycle/runtime.test.ts b/packages/harness/test/lifecycle/runtime.test.ts index 9f724b567..ccbb5a79c 100644 --- a/packages/harness/test/lifecycle/runtime.test.ts +++ b/packages/harness/test/lifecycle/runtime.test.ts @@ -1,3 +1,4 @@ +import { Storage } from "../../src/storage/storage" import { expect, test } from "bun:test" import { RuntimeHandle } from "../../src/lifecycle/runtime" import { ServerProcessLock } from "@ericsanchezok/synergy-harness/util/server-process-lock" @@ -21,6 +22,7 @@ for (const fails of [false, true]) { const owner: RolloutSchema.Owner = { kind: "operation", scopeID: "test", operationID: crypto.randomUUID() } let ownersAtStop: RolloutSchema.Owner[] | undefined const runtime = await RuntimeHandle.open({ + storage: Storage.current(), mode: "oneshot", services: { transport: { @@ -52,6 +54,7 @@ for (const fails of [false, true]) { test("local runtime owns its home without a transport and releases it exactly once", async () => { const calls: string[] = [] const runtime = await RuntimeHandle.open({ + storage: Storage.current(), mode: "oneshot", services: { initializeExtensions: async () => { @@ -65,7 +68,7 @@ test("local runtime owns its home without a transport and releases it exactly on try { expect(runtime.server).toBeUndefined() expect((await ServerProcessLock.read())?.mode).toBe("oneshot") - await expect(RuntimeHandle.open({ mode: "oneshot" })).rejects.toThrow("already owns") + await expect(RuntimeHandle.open({ storage: Storage.current(), mode: "oneshot" })).rejects.toThrow("already owns") await Promise.all([runtime.close(), runtime.close()]) expect(calls).toEqual(["initialize", "dispose"]) expect(await ServerProcessLock.read()).toBeUndefined() @@ -80,6 +83,7 @@ test("startup failure disposes initialized resources and releases home ownership try { await expect( RuntimeHandle.open({ + storage: Storage.current(), mode: "oneshot", services: { initializeExtensions: async () => { diff --git a/packages/harness/test/migration/concurrent-write.test.ts b/packages/harness/test/migration/concurrent-write.test.ts index b52d0c022..5d89e7a87 100644 --- a/packages/harness/test/migration/concurrent-write.test.ts +++ b/packages/harness/test/migration/concurrent-write.test.ts @@ -1,43 +1,23 @@ import { describe, expect, test, afterEach } from "bun:test" -import { mkdirSync, writeFileSync, readFileSync, unlinkSync, existsSync } from "node:fs" -import path from "node:path" +import { Storage } from "../../src/storage/storage" import { MigrationRegistry } from "../../src/migration/registry" import { resetMigrations, runMigrations } from "../../src/migration" import type { Migration } from "../../src/migration/types" -const dataDir = path.join(process.env["SYNERGY_TEST_HOME"]!, ".synergy", "data") const TEST_DOMAIN = "conc-test" - -function domainLogPath(domain: string): string { - return path.join(dataDir, "meta", "migration", `log-${domain}.json`) -} - -function legacyLogPath(): string { - return path.join(dataDir, "meta", "migration", "log.json") -} - -function writeLog(filePath: string, data: Record): void { - mkdirSync(path.dirname(filePath), { recursive: true }) - writeFileSync(filePath, JSON.stringify(data)) -} +const domainLogPath = (domain: string) => ["meta", "migration", `log-${domain}`] +const legacyLogPath = () => ["meta", "migration", "log"] +const writeLog = Storage.write describe("concurrent migration tracking writes", () => { - afterEach(() => { - for (const file of [ - domainLogPath(TEST_DOMAIN), - domainLogPath("library"), - domainLogPath("engram"), - legacyLogPath(), - ]) { - try { - unlinkSync(file) - } catch {} - } + afterEach(async () => { + for (const key of [domainLogPath(TEST_DOMAIN), domainLogPath("library"), domainLogPath("engram"), legacyLogPath()]) + await Storage.remove(key) MigrationRegistry.unregister(TEST_DOMAIN) resetMigrations() }) - test("completion markers from another instance are merged, not overwritten", async () => { + test("completion markers from another task are merged, not overwritten", async () => { let markEntered!: () => void const entered = new Promise((resolve) => (markEntered = resolve)) let releaseA!: () => void @@ -51,19 +31,19 @@ describe("concurrent migration tracking writes", () => { await gateA }, } - // Instance A only knows about mA (e.g. an older CLI whose registry lacks mB). + // Task A only knows about mA (e.g. an older CLI whose registry lacks mB). MigrationRegistry.register(TEST_DOMAIN, [mA]) const runPromise = runMigrations({ output: "silent", targetDomain: TEST_DOMAIN }) await entered - // Instance B (a concurrent process) completes mB while A is still running mA. - writeLog(domainLogPath(TEST_DOMAIN), { "20260806-conc-b": Date.now() }) + // Task B (the same namespace owner) completes mB while A is still running mA. + await writeLog(domainLogPath(TEST_DOMAIN), { "20260806-conc-b": Date.now() }) releaseA() await runPromise - const data = JSON.parse(readFileSync(domainLogPath(TEST_DOMAIN), "utf-8")) + const data = await Storage.read(domainLogPath(TEST_DOMAIN)) expect(data).toHaveProperty("20260806-conc-a") // A's save must not drop B's marker. expect(data).toHaveProperty("20260806-conc-b") @@ -79,14 +59,14 @@ describe("concurrent migration tracking writes", () => { // Old-version single log plus a marker another instance already persisted // in the per-domain log before this instance converts the old format. - writeLog(legacyLogPath(), { "20260806-conc-a": Date.now() }) - writeLog(domainLogPath(TEST_DOMAIN), { "20260806-conc-b": Date.now() }) + await writeLog(legacyLogPath(), { "20260806-conc-a": Date.now() }) + await writeLog(domainLogPath(TEST_DOMAIN), { "20260806-conc-b": Date.now() }) await runMigrations({ output: "silent", targetDomain: TEST_DOMAIN }) // The old single log is consumed. - expect(existsSync(legacyLogPath())).toBe(false) - const data = JSON.parse(readFileSync(domainLogPath(TEST_DOMAIN), "utf-8")) + expect((await Storage.readMany([legacyLogPath()]))[0]).toBeUndefined() + const data = await Storage.read(domainLogPath(TEST_DOMAIN)) expect(data).toHaveProperty("20260806-conc-a") // Conversion must not drop the concurrent per-domain marker. expect(data).toHaveProperty("20260806-conc-b") diff --git a/packages/harness/test/migration/context.test.ts b/packages/harness/test/migration/context.test.ts index 7a6a03c5c..d39affce2 100644 --- a/packages/harness/test/migration/context.test.ts +++ b/packages/harness/test/migration/context.test.ts @@ -1,22 +1,18 @@ import { describe, expect, test, afterEach } from "bun:test" -import { unlinkSync } from "node:fs" -import path from "node:path" +import { Storage } from "../../src/storage/storage" import { MigrationRegistry } from "../../src/migration/registry" import { resetMigrations, runMigrations } from "../../src/migration" import type { MigrationContext } from "../../src/migration/types" -const dataDir = path.join(process.env["SYNERGY_TEST_HOME"]!, ".synergy", "data") const TEST_DOMAIN = "test-context" -function trackingPath(domain: string): string { - return path.join(dataDir, "meta", "migration", `log-${domain}.json`) -} +const trackingPath = (domain: string) => ["meta", "migration", `log-${domain}`] describe("two-param up() receives MigrationContext", () => { - afterEach(() => { + afterEach(async () => { const p = trackingPath(TEST_DOMAIN) try { - unlinkSync(p) + await Storage.remove(p) } catch {} MigrationRegistry.unregister(TEST_DOMAIN) resetMigrations() diff --git a/packages/harness/test/migration/display.test.ts b/packages/harness/test/migration/display.test.ts index d42094454..2636d217b 100644 --- a/packages/harness/test/migration/display.test.ts +++ b/packages/harness/test/migration/display.test.ts @@ -1,24 +1,20 @@ import { describe, expect, test, afterEach } from "bun:test" -import { unlinkSync, writeFileSync, mkdirSync } from "node:fs" -import path from "node:path" +import { Storage } from "../../src/storage/storage" import { MigrationRegistry } from "../../src/migration/registry" import { resetMigrations, runMigrations } from "../../src/migration" -const dataDir = path.join(process.env["SYNERGY_TEST_HOME"]!, ".synergy", "data") const TEST_DOMAIN = "test-display" -function trackingPath(domain: string): string { - return path.join(dataDir, "meta", "migration", `log-${domain}.json`) -} +const trackingPath = (domain: string) => ["meta", "migration", `log-${domain}`] describe("summary display when all domains up to date", () => { let originalWrite: typeof process.stderr.write - afterEach(() => { + afterEach(async () => { process.stderr.write = originalWrite const p = trackingPath(TEST_DOMAIN) try { - unlinkSync(p) + await Storage.remove(p) } catch {} MigrationRegistry.unregister(TEST_DOMAIN) resetMigrations() @@ -38,8 +34,8 @@ describe("summary display when all domains up to date", () => { // Pre-write tracking entry so the migration looks "completed" const p = trackingPath(TEST_DOMAIN) - mkdirSync(path.dirname(p), { recursive: true }) - writeFileSync(p, JSON.stringify({ "20260615-display-test": Date.now() })) + + await Storage.write(p, { "20260615-display-test": Date.now() }) // Capture stderr to inspect output originalWrite = process.stderr.write.bind(process.stderr) diff --git a/packages/harness/test/migration/dry-run.test.ts b/packages/harness/test/migration/dry-run.test.ts index d1c5c9fd2..4d0a6cd6d 100644 --- a/packages/harness/test/migration/dry-run.test.ts +++ b/packages/harness/test/migration/dry-run.test.ts @@ -1,25 +1,15 @@ import { describe, expect, test, afterEach } from "bun:test" -import { existsSync, readFileSync, unlinkSync } from "node:fs" -import path from "node:path" +import { Storage } from "../../src/storage/storage" import { MigrationRegistry } from "../../src/migration/registry" import { resetMigrations, runMigrations } from "../../src/migration" import type { Migration } from "../../src/migration/types" -// Use the preload's test home directory -const dataDir = path.join(process.env["SYNERGY_TEST_HOME"]!, ".synergy", "data") const TEST_DOMAIN = "test-dry-run" - -function trackingPath(domain: string): string { - return path.join(dataDir, "meta", "migration", `log-${domain}.json`) -} +const trackingPath = (domain: string) => ["meta", "migration", `log-${domain}`] describe("runMigrations dry run", () => { - afterEach(() => { - // Clean migration tracking and reset state - const p = trackingPath(TEST_DOMAIN) - try { - unlinkSync(p) - } catch {} + afterEach(async () => { + await Storage.remove(trackingPath(TEST_DOMAIN)) MigrationRegistry.unregister(TEST_DOMAIN) resetMigrations() }) @@ -42,7 +32,7 @@ describe("runMigrations dry run", () => { expect(upWasCalled).toBe(false) }) - test("dryRun: no tracking file is created after dry run", async () => { + test("dryRun: no tracking record is created after dry run", async () => { const testMigration: Migration = { id: "20260602-test-no-tracking", description: "Test dry-run no tracking", @@ -54,10 +44,10 @@ describe("runMigrations dry run", () => { await runMigrations({ dryRun: true, targetDomain: TEST_DOMAIN }) const p = trackingPath(TEST_DOMAIN) - expect(existsSync(p)).toBe(false) + expect((await Storage.readMany([p]))[0]).toBeUndefined() }) - test("non-dryRun: migration executes and tracking file is created", async () => { + test("non-dryRun: migration executes and tracking record is created", async () => { let upWasCalled = false const testMigration: Migration = { @@ -83,9 +73,9 @@ describe("runMigrations dry run", () => { ) const p = trackingPath(TEST_DOMAIN) - expect(existsSync(p)).toBe(true) + expect((await Storage.readMany([p]))[0]).toBeDefined() - const data = JSON.parse(readFileSync(p, "utf-8")) + const data = await Storage.read(p) expect(data).toHaveProperty("20260603-test-executes") }) @@ -121,7 +111,7 @@ describe("runMigrations dry run", () => { expect(calledIds).toEqual([]) const p = trackingPath(TEST_DOMAIN) - expect(existsSync(p)).toBe(false) + expect((await Storage.readMany([p]))[0]).toBeUndefined() }) test("silent output returns summary without writing to stderr", async () => { diff --git a/packages/harness/test/migration/owner-ledger.test.ts b/packages/harness/test/migration/owner-ledger.test.ts index 5a84280db..a00b325bf 100644 --- a/packages/harness/test/migration/owner-ledger.test.ts +++ b/packages/harness/test/migration/owner-ledger.test.ts @@ -9,6 +9,8 @@ test("optional migration owners retain their original ledger and leave unloaded "--eval", ` import assert from "node:assert/strict" + const { StorageMaintenance } = await import("@ericsanchezok/synergy-harness/storage/maintenance") + await using storageHandle = await StorageMaintenance.open({ migrate: false }) const { MigrationRegistry } = await import("@ericsanchezok/synergy-harness/migration/registry") const { runMigrations } = await import("@ericsanchezok/synergy-harness/migration") const { Storage } = await import("@ericsanchezok/synergy-harness/storage/storage") diff --git a/packages/harness/test/migration/registry-lock.test.ts b/packages/harness/test/migration/registry-lock.test.ts index 398f1be15..150803fa7 100644 --- a/packages/harness/test/migration/registry-lock.test.ts +++ b/packages/harness/test/migration/registry-lock.test.ts @@ -11,6 +11,8 @@ test("runtime registration lock rejects late migration domains without changing "--eval", ` import assert from "node:assert/strict" + const { StorageMaintenance } = await import("@ericsanchezok/synergy-harness/storage/maintenance") + await using storageHandle = await StorageMaintenance.open({ migrate: false }) import fs from "node:fs/promises" import path from "node:path" const { MigrationRegistry } = await import(${JSON.stringify(registry)}) @@ -20,7 +22,8 @@ test("runtime registration lock rejects late migration domains without changing const file = path.join(process.env.SYNERGY_TEST_HOME, ".synergy/data/meta/migration/log.json") const legacy = { "optional-late": { id: "optional-late", status: "completed", timestamp: 123 } } await fs.mkdir(path.dirname(file), { recursive: true }) - await Bun.write(file, JSON.stringify(legacy)) + const { Storage } = await import("@ericsanchezok/synergy-harness/storage/storage") + await Storage.write(["meta", "migration", "log"], legacy) const snapshot = MigrationRegistry.list() snapshot.get("known-before-open")[0].id = "changed-via-snapshot" snapshot.get("known-before-open")[0].dependsOn.push("missing") @@ -40,7 +43,7 @@ test("runtime registration lock rejects late migration domains without changing assert.throws(() => MigrationRegistry.register("known-before-open", [...known]), /before opening the runtime/) assert.throws(() => MigrationRegistry.register("optional-late", [{ id: "optional-late", description: "Late", async up() {} }]), /before opening the runtime/) await runMigrations({ targetDomain: "known-before-open", output: "silent" }) - assert.deepEqual(JSON.parse(await Bun.file(file).text()), legacy) + assert.deepEqual(await Storage.read(["meta", "migration", "log"]), legacy) assert.equal(MigrationRegistry.list().has("optional-late"), false) `, ], diff --git a/packages/harness/test/migration/retry.test.ts b/packages/harness/test/migration/retry.test.ts index 1c5c88fc3..1d37e721c 100644 --- a/packages/harness/test/migration/retry.test.ts +++ b/packages/harness/test/migration/retry.test.ts @@ -1,23 +1,15 @@ import { afterEach, beforeEach, describe, expect, test } from "bun:test" -import { unlinkSync } from "node:fs" -import path from "node:path" +import { Storage } from "../../src/storage/storage" import { ensureMigrations, resetMigrations } from "../../src/migration" import { MigrationRegistry } from "../../src/migration/registry" const TEST_DOMAIN = "test-migration-retry" -const trackingPath = path.join( - process.env["SYNERGY_TEST_HOME"]!, - ".synergy", - "data", - "meta", - "migration", - `log-${TEST_DOMAIN}.json`, -) +const trackingPath = ["meta", "migration", `log-${TEST_DOMAIN}`] describe("ensureMigrations failure recovery", () => { - const reset = () => { + const reset = async () => { try { - unlinkSync(trackingPath) + await Storage.remove(trackingPath) } catch {} MigrationRegistry.unregister(TEST_DOMAIN) resetMigrations() diff --git a/packages/harness/test/migration/rollback.test.ts b/packages/harness/test/migration/rollback.test.ts index 7f595d7b5..11dced3b7 100644 --- a/packages/harness/test/migration/rollback.test.ts +++ b/packages/harness/test/migration/rollback.test.ts @@ -1,22 +1,18 @@ import { describe, expect, test, afterEach } from "bun:test" -import { existsSync, readFileSync, unlinkSync, writeFileSync } from "node:fs" -import path from "node:path" +import { Storage } from "../../src/storage/storage" import { MigrationRegistry } from "../../src/migration/registry" import { rollbackMigrations, resetMigrations, runMigrations } from "../../src/migration" import type { Migration } from "../../src/migration/types" -const dataDir = path.join(process.env["SYNERGY_TEST_HOME"]!, ".synergy", "data") const TEST_DOMAIN = "test-rollback" -function trackingPath(domain: string): string { - return path.join(dataDir, "meta", "migration", `log-${domain}.json`) -} +const trackingPath = (domain: string) => ["meta", "migration", `log-${domain}`] describe("rollbackMigrations", () => { - afterEach(() => { + afterEach(async () => { const p = trackingPath(TEST_DOMAIN) try { - unlinkSync(p) + await Storage.remove(p) } catch {} MigrationRegistry.unregister(TEST_DOMAIN) resetMigrations() @@ -60,7 +56,7 @@ describe("rollbackMigrations", () => { // down() iterates toRollback in the collected order: [b, a] expect(called).toEqual(["b", "a"]) - const after = JSON.parse(readFileSync(trackingPath(TEST_DOMAIN), "utf-8")) + const after = await Storage.read>(trackingPath(TEST_DOMAIN)) expect(after).not.toHaveProperty("20260605-rb-a") expect(after).not.toHaveProperty("20260605-rb-b") expect(after).toHaveProperty("20260605-rb-c") @@ -94,7 +90,7 @@ describe("rollbackMigrations", () => { expect(called).toEqual(["a"]) - const after = JSON.parse(readFileSync(trackingPath(TEST_DOMAIN), "utf-8")) + const after = await Storage.read>(trackingPath(TEST_DOMAIN)) expect(after).not.toHaveProperty("20260605b-rb-a") expect(after).toHaveProperty("20260605b-rb-b") }) @@ -135,7 +131,7 @@ describe("rollbackMigrations", () => { expect(called).toEqual(["c", "b", "a"]) - const after = JSON.parse(readFileSync(trackingPath(TEST_DOMAIN), "utf-8")) + const after = await Storage.read>(trackingPath(TEST_DOMAIN)) expect(Object.keys(after)).toHaveLength(0) }) @@ -159,7 +155,7 @@ describe("rollbackMigrations", () => { // Rollback to oldest: m1 (no down) is unmarked, m2 stays await rollbackMigrations(TEST_DOMAIN, "20260607-nodown-a") - const after = JSON.parse(readFileSync(trackingPath(TEST_DOMAIN), "utf-8")) + const after = await Storage.read>(trackingPath(TEST_DOMAIN)) expect(after).not.toHaveProperty("20260607-nodown-a") expect(after).toHaveProperty("20260607-withdown-b") }) @@ -199,9 +195,9 @@ describe("rollbackMigrations", () => { // Manually remove b's tracking entry to create a gap const p = trackingPath(TEST_DOMAIN) - const logData = JSON.parse(readFileSync(p, "utf-8")) + const logData = await Storage.read>(p) delete logData["20260610-gap-b"] - writeFileSync(p, JSON.stringify(logData, null, 2)) + await Storage.write(p, logData) // Rollback newest: starts at c, hits b (untracked) → stops await rollbackMigrations(TEST_DOMAIN, "20260610-gap-c") @@ -209,7 +205,7 @@ describe("rollbackMigrations", () => { // Only c rollback was attempted (b was untracked, breaks the chain) expect(called).toEqual(["c"]) - const after = JSON.parse(readFileSync(p, "utf-8")) + const after = await Storage.read>(p) expect(after).not.toHaveProperty("20260610-gap-c") expect(after).toHaveProperty("20260610-gap-a") }) diff --git a/packages/harness/test/migration/tracking-migration.test.ts b/packages/harness/test/migration/tracking-migration.test.ts index 3e5332e06..c4905c14f 100644 --- a/packages/harness/test/migration/tracking-migration.test.ts +++ b/packages/harness/test/migration/tracking-migration.test.ts @@ -1,46 +1,26 @@ import { describe, expect, test, afterEach } from "bun:test" -import { mkdirSync, writeFileSync, existsSync, unlinkSync, readFileSync } from "node:fs" -import path from "node:path" +import { Storage } from "../../src/storage/storage" import { MigrationRegistry } from "../../src/migration/registry" import { resetMigrations, runMigrations } from "../../src/migration" import type { Migration } from "../../src/migration/types" -const dataDir = path.join(process.env["SYNERGY_TEST_HOME"]!, ".synergy", "data") - -const oldLogPath = path.join(dataDir, "meta", "migration", "log.json") +const oldLogPath = ["meta", "migration", "log"] const TEST_DOMAINS = ["track-test-a", "track-test-b", "track-test-c"] -function domainLogPath(domain: string): string { - return path.join(dataDir, "meta", "migration", `log-${domain}.json`) -} +const domainLogPath = (domain: string) => ["meta", "migration", `log-${domain}`] describe("tracking data migration (log.json → log-{domain}.json)", () => { - afterEach(() => { + afterEach(async () => { // Clean up test tracking files try { - unlinkSync(oldLogPath) + await Storage.remove(oldLogPath) } catch {} for (const domain of TEST_DOMAINS) { try { - unlinkSync(domainLogPath(domain)) + await Storage.remove(domainLogPath(domain)) } catch {} MigrationRegistry.unregister(domain) } - // Remove any other log files in the migration dir - const metaDir = path.join(dataDir, "meta", "migration") - try { - const dir = Array.from(new Bun.Glob("*.json").scanSync({ cwd: metaDir, onlyFiles: true })) - for (const file of dir) { - if (file.startsWith("log-")) { - const p = path.join(metaDir, file) - if (p !== oldLogPath && !TEST_DOMAINS.some((d) => p === domainLogPath(d))) { - try { - unlinkSync(p) - } catch {} - } - } - } - } catch {} resetMigrations() }) @@ -69,13 +49,13 @@ describe("tracking data migration (log.json → log-{domain}.json)", () => { MigrationRegistry.register(TEST_DOMAINS[2], [mC]) // Create the old log.json with entries - mkdirSync(path.dirname(oldLogPath), { recursive: true }) + const oldLog: Record = { "20260609-track-a": now, "20260609-track-b": now + 1, "20260609-track-c": now + 2, } - writeFileSync(oldLogPath, JSON.stringify(oldLog, null, 2)) + await Storage.write(oldLogPath, oldLog) // runMigrations always migrates old tracking data before applying the target // domain filter, so one test domain is enough to exercise the split without @@ -83,13 +63,13 @@ describe("tracking data migration (log.json → log-{domain}.json)", () => { await runMigrations({ output: "silent", targetDomain: TEST_DOMAINS[0] }) // Old log should be deleted - expect(existsSync(oldLogPath)).toBe(false) + expect((await Storage.readMany([oldLogPath]))[0] !== undefined).toBe(false) // Per-domain logs should exist for (const [i, domain] of TEST_DOMAINS.entries()) { const p = domainLogPath(domain) - expect(existsSync(p)).toBe(true) - const data = JSON.parse(readFileSync(p, "utf-8")) + expect((await Storage.readMany([p]))[0] !== undefined).toBe(true) + const data = await Storage.read>(p) const expectedKeys = i === 0 ? ["20260609-track-a"] : i === 1 ? ["20260609-track-b"] : ["20260609-track-c"] for (const key of expectedKeys) { expect(data).toHaveProperty(key) @@ -107,23 +87,23 @@ describe("tracking data migration (log.json → log-{domain}.json)", () => { MigrationRegistry.register(TEST_DOMAINS[0], [mA]) // Create old log - mkdirSync(path.dirname(oldLogPath), { recursive: true }) - writeFileSync(oldLogPath, JSON.stringify({ "20260610-idem-a": Date.now() })) + + await Storage.write(oldLogPath, { "20260610-idem-a": Date.now() }) // First run: migrates old log await runMigrations({ output: "silent", targetDomain: TEST_DOMAINS[0] }) - expect(existsSync(oldLogPath)).toBe(false) + expect((await Storage.readMany([oldLogPath]))[0] !== undefined).toBe(false) - const firstData = JSON.parse(readFileSync(domainLogPath(TEST_DOMAINS[0]), "utf-8")) + const firstData = await Storage.read>(domainLogPath(TEST_DOMAINS[0])) // Clear completed state so we can run again resetMigrations() // Second run: no old log to migrate, no new migrations to run await runMigrations({ output: "silent", targetDomain: TEST_DOMAINS[0] }) - expect(existsSync(oldLogPath)).toBe(false) + expect((await Storage.readMany([oldLogPath]))[0] !== undefined).toBe(false) - const secondData = JSON.parse(readFileSync(domainLogPath(TEST_DOMAINS[0]), "utf-8")) + const secondData = await Storage.read>(domainLogPath(TEST_DOMAINS[0])) expect(secondData).toEqual(firstData) }) @@ -139,32 +119,32 @@ describe("tracking data migration (log.json → log-{domain}.json)", () => { }, }) MigrationRegistry.register(TEST_DOMAINS[0], [migration(knownID)]) - mkdirSync(path.dirname(oldLogPath), { recursive: true }) - writeFileSync(oldLogPath, JSON.stringify({ [knownID]: 100, [deferredID]: 200 })) + + await Storage.write(oldLogPath, { [knownID]: 100, [deferredID]: 200 }) await runMigrations({ output: "silent", targetDomain: TEST_DOMAINS[0] }) - expect(JSON.parse(readFileSync(domainLogPath(TEST_DOMAINS[0]), "utf-8"))).toEqual({ [knownID]: 100 }) - expect(JSON.parse(readFileSync(oldLogPath, "utf-8"))).toEqual({ [deferredID]: 200 }) + expect(await Storage.read>(domainLogPath(TEST_DOMAINS[0]))).toEqual({ [knownID]: 100 }) + expect(await Storage.read>(oldLogPath)).toEqual({ [deferredID]: 200 }) await runMigrations({ output: "silent", targetDomain: TEST_DOMAINS[0] }) - expect(JSON.parse(readFileSync(oldLogPath, "utf-8"))).toEqual({ [deferredID]: 200 }) + expect(await Storage.read>(oldLogPath)).toEqual({ [deferredID]: 200 }) MigrationRegistry.register(TEST_DOMAINS[1], [migration(deferredID)]) await runMigrations({ output: "silent", targetDomain: TEST_DOMAINS[1] }) - expect(JSON.parse(readFileSync(domainLogPath(TEST_DOMAINS[1]), "utf-8"))).toEqual({ [deferredID]: 200 }) - expect(existsSync(oldLogPath)).toBe(false) + expect(await Storage.read>(domainLogPath(TEST_DOMAINS[1]))).toEqual({ [deferredID]: 200 }) + expect((await Storage.readMany([oldLogPath]))[0] !== undefined).toBe(false) expect(executed).toBe(0) }) test("keeps a legacy log unchanged when none of its migrations are registered", async () => { const legacy = { "20260908-uninstalled-domain": 123 } - mkdirSync(path.dirname(oldLogPath), { recursive: true }) - writeFileSync(oldLogPath, JSON.stringify(legacy)) + + await Storage.write(oldLogPath, legacy) await runMigrations({ output: "silent", targetDomain: TEST_DOMAINS[0] }) - expect(JSON.parse(readFileSync(oldLogPath, "utf-8"))).toEqual(legacy) - expect(existsSync(domainLogPath(TEST_DOMAINS[0]))).toBe(false) + expect(await Storage.read>(oldLogPath)).toEqual(legacy) + expect((await Storage.readMany([domainLogPath(TEST_DOMAINS[0])]))[0] !== undefined).toBe(false) }) test("no old log file: migration is a no-op", async () => { @@ -177,8 +157,8 @@ describe("tracking data migration (log.json → log-{domain}.json)", () => { MigrationRegistry.register(TEST_DOMAINS[0], [mA]) // No old log file, and no per-domain log yet - expect(existsSync(oldLogPath)).toBe(false) - expect(existsSync(domainLogPath(TEST_DOMAINS[0]))).toBe(false) + expect((await Storage.readMany([oldLogPath]))[0] !== undefined).toBe(false) + expect((await Storage.readMany([domainLogPath(TEST_DOMAINS[0])]))[0] !== undefined).toBe(false) // This should just run the migration (since it's not tracked) // Actually, running with targetDomain to avoid running all real migrations @@ -186,8 +166,8 @@ describe("tracking data migration (log.json → log-{domain}.json)", () => { // The migration runs and creates the per-domain tracking file const p = domainLogPath(TEST_DOMAINS[0]) - expect(existsSync(p)).toBe(true) - const data = JSON.parse(readFileSync(p, "utf-8")) + expect((await Storage.readMany([p]))[0] !== undefined).toBe(true) + const data = await Storage.read>(p) expect(data).toHaveProperty("20260611-noold-a") }) }) diff --git a/packages/harness/test/session/continuation-kernel-empty-worker.ts b/packages/harness/test/session/continuation-kernel-empty-worker.ts index 7c92e903a..6fd82c81b 100644 --- a/packages/harness/test/session/continuation-kernel-empty-worker.ts +++ b/packages/harness/test/session/continuation-kernel-empty-worker.ts @@ -1,3 +1,7 @@ +import fs from "node:fs/promises" +import os from "node:os" +import path from "node:path" +import { StorageMaintenance } from "../../src/storage/maintenance" import { Log } from "../../src/util/log" import { ContinuationKernel } from "../../src/session/continuation-kernel" @@ -10,6 +14,14 @@ import { ContinuationKernel } from "../../src/session/continuation-kernel" * goes to stderr via Log.init({ print: true }). */ -await Log.init({ print: true }) -const result = await ContinuationKernel.propose("ses_does_not_exist") -console.log(`PROPOSE_RESULT:${result === undefined ? "undefined" : JSON.stringify(result)}`) +const home = await fs.mkdtemp(path.join(os.tmpdir(), "synergy-empty-policy-")) +delete process.env.SYNERGY_HOME +process.env.SYNERGY_TEST_HOME = home +try { + await using storage = await StorageMaintenance.open() + await Log.init({ print: true }) + const result = await ContinuationKernel.propose("ses_does_not_exist") + console.log(`PROPOSE_RESULT:${result === undefined ? "undefined" : JSON.stringify(result)}`) +} finally { + await fs.rm(home, { recursive: true, force: true }) +} diff --git a/packages/harness/test/session/fixtures/restart-while-queued-worker.ts b/packages/harness/test/session/fixtures/restart-while-queued-worker.ts index 8f75deda6..aa6b66a67 100644 --- a/packages/harness/test/session/fixtures/restart-while-queued-worker.ts +++ b/packages/harness/test/session/fixtures/restart-while-queued-worker.ts @@ -1,3 +1,5 @@ +import { StorageMaintenance } from "../../../src/storage/maintenance" +await using storageHandle = await StorageMaintenance.open({ migrate: false, recover: true }) import fs from "fs/promises" import path from "path" import { Global } from "../../../src/global" @@ -115,6 +117,7 @@ if (phase === "enqueue") { }) // Exits without draining the inbox: the queued task must survive in the // durable store for a fresh process to recover. + await storageHandle.close() process.exit(0) } @@ -164,6 +167,7 @@ if (phase === "recover") { ) }, }) + await storageHandle.close() process.exit(0) } diff --git a/packages/harness/test/session/mutation-serialization.test.ts b/packages/harness/test/session/mutation-serialization.test.ts index c753c1911..baf7d5901 100644 --- a/packages/harness/test/session/mutation-serialization.test.ts +++ b/packages/harness/test/session/mutation-serialization.test.ts @@ -19,12 +19,8 @@ function pauseFirstSessionInfoUpdate(infoPath: string[]) { const release = Promise.withResolvers() const originalUpdate = Storage.update let updates = 0 - const spy = spyOn(Storage, "update").mockImplementation((async ( - key: string[], - editor: (draft: T) => void, - options?: Storage.WriteOptions, - ) => { - const result = await originalUpdate(key, editor, options) + const spy = spyOn(Storage, "update").mockImplementation((async (key: string[], editor: (draft: T) => void) => { + const result = await originalUpdate(key, editor) if (sameKey(key, infoPath) && ++updates === 1) { reached.resolve() await release.promise @@ -44,12 +40,12 @@ function pauseFirstIndexWrite(indexPath: string[]) { const release = Promise.withResolvers() const originalWrite = Storage.write let writes = 0 - const spy = spyOn(Storage, "write").mockImplementation(async (key, content, options) => { + const spy = spyOn(Storage, "write").mockImplementation(async (key, content) => { if (sameKey(key, indexPath) && ++writes === 1) { reached.resolve() await release.promise } - return originalWrite(key, content, options) + return originalWrite(key, content) }) return { reached: reached.promise, diff --git a/packages/harness/test/session/part-write-buffer.test.ts b/packages/harness/test/session/part-write-buffer.test.ts index 2dca85b8b..c4791ebe8 100644 --- a/packages/harness/test/session/part-write-buffer.test.ts +++ b/packages/harness/test/session/part-write-buffer.test.ts @@ -12,6 +12,34 @@ function recorder() { } describe("PartWriteBuffer", () => { + test("terminal writes wait for an already executing streaming write", async () => { + const writes: string[] = [] + let release!: () => void + const blocked = new Promise((resolve) => { + release = resolve + }) + const buffer = new PartWriteBuffer(async (_path, value) => { + if (value === "stream") await blocked + writes.push(value) + }, 1) + buffer.defer("part", "part", "stream") + await Bun.sleep(10) + const terminal = buffer.writeNow("part", "part", "complete") + expect(writes).toEqual([]) + release() + await terminal + await buffer.flushAll() + expect(writes).toEqual(["stream", "complete"]) + }) + + test("draining includes timer writes and reports background persistence failure", async () => { + const buffer = new PartWriteBuffer(async () => { + throw new Error("disk full") + }, 1) + buffer.defer("part", "part", "stream") + await Bun.sleep(10) + await expect(buffer.flushAll()).rejects.toThrow("disk full") + }) test("coalesces deferred writes: many defers, one flush writes the latest", () => { const r = recorder() const buf = new PartWriteBuffer(r.write, 10_000) diff --git a/packages/harness/test/session/rollback.test.ts b/packages/harness/test/session/rollback.test.ts index 2f5a682c1..cad990fd5 100644 --- a/packages/harness/test/session/rollback.test.ts +++ b/packages/harness/test/session/rollback.test.ts @@ -583,8 +583,8 @@ describe("rollback acknowledgment", () => { const releaseMetadataUpdate = Promise.withResolvers() const originalUpdate = Storage.update let pauseNextInfoUpdate = true - using _update = spyOn(Storage, "update").mockImplementation(async (key, editor, options) => { - const result = await originalUpdate(key, editor, options) + using _update = spyOn(Storage, "update").mockImplementation(async (key, editor) => { + const result = await originalUpdate(key, editor) if (pauseNextInfoUpdate && key.join("/") === infoPath.join("/")) { pauseNextInfoUpdate = false metadataPersisted.resolve() diff --git a/packages/harness/test/session/rollout-archive.test.ts b/packages/harness/test/session/rollout-archive.test.ts index 02e378a6c..553b14672 100644 --- a/packages/harness/test/session/rollout-archive.test.ts +++ b/packages/harness/test/session/rollout-archive.test.ts @@ -1,3 +1,4 @@ +import { Storage } from "../../src/storage/storage" import { expect, test } from "bun:test" import { Uint8ArrayWriter, Uint8ArrayReader, ZipWriter, ZipReader } from "@zip.js/zip.js" import { fixture, complete } from "@ericsanchezok/synergy-harness/test/support/rollout" @@ -206,6 +207,7 @@ test("rollout ZIP retains file snapshot objects after the source store is remove await RolloutArchive.write({ sessionID: session.id, runID: rootID }, writer) await Session.remove(session.id) await fs.rm(SnapshotStore.root(session.scope.id), { recursive: true, force: true }) + await Storage.removeTree(["snapshot-v2", session.scope.id]) const restored = await SessionImport.fromBuffer(await writer.getData()) try { expect(await SnapshotStore.owns(session.scope.id, restored.rootSessionID, hash)).toBe(true) diff --git a/packages/harness/test/session/rollout-artifact.test.ts b/packages/harness/test/session/rollout-artifact.test.ts index a057c1faf..2f00e52c6 100644 --- a/packages/harness/test/session/rollout-artifact.test.ts +++ b/packages/harness/test/session/rollout-artifact.test.ts @@ -1,3 +1,5 @@ +import fs from "node:fs/promises" +import path from "node:path" import { describe, expect, spyOn, test } from "bun:test" import { RolloutArtifact } from "../../src/session/rollout/artifact" import { RolloutRecordingError } from "../../src/session/rollout/error" @@ -45,7 +47,9 @@ describe("rollout artifacts", () => { const second = await RolloutArtifact.write(target, source(), "application/octet-stream") expect(first.id).not.toBe(second.id) expect(first.sha256).toBe(second.sha256) - expect(await Storage.scan([...RolloutArtifact.root(target), "blobs"], { strict: true })).toHaveLength(1) + expect( + await fs.readdir(path.join(Storage.current().artifactDirectory, ...RolloutArtifact.root(target), "blobs")), + ).toHaveLength(1) }) test("preserves content across producer chunk boundaries", async () => { @@ -112,11 +116,11 @@ describe("rollout artifacts", () => { test("does not publish completion when the final commit fails", async () => { const target = owner() const original = Storage.write.bind(Storage) - using write = spyOn(Storage, "write").mockImplementation(async (key, data, options) => { + using write = spyOn(Storage, "write").mockImplementation(async (key, data) => { if (data && typeof data === "object" && "status" in data && data.status === "complete") { throw Object.assign(new Error("disk full"), { code: "ENOSPC" }) } - return original(key, data, options) + return original(key, data) }) async function* source() { yield new Uint8Array([1]) diff --git a/packages/harness/test/session/rollout-continuation.test.ts b/packages/harness/test/session/rollout-continuation.test.ts index 718034d28..c423b48d7 100644 --- a/packages/harness/test/session/rollout-continuation.test.ts +++ b/packages/harness/test/session/rollout-continuation.test.ts @@ -14,7 +14,7 @@ async function notify(sessionID: string, rootID: string) { noReply: true, parts: [{ type: "text", text: "Child task completed" }], }) - for (const item of await SessionInbox.drainSteer(sessionID)) + for (const item of await SessionInbox.peekSteer(sessionID)) await SessionInbox.materializeItem(item, rootID, { guiding: true }) } diff --git a/packages/harness/test/session/rollout-journal.test.ts b/packages/harness/test/session/rollout-journal.test.ts index b36c2e647..c10ecd9dc 100644 --- a/packages/harness/test/session/rollout-journal.test.ts +++ b/packages/harness/test/session/rollout-journal.test.ts @@ -26,9 +26,9 @@ test("a failed commit never reuses an allocated sequence or overwrites its evide const key = [...root, "runs", "run", "info"] const original = Storage.write.bind(Storage) { - using write = spyOn(Storage, "write").mockImplementation(async (path, value, options) => { + using write = spyOn(Storage, "write").mockImplementation(async (path, value) => { if (path.join("/") === key.join("/")) throw new Error("projection unavailable") - return original(path, value, options) + return original(path, value) }) await expect(RolloutJournal.write(target, key, { status: "running" })).rejects.toMatchObject({ name: "RolloutRecordingError", @@ -49,9 +49,9 @@ test("recovery restores committed projections without replaying execution", asyn const key = [...RolloutArtifact.root(target), "runs", "run", "info"] const original = Storage.write.bind(Storage) { - using write = spyOn(Storage, "write").mockImplementation(async (path, value, options) => { + using write = spyOn(Storage, "write").mockImplementation(async (path, value) => { if (path.join("/") === key.join("/")) throw new Error("interrupted projection") - return original(path, value, options) + return original(path, value) }) await expect(RolloutJournal.write(target, key, { status: "running" })).rejects.toThrow() } @@ -60,19 +60,28 @@ test("recovery restores committed projections without replaying execution", asyn expect(await RolloutJournal.recover(target)).toEqual({ recovered: 0, gaps: [] }) }) -test("a missing reserved event remains an explicit gap at later read boundaries", async () => { +test("an interrupted evidence transaction does not leave a newly allocated gap", async () => { const target = owner() const root = RolloutArtifact.root(target) const key = [...root, "runs", "run", "info"] const original = Storage.write.bind(Storage) { - using write = spyOn(Storage, "write").mockImplementation(async (path, value, options) => { + using write = spyOn(Storage, "write").mockImplementation(async (path, value) => { if (path.includes("events")) throw new Error("interrupted event") - return original(path, value, options) + return original(path, value) }) await expect(RolloutJournal.write(target, key, { status: "running" })).rejects.toThrow() } + expect(await RolloutJournal.head(target)).toEqual({ allocated: 0, committed: 0 }) await RolloutJournal.write(target, key, { status: "failed" }) + expect(await RolloutJournal.head(target)).toEqual({ allocated: 1, committed: 1 }) +}) + +test("a historical missing reserved event remains an explicit gap", async () => { + const target = owner() + const root = RolloutArtifact.root(target) + await Storage.write([...root, "journal", "head"], { allocated: 1, committed: 0 }) + await RolloutJournal.write(target, [...root, "runs", "run", "info"], { status: "failed" }) const events = [] for await (const event of RolloutJournal.events(target, 2)) events.push(event) expect(events[0]).toMatchObject({ seq: 1, kind: "gap" }) diff --git a/packages/harness/test/session/rollout-migration-attachments.test.ts b/packages/harness/test/session/rollout-migration-attachments.test.ts index ca9a3ff38..aa4670d05 100644 --- a/packages/harness/test/session/rollout-migration-attachments.test.ts +++ b/packages/harness/test/session/rollout-migration-attachments.test.ts @@ -105,7 +105,7 @@ test("artifact storage failure still blocks migration and retries after storage mime: "text/plain", url: "data:text/plain;base64,b3JpZ2luYWw=", }) - const directory = path.join(Global.Path.data, ...RolloutArtifact.root(call.owner), "artifacts") + const directory = path.join(Global.Path.data, ...RolloutArtifact.root(call.owner), "blobs") const backup = directory + "-fixture" await mkdir(directory, { recursive: true }) await rename(directory, backup) diff --git a/packages/harness/test/session/schema-registry.test.ts b/packages/harness/test/session/schema-registry.test.ts index 2b4c057c3..cfd2fb421 100644 --- a/packages/harness/test/session/schema-registry.test.ts +++ b/packages/harness/test/session/schema-registry.test.ts @@ -9,6 +9,8 @@ test("core preserves unknown kinds and nested owner metadata through read update "--eval", ` import assert from "node:assert/strict" + const { StorageMaintenance } = await import("@ericsanchezok/synergy-harness/storage/maintenance") + await using storageHandle = await StorageMaintenance.open({ migrate: false }) import z from "zod" const { Scope } = await import("@ericsanchezok/synergy-harness/scope") const { ScopeContext } = await import("@ericsanchezok/synergy-harness/scope/context") diff --git a/packages/harness/test/session/staging.test.ts b/packages/harness/test/session/staging.test.ts new file mode 100644 index 000000000..be59b7c9e --- /dev/null +++ b/packages/harness/test/session/staging.test.ts @@ -0,0 +1,19 @@ +import { expect, test } from "bun:test" +import { tmpdir } from "../support/fixture" +import { Identifier } from "../../src/id/id" +import { SessionStaging } from "../../src/session/staging" +import { Storage } from "../../src/storage/storage" + +test("unpublished Session import identities remain reserved and recover without exposing partial records", async () => { + await using tmp = await tmpdir({ git: true }) + const scope = await tmp.scope() + const sessionID = Identifier.ascending("session") + await SessionStaging.begin(scope.id, [sessionID]) + await Storage.write(["sessions", scope.id, sessionID, "draft"], { retained: true }) + await expect(SessionStaging.begin(scope.id, [sessionID])).rejects.toThrow("reserved") + expect(await Storage.readMany([["session_index", sessionID]])).toEqual([undefined]) + await SessionStaging.recover() + expect(await Storage.list(["sessions", scope.id, sessionID])).toEqual([]) + expect(await Storage.list(["storage_staging"])).toEqual([]) + await SessionStaging.recover() +}) diff --git a/packages/harness/test/session/transaction-atomicity.test.ts b/packages/harness/test/session/transaction-atomicity.test.ts new file mode 100644 index 000000000..38d1eba1b --- /dev/null +++ b/packages/harness/test/session/transaction-atomicity.test.ts @@ -0,0 +1,50 @@ +import { expect, spyOn, test } from "bun:test" +import { Storage } from "../../src/storage/storage" +import { StoragePath } from "../../src/storage/path" +import { Session } from "../../src/session" +import { ScopeContext } from "../../src/scope/context" +import { tmpdir } from "../support/fixture" + +test("a failed session deletion preserves the complete aggregate and snapshot ownership", async () => { + await using tmp = await tmpdir({ git: true }) + const scope = await tmp.scope() + await ScopeContext.provide({ + scope, + fn: async () => { + const session = await Session.create({ title: "must survive" }) + const original = Storage.remove + using failure = spyOn(Storage, "remove").mockImplementation(async (key) => { + if (key[0] === "session_index" && key[1] === session.id) throw new Error("injected delete failure") + return original(key) + }) + await expect(Session.remove(session.id)).rejects.toThrow("injected delete failure") + expect((await Session.get(session.id)).title).toBe("must survive") + expect((await Storage.readMany([StoragePath.snapshotOwner(scope.id, session.id)]))[0]).toBeUndefined() + }, + }) +}) + +test("a failed session creation publishes neither a session nor an index", async () => { + await using tmp = await tmpdir({ git: true }) + const scope = await tmp.scope() + await ScopeContext.provide({ + scope, + fn: async () => { + const original = Storage.write + using failure = spyOn(Storage, "write").mockImplementation(async (key, value) => { + if (key[0] === "session_index") throw new Error("injected index failure") + return original(key, value) + }) + await expect(Session.create({ title: "rollback creation" })).rejects.toThrow("injected index failure") + expect(await Storage.scan(["sessions", scope.id])).toEqual([]) + }, + }) +}) + +test("session deletion resolves its owner without an ambient Scope", async () => { + await using tmp = await tmpdir({ git: true }) + const scope = await tmp.scope() + const session = await ScopeContext.provide({ scope, fn: () => Session.create({ title: "offline deletion" }) }) + await Session.remove(session.id) + expect(await Storage.readMany([["sessions", scope.id, session.id, "info"]])).toEqual([undefined]) +}) diff --git a/packages/harness/test/session/wake-retry.test.ts b/packages/harness/test/session/wake-retry.test.ts index 3aa21404b..3b9f39e7c 100644 --- a/packages/harness/test/session/wake-retry.test.ts +++ b/packages/harness/test/session/wake-retry.test.ts @@ -148,15 +148,13 @@ describe("session wake retry", () => { const loop = spyOn(SessionInvoke, "loop").mockImplementation((async () => { attempts++ if (attempts === 1) { - // Mirrors the real loop ordering: steer items are drained (deleted) - // before materialization, so the InvalidUrlError from the real - // attachment capture surfaces after the item is already gone. - const steerItems = await SessionInbox.drainSteer(session.id) + // Failed materialization parks the original payload so later work can proceed. + const steerItems = await SessionInbox.peekSteer(session.id) expect(steerItems.length).toBe(1) for (const item of steerItems) await SessionInbox.materializeItem(item, root.info.id) return {} as never } - expect((await SessionInbox.drainSteer(session.id)).length).toBe(0) + expect((await SessionInbox.peekSteer(session.id)).length).toBe(0) const task = await SessionInbox.peekTask(session.id) expect(task).toBeDefined() await SessionInbox.materializeItem(task!) @@ -169,6 +167,7 @@ describe("session wake retry", () => { SessionManager.scheduleWake(session.id, "test") await waitFor(() => committed) expect(attempts).toBe(2) + expect((await SessionInbox.list(session.id)).some((item) => item.status === "failed")).toBe(true) expect(await SessionInbox.peekTask(session.id)).toBeUndefined() }, }) diff --git a/packages/harness/test/snapshot/lease.test.ts b/packages/harness/test/snapshot/lease.test.ts index 0d4e39da0..f95e7897c 100644 --- a/packages/harness/test/snapshot/lease.test.ts +++ b/packages/harness/test/snapshot/lease.test.ts @@ -22,7 +22,7 @@ test("maintenance excludes another process and recovers its abandoned lease", as await SnapshotLease.use(${JSON.stringify(key)}, true, async () => { process.stdout.write("ready\\n"); await Bun.sleep(60000); - }); + }, { dataRoot: ${JSON.stringify(Global.Path.data)} }); `, ], { env: process.env, stdout: "pipe", stderr: "pipe" }, diff --git a/packages/harness/test/storage/bootstrap.test.ts b/packages/harness/test/storage/bootstrap.test.ts new file mode 100644 index 000000000..6500cb5c2 --- /dev/null +++ b/packages/harness/test/storage/bootstrap.test.ts @@ -0,0 +1,136 @@ +import { AtomicFile } from "../../src/storage/atomic-file" +import { afterAll, expect, spyOn, test } from "bun:test" +import fs from "node:fs/promises" +import path from "node:path" +import { StorageBootstrap } from "../../src/storage/bootstrap" + +const fixtures = await fs.mkdtemp(path.join(process.env.SYNERGY_TEST_ROOT!, "storage-bootstrap-")) +afterAll(() => fs.rm(fixtures, { recursive: true, force: true })) + +async function home() { + const root = path.join(fixtures, crypto.randomUUID(), ".synergy") + await fs.mkdir(path.join(root, "data"), { recursive: true }) + return root +} + +test("keeps JSON intact until validation and activation and then uses only the database", async () => { + const root = await home() + const legacy = path.join(root, "data", "notes", "scope", "note.json") + await fs.mkdir(path.dirname(legacy), { recursive: true }) + await Bun.write(legacy, JSON.stringify({ text: "retained" })) + const prepared = await StorageBootstrap.prepare({ root }) + try { + expect(prepared.manifest.phase).toBe("validating") + expect(await Bun.file(legacy).exists()).toBe(true) + expect(await prepared.store.read<{ text: string }>(["notes", "scope", "note"])).toEqual({ text: "retained" }) + await prepared.activate() + expect(await Bun.file(legacy).exists()).toBe(false) + expect(prepared.manifest.phase).toBe("active") + } finally { + await prepared.store.close() + } + const reopened = await StorageBootstrap.prepare({ root }) + try { + expect(reopened.manifest.phase).toBe("active") + expect(await reopened.store.read<{ text: string }>(["notes", "scope", "note"])).toEqual({ text: "retained" }) + } finally { + await reopened.store.close() + } +}) + +test("does not recreate a missing active SQLite database", async () => { + const root = await home() + const prepared = await StorageBootstrap.prepare({ root }) + await prepared.activate() + const filename = prepared.store.options.backend === "sqlite" ? prepared.store.options.filename : "" + await prepared.store.close() + await fs.unlink(filename) + const failure = await StorageBootstrap.prepare({ root }).then( + () => undefined, + (error: unknown) => error, + ) + expect(failure).toMatchObject({ name: "StorageIntegrityError" }) + expect(await Bun.file(filename).exists()).toBe(false) +}) + +test("resumes prepared data after a domain migration failure without opening task admission", async () => { + const root = await home() + const prepared = await StorageBootstrap.prepare({ root }) + await prepared.store.write(["domain", "checkpoint"], { done: true }) + await prepared.store.close() + const resumed = await StorageBootstrap.prepare({ root }) + try { + expect(resumed.manifest.phase).toBe("validating") + expect(await resumed.store.read<{ done: boolean }>(["domain", "checkpoint"])).toEqual({ done: true }) + await resumed.activate() + } finally { + await resumed.store.close() + } +}) + +test("rejects legacy records recreated by an old writer after activation", async () => { + const root = await home() + const prepared = await StorageBootstrap.prepare({ root }) + await prepared.activate() + await prepared.store.close() + await fs.mkdir(path.join(root, "data", "projects"), { recursive: true }) + await Bun.write(path.join(root, "data", "projects", "unexpected.json"), "{}") + const failure = await StorageBootstrap.prepare({ root }).then( + () => undefined, + (error: unknown) => error, + ) + expect(failure).toMatchObject({ name: "StorageIntegrityError" }) +}) + +test("explicit target migration preserves data and switches configuration only after verification", async () => { + const root = await home() + const prepared = await StorageBootstrap.prepare({ root }) + await prepared.activate() + await prepared.store.write(["notes", "scope", "note"], { value: "move me" }) + await StorageBootstrap.migrateTarget({ + root, + store: prepared.store, + configuration: { backend: "sqlite", filename: "data/storage/relocated.sqlite" }, + }) + await prepared.store.close() + const moved = await StorageBootstrap.prepare({ root }) + try { + expect(await moved.store.read<{ value: string }>(["notes", "scope", "note"])).toEqual({ value: "move me" }) + expect(moved.store.options).toMatchObject({ + backend: "sqlite", + filename: path.join(root, "data/storage/relocated.sqlite"), + }) + } finally { + await moved.store.close() + } +}) + +test("resumes a target switch interrupted between configuration and manifest activation", async () => { + const root = await home() + const prepared = await StorageBootstrap.prepare({ root }) + await prepared.activate() + await prepared.store.write(["notes", "scope", "retained"], { value: 7 }) + const original = AtomicFile.writeJsonAtomic + { + using failure = spyOn(AtomicFile, "writeJsonAtomic").mockImplementation(async (filename, value, options) => { + if (filename.endsWith("manifest.json")) throw new Error("activation interrupted") + return original(filename, value, options) + }) + await expect( + StorageBootstrap.migrateTarget({ + root, + store: prepared.store, + configuration: { backend: "sqlite", filename: "data/storage/next.sqlite" }, + }), + ).rejects.toThrow("activation interrupted") + } + await prepared.store.close() + await expect(StorageBootstrap.prepare({ root })).rejects.toThrow("interrupted storage switch") + expect(await StorageBootstrap.resumeTargetSwitch(root)).toBe(true) + const resumed = await StorageBootstrap.prepare({ root }) + try { + expect(await resumed.store.read<{ value: number }>(["notes", "scope", "retained"])).toEqual({ value: 7 }) + } finally { + await resumed.store.close() + } +}) diff --git a/packages/harness/test/storage/context.test.ts b/packages/harness/test/storage/context.test.ts new file mode 100644 index 000000000..07fd7dfaa --- /dev/null +++ b/packages/harness/test/storage/context.test.ts @@ -0,0 +1,74 @@ +import { afterAll, expect, test } from "bun:test" +import fs from "node:fs/promises" +import path from "node:path" +import { Storage } from "../../src/storage/storage" +import { TransactionalStore } from "../../src/storage/transactional-store" + +const root = await fs.mkdtemp(path.join(process.env.SYNERGY_TEST_ROOT!, "storage-context-")) +const store = await TransactionalStore.open({ + backend: "sqlite", + namespace: "context", + filename: path.join(root, "store.sqlite"), +}) +afterAll(async () => { + await store.close() + await fs.rm(root, { recursive: true, force: true }) +}) + +test("nested domain writes join the caller's transaction and publish only after commit", async () => { + const published: number[] = [] + await Storage.provide({ store, artifactDirectory: root }, async () => { + await expect( + Storage.transaction(async () => { + await Storage.write(["left"], { value: 1 }) + await Storage.transaction(async () => { + await Storage.write(["right"], { value: 2 }) + Storage.afterCommit(() => { + published.push(1) + }) + }) + expect(published).toEqual([]) + throw new Error("rollback") + }), + ).rejects.toThrow("rollback") + expect(await Storage.readMany([["left"], ["right"]])).toEqual([undefined, undefined]) + expect(published).toEqual([]) + await Storage.transaction(async () => { + await Storage.write(["left"], { value: 3 }) + Storage.afterCommit(async () => { + published.push((await Storage.read<{ value: number }>(["left"])).value) + }) + }) + expect(published).toEqual([3]) + }) +}) + +test("snapshots read their own stable view and reject transitive writes", async () => { + await Storage.provide({ store, artifactDirectory: root }, async () => { + await Storage.write(["snapshot"], 1) + await expect( + Storage.snapshot(async () => { + expect(await Storage.read(["snapshot"])).toBe(1) + await Storage.write(["snapshot"], 2) + }), + ).rejects.toThrow("read-only") + expect(await Storage.read(["snapshot"])).toBe(1) + }) +}) + +test("notification failure does not turn a confirmed commit into a retryable write failure", async () => { + await Storage.provide({ store, artifactDirectory: root }, async () => { + let lastObserver = false + await Storage.transaction(async () => { + await Storage.write(["confirmed"], 42) + Storage.afterCommit(() => { + throw new Error("observer unavailable") + }) + Storage.afterCommit(() => { + lastObserver = true + }) + }) + expect(await Storage.read(["confirmed"])).toBe(42) + expect(lastObserver).toBe(true) + }) +}) diff --git a/packages/harness/test/storage/crash-recovery.test.ts b/packages/harness/test/storage/crash-recovery.test.ts new file mode 100644 index 000000000..081c9e255 --- /dev/null +++ b/packages/harness/test/storage/crash-recovery.test.ts @@ -0,0 +1,95 @@ +import { expect, test } from "bun:test" +import fs from "node:fs/promises" +import path from "node:path" +import { TransactionalStore } from "../../src/storage/transactional-store" + +const entry = new URL("../../src/storage/transactional-store.ts", import.meta.url).href + +for (const stage of ["inside", "committed"] as const) { + test(`process loss ${stage} a transaction preserves its atomic boundary`, async () => { + const root = await fs.mkdtemp(path.join(process.env.SYNERGY_TEST_ROOT!, "storage-crash-")) + const filename = path.join(root, "agent.sqlite") + const ready = Promise.withResolvers() + const child = Bun.spawn({ + cmd: [ + process.execPath, + "--eval", + ` + const { TransactionalStore } = await import(process.env.TEST_STORAGE_MODULE) + const store = await TransactionalStore.open({ backend: "sqlite", filename: process.env.TEST_STORAGE_FILE, namespace: "crash" }) + await store.transaction(async (tx) => { + await tx.write(["session"], { title: "committed" }) + if (process.env.TEST_STORAGE_STAGE === "inside") { + process.send({ ready: true }) + await new Promise(() => {}) + } + await tx.write(["index"], { session: "session" }) + return { accepted: true } + }, { operationID: "command", requestHash: "input" }) + process.send({ ready: true }) + await new Promise(() => {}) + `, + ], + env: { ...process.env, TEST_STORAGE_MODULE: entry, TEST_STORAGE_FILE: filename, TEST_STORAGE_STAGE: stage }, + stdout: "ignore", + stderr: "pipe", + ipc(message: unknown) { + if (message && typeof message === "object" && "ready" in message) ready.resolve() + }, + onExit(_child, code) { + ready.reject(new Error(`Child exited before the crash boundary: ${code}`)) + }, + }) + const stderr = new Response(child.stderr).text() + const deadline = setTimeout(() => ready.reject(new Error("Storage crash fixture did not become ready")), 10_000) + try { + await ready.promise + child.kill("SIGKILL") + await child.exited + await stderr + let reopened: TransactionalStore | undefined + for (let attempt = 0; attempt < 100; attempt++) { + try { + reopened = await TransactionalStore.open({ + backend: "sqlite", + filename, + namespace: "crash", + recover: true, + mustExist: true, + }) + break + } catch (error) { + if (attempt === 99) throw error + await Bun.sleep(20) + } + } + if (!reopened) throw new Error("Could not reopen the interrupted dataset") + try { + if (stage === "inside") { + expect(await reopened.readMany([["session"], ["index"]])).toEqual([undefined, undefined]) + expect(await reopened.operationReceipt("command")).toBeUndefined() + } else { + expect(await reopened.readMany([["session"], ["index"]])).toEqual([ + { title: "committed" }, + { session: "session" }, + ]) + expect( + await reopened.transaction<{ accepted: boolean }>( + async () => { + throw new Error("must not replay committed work") + }, + { operationID: "command", requestHash: "input" }, + ), + ).toEqual({ accepted: true }) + } + } finally { + await reopened.close() + } + } finally { + clearTimeout(deadline) + child.kill() + await child.exited + await fs.rm(root, { recursive: true, force: true }) + } + }, 15_000) +} diff --git a/packages/harness/test/storage/fixtures/README.md b/packages/harness/test/storage/fixtures/README.md new file mode 100644 index 000000000..a85f816e2 --- /dev/null +++ b/packages/harness/test/storage/fixtures/README.md @@ -0,0 +1,5 @@ +# Released JSON upgrade fixtures + +Each fixture records a published tag, exact commit and the storage/session/message writer schemas used to reconstruct that release's minimal on-disk records. These are schema-derived synthetic fixtures, not copies of private user data. `futureOwner` and the unknown migration owner are deliberate preservation probes added to each fixture. + +`released-upgrade.test.ts` starts the real maintenance bootstrap in a separate process and isolated Home. It checks activation, retired JSON removal, migrated message semantics, rebuilt indexes, unknown-field preservation and database relationship verification. Importer tests separately exercise malformed bytes, global corruption, interrupted checkpoints, source drift, read-only files and symbolic links. diff --git a/packages/harness/test/storage/fixtures/v1.2.33.json b/packages/harness/test/storage/fixtures/v1.2.33.json new file mode 100644 index 000000000..461cf7398 --- /dev/null +++ b/packages/harness/test/storage/fixtures/v1.2.33.json @@ -0,0 +1,90 @@ +{ + "tag": "v1.2.33", + "commit": "bb6302e10b6c64aff383d597c4bcbe030cd65a4e", + "sources": [ + "packages/synergy/src/storage/path.ts", + "packages/synergy/src/session/types.ts", + "packages/synergy/src/session/message-v2.ts" + ], + "records": [ + { + "key": ["sessions", "home", "ses_00000000000000000000000001", "info"], + "value": { + "id": "ses_00000000000000000000000001", + "scope": { + "id": "home", + "type": "home" + }, + "title": "Historical conversation", + "version": "1.2.33", + "time": { + "created": 1700000000000, + "updated": 1700000000000 + }, + "futureOwner": { + "retained": true + }, + "allowAll": true, + "pendingReply": true + } + }, + { + "key": ["session_index", "ses_00000000000000000000000001"], + "value": { + "id": "ses_00000000000000000000000001", + "scopeID": "home" + } + }, + { + "key": [ + "sessions", + "home", + "ses_00000000000000000000000001", + "messages", + "msg_00000000000000000000000001", + "info" + ], + "value": { + "id": "msg_00000000000000000000000001", + "sessionID": "ses_00000000000000000000000001", + "role": "user", + "time": { + "created": 1700000000000 + }, + "agent": "synergy", + "model": { + "providerID": "openai", + "modelID": "gpt-4.1" + }, + "metadata": { + "fixture": "released-writer" + } + } + }, + { + "key": [ + "sessions", + "home", + "ses_00000000000000000000000001", + "messages", + "msg_00000000000000000000000001", + "parts", + "prt_00000000000000000000000001" + ], + "value": { + "id": "prt_00000000000000000000000001", + "sessionID": "ses_00000000000000000000000001", + "messageID": "msg_00000000000000000000000001", + "type": "text", + "text": "Keep the original transcript.", + "synthetic": false + } + }, + { + "key": ["meta", "migration", "log"], + "value": { + "unloaded-fixture-owner": 1700000000000 + } + } + ] +} diff --git a/packages/harness/test/storage/fixtures/v2.4.4.json b/packages/harness/test/storage/fixtures/v2.4.4.json new file mode 100644 index 000000000..cd3a0aa78 --- /dev/null +++ b/packages/harness/test/storage/fixtures/v2.4.4.json @@ -0,0 +1,99 @@ +{ + "tag": "v2.4.4", + "commit": "32a25fac05ab0b3362ab90b809888b0aad4391e2", + "sources": [ + "packages/synergy/src/storage/path.ts", + "packages/synergy/src/session/types.ts", + "packages/synergy/src/session/message-v2.ts" + ], + "records": [ + { + "key": ["sessions", "home", "ses_00000000000000000000000001", "info"], + "value": { + "id": "ses_00000000000000000000000001", + "scope": { + "id": "home", + "type": "home" + }, + "title": "Historical conversation", + "version": "2.4.4", + "time": { + "created": 1700000000000, + "updated": 1700000000000 + }, + "futureOwner": { + "retained": true + }, + "controlProfile": "guarded", + "completionNotice": { + "unread": true, + "silent": false, + "unreadCount": 1 + } + } + }, + { + "key": ["session_index", "ses_00000000000000000000000001"], + "value": { + "id": "ses_00000000000000000000000001", + "scopeID": "home" + } + }, + { + "key": [ + "sessions", + "home", + "ses_00000000000000000000000001", + "messages", + "msg_00000000000000000000000001", + "info" + ], + "value": { + "id": "msg_00000000000000000000000001", + "sessionID": "ses_00000000000000000000000001", + "role": "user", + "time": { + "created": 1700000000000 + }, + "agent": "synergy", + "model": { + "providerID": "openai", + "modelID": "gpt-4.1" + }, + "metadata": { + "fixture": "released-writer" + }, + "isRoot": true, + "visible": true, + "includeInContext": true, + "origin": { + "type": "user" + } + } + }, + { + "key": [ + "sessions", + "home", + "ses_00000000000000000000000001", + "messages", + "msg_00000000000000000000000001", + "parts", + "prt_00000000000000000000000001" + ], + "value": { + "id": "prt_00000000000000000000000001", + "sessionID": "ses_00000000000000000000000001", + "messageID": "msg_00000000000000000000000001", + "type": "text", + "text": "Keep the original transcript." + } + }, + { + "key": ["meta", "migration", "log"], + "value": { + "unloaded-fixture-owner": 1700000000000 + } + } + ] +} diff --git a/packages/harness/test/storage/fixtures/v3.0.22.json b/packages/harness/test/storage/fixtures/v3.0.22.json new file mode 100644 index 000000000..8023d12ec --- /dev/null +++ b/packages/harness/test/storage/fixtures/v3.0.22.json @@ -0,0 +1,103 @@ +{ + "tag": "v3.0.22", + "commit": "024dd683e091d9fce3d1d26b79b2e188ce636b52", + "sources": [ + "packages/synergy/src/storage/path.ts", + "packages/synergy/src/session/types.ts", + "packages/synergy/src/session/message-v2.ts" + ], + "records": [ + { + "key": ["sessions", "home", "ses_00000000000000000000000001", "info"], + "value": { + "id": "ses_00000000000000000000000001", + "scope": { + "id": "home", + "type": "home" + }, + "title": "Historical conversation", + "version": "3.0.22", + "time": { + "created": 1700000000000, + "updated": 1700000000000 + }, + "futureOwner": { + "retained": true + }, + "controlProfile": "guarded", + "completionNotice": { + "unread": true, + "silent": false, + "unreadCount": 1 + }, + "workflow": { + "kind": "plan" + }, + "category": "home" + } + }, + { + "key": ["session_index", "ses_00000000000000000000000001"], + "value": { + "id": "ses_00000000000000000000000001", + "scopeID": "home" + } + }, + { + "key": [ + "sessions", + "home", + "ses_00000000000000000000000001", + "messages", + "msg_00000000000000000000000001", + "info" + ], + "value": { + "id": "msg_00000000000000000000000001", + "sessionID": "ses_00000000000000000000000001", + "role": "user", + "time": { + "created": 1700000000000 + }, + "agent": "synergy", + "model": { + "providerID": "openai", + "modelID": "gpt-4.1" + }, + "metadata": { + "fixture": "released-writer" + }, + "isRoot": true, + "visible": true, + "includeInContext": true, + "origin": { + "type": "user" + } + } + }, + { + "key": [ + "sessions", + "home", + "ses_00000000000000000000000001", + "messages", + "msg_00000000000000000000000001", + "parts", + "prt_00000000000000000000000001" + ], + "value": { + "id": "prt_00000000000000000000000001", + "sessionID": "ses_00000000000000000000000001", + "messageID": "msg_00000000000000000000000001", + "type": "text", + "text": "Keep the original transcript." + } + }, + { + "key": ["meta", "migration", "log"], + "value": { + "unloaded-fixture-owner": 1700000000000 + } + } + ] +} diff --git a/packages/harness/test/storage/legacy-import.test.ts b/packages/harness/test/storage/legacy-import.test.ts new file mode 100644 index 000000000..edb94aa5e --- /dev/null +++ b/packages/harness/test/storage/legacy-import.test.ts @@ -0,0 +1,160 @@ +import { afterAll, expect, test } from "bun:test" +import fs from "node:fs/promises" +import path from "node:path" +import { LegacyJsonImporter } from "../../src/storage/legacy-import" +import { TransactionalStore } from "../../src/storage/transactional-store" + +const root = await fs.mkdtemp(path.join(process.env.SYNERGY_TEST_ROOT!, "legacy-import-")) +afterAll(() => fs.rm(root, { recursive: true, force: true })) + +async function fixture() { + const directory = path.join(root, crypto.randomUUID()) + const data = path.join(directory, "data") + await fs.mkdir(data, { recursive: true }) + const store = await TransactionalStore.open({ + backend: "sqlite", + filename: path.join(directory, "target.sqlite"), + namespace: crypto.randomUUID(), + }) + return { directory, data, store, backup: path.join(directory, "backup") } +} + +async function json(root: string, key: string[], value: unknown) { + const target = path.join(root, ...key) + ".json" + await fs.mkdir(path.dirname(target), { recursive: true }) + await Bun.write(target, JSON.stringify(value)) + return target +} + +test("backs up and imports historical records without dropping unloaded fields or migration ledgers", async () => { + const fixtureData = await fixture() + const { store, data, backup } = fixtureData + try { + const session = { id: "session", projectID: "scope", title: "old", unknownOwner: { value: 2 } } + await json(data, ["sessions", "scope", "session", "info"], session) + await json(data, ["meta", "migration", "log-workflows"], { historical: 123 }) + await json(data, ["auth", "provider-auth"], { private: "must-not-become-a-record" }) + const importer = new LegacyJsonImporter({ dataRoot: data, backupRoot: backup, store }) + const result = await importer.run() + expect(result.imported).toBe(2) + expect(result.quarantined).toBe(0) + expect(await store.read>(["sessions", "scope", "session", "info"])).toEqual(session) + expect(await store.read>(["meta", "migration", "log-workflows"])).toEqual({ + historical: 123, + }) + expect(await store.readMany([["auth", "provider-auth"]])).toEqual([undefined]) + expect(await Bun.file(path.join(backup, "data", "sessions", "scope", "session", "info.json")).json()).toEqual( + session, + ) + expect(await Bun.file(path.join(data, "sessions", "scope", "session", "info.json")).exists()).toBe(true) + expect(await importer.run()).toEqual(result) + } finally { + await store.close() + } +}) + +test("quarantines malformed evidence with original bytes and blocks the affected session", async () => { + const { store, data, backup } = await fixture() + try { + const target = await json(data, ["sessions", "scope", "broken", "messages", "message", "info"], {}) + await Bun.write(target, "{broken-json") + await json(data, ["sessions", "scope", "good", "info"], { id: "good" }) + const result = await new LegacyJsonImporter({ dataRoot: data, backupRoot: backup, store }).run() + expect(result.quarantined).toBe(1) + expect(await Bun.file(path.join(backup, "data", path.relative(data, target))).text()).toBe("{broken-json") + expect(await store.read>(["storage_recovery", "sessions", "broken", "info"])).toMatchObject( + { blocked: true }, + ) + expect(await store.read>(["sessions", "scope", "good", "info"])).toEqual({ id: "good" }) + } finally { + await store.close() + } +}) + +test("resumes after interruption without overwriting committed imported data", async () => { + const { store, data, backup } = await fixture() + try { + for (let index = 0; index < 5; index++) await json(data, ["notes", "scope", `note-${index}`], { index }) + let interrupted = false + const importer = new LegacyJsonImporter({ + dataRoot: data, + backupRoot: backup, + store, + progress: (progress) => { + if (progress.stage === "import" && progress.current === 2) { + interrupted = true + throw new Error("interrupted") + } + }, + }) + const failure = await importer.run().then( + () => undefined, + (error: unknown) => error, + ) + expect(failure).toMatchObject({ message: "interrupted" }) + expect(interrupted).toBe(true) + const result = await new LegacyJsonImporter({ dataRoot: data, backupRoot: backup, store }).run() + expect(result.imported).toBe(5) + expect(await store.list(["notes", "scope"])).toHaveLength(5) + } finally { + await store.close() + } +}) + +test("refuses source changes after backup instead of mixing historical snapshots", async () => { + const { store, data, backup } = await fixture() + try { + const target = await json(data, ["notes", "scope", "note"], { value: 1 }) + const importer = new LegacyJsonImporter({ dataRoot: data, backupRoot: backup, store }) + await importer.run() + await Bun.write(target, JSON.stringify({ value: 2 })) + const failure = await importer.run().then( + () => undefined, + (error: unknown) => error, + ) + expect(failure).toMatchObject({ name: "StorageIntegrityError" }) + expect(await store.read>(["notes", "scope", "note"])).toEqual({ value: 1 }) + } finally { + await store.close() + } +}) + +test("backup retains workspace symlinks, read-only evidence and global configuration", async () => { + const { store, data, backup, directory } = await fixture() + try { + await fs.mkdir(path.join(data, "worktree"), { recursive: true }) + await fs.symlink("unmounted-volume", path.join(data, "worktree", "linked")) + await Bun.write(path.join(data, "artifact"), "read-only evidence") + await fs.chmod(path.join(data, "artifact"), 0o444) + await Bun.write(path.join(directory, "config", "synergy.d", "10-models.jsonc"), '{"model":"old-model"}') + const result = await new LegacyJsonImporter({ dataRoot: data, backupRoot: backup, store }).run() + expect(result.retained).toBe(3) + expect(await fs.readlink(path.join(backup, "data", "worktree", "linked"))).toBe("unmounted-volume") + expect(await Bun.file(path.join(backup, "data", "artifact")).text()).toBe("read-only evidence") + expect( + await Bun.file(path.join(backup, "data", "@home", "config", "synergy.d", "10-models.jsonc")).text(), + ).toContain("old-model") + expect(await Bun.file(path.join(backup, "inventory.ndjson")).text()).toContain('"linkTarget":"unmounted-volume"') + } finally { + await store.close() + } +}) + +for (const key of [ + ["projects", "broken"], + ["meta", "migration", "log-session"], +]) { + test(`corrupt global authority blocks activation and repeated resume: ${key.join("/")}`, async () => { + const { store, data, backup } = await fixture() + try { + const target = await json(data, key, {}) + await Bun.write(target, "{broken") + for (let attempt = 0; attempt < 2; attempt++) + await expect(new LegacyJsonImporter({ dataRoot: data, backupRoot: backup, store }).run()).rejects.toThrow() + expect(await Bun.file(path.join(backup, "data", path.relative(data, target))).text()).toBe("{broken") + expect(await Bun.file(target).text()).toBe("{broken") + } finally { + await store.close() + } + }) +} diff --git a/packages/harness/test/storage/portable.test.ts b/packages/harness/test/storage/portable.test.ts new file mode 100644 index 000000000..e336e6361 --- /dev/null +++ b/packages/harness/test/storage/portable.test.ts @@ -0,0 +1,63 @@ +import { expect, test } from "bun:test" +import fs from "node:fs/promises" +import path from "node:path" +import { TransactionalStore } from "../../src/storage/transactional-store" +import { StoragePortable } from "../../src/storage/portable" + +async function fixture() { + const root = await fs.mkdtemp(path.join(process.env.SYNERGY_TEST_ROOT!, "portable-")) + const source = await TransactionalStore.open({ + backend: "sqlite", + namespace: "source", + filename: path.join(root, "source.sqlite"), + }) + const target = await TransactionalStore.open({ + backend: "sqlite", + namespace: "target", + filename: path.join(root, "target.sqlite"), + }) + return { + root, + source, + target, + async [Symbol.asyncDispose]() { + await source.close() + await target.close() + await fs.rm(root, { recursive: true, force: true }) + }, + } +} + +test("portable data preserves revisions, unknown fields, and committed command receipts", async () => { + await using data = await fixture() + await data.source.transaction( + async (tx) => { + await tx.write(["record"], { future: { field: 3 } }) + return { accepted: true } + }, + { operationID: "command", requestHash: "same-input" }, + ) + const filename = path.join(data.root, "records.ndjson") + await StoragePortable.exportFile(data.source, filename) + await StoragePortable.importFile(data.target, filename) + expect(await data.target.versioned(["record"])).toEqual(await data.source.versioned(["record"])) + expect( + await data.target.transaction<{ accepted: boolean }>( + async () => { + throw new Error("must not replay") + }, + { operationID: "command", requestHash: "same-input" }, + ), + ).toEqual({ accepted: true }) +}) + +test("a damaged portable archive never exposes a partial imported aggregate", async () => { + await using data = await fixture() + await data.source.write(["sessions", "scope", "one", "info"], { id: "one" }) + const filename = path.join(data.root, "records.ndjson") + await StoragePortable.exportFile(data.source, filename) + const text = await Bun.file(filename).text() + await Bun.write(filename, text.replace('"id":"one"', '"id":"two"')) + await expect(StoragePortable.importFile(data.target, filename)).rejects.toThrow("checksum") + expect(await data.target.list([])).toEqual([]) +}) diff --git a/packages/harness/test/storage/postgres-ownership.test.ts b/packages/harness/test/storage/postgres-ownership.test.ts new file mode 100644 index 000000000..b930cb6bc --- /dev/null +++ b/packages/harness/test/storage/postgres-ownership.test.ts @@ -0,0 +1,42 @@ +import { expect, test } from "bun:test" +import { SQL } from "bun" +import { createHash } from "node:crypto" +import { TransactionalStore } from "../../src/storage/transactional-store" + +const url = process.env.SYNERGY_TEST_POSTGRES_URL + +test.skipIf(!url)( + "PostgreSQL ownership loss rolls back work and requires explicit takeover", + async () => { + const namespace = crypto.randomUUID() + const store = await TransactionalStore.open({ backend: "postgres", namespace, url: url! }) + const admin = new SQL(url!) + let replacement: TransactionalStore | undefined + try { + await store.write(["value"], { count: 1 }) + const digest = createHash("sha256").update(namespace).digest() + const [owner] = + await admin`SELECT pid FROM pg_locks WHERE locktype = 'advisory' AND classid::bigint = ${digest.readUInt32BE(0)} AND objid::bigint = ${digest.readUInt32BE(4)} AND objsubid = 2 AND granted` + expect(owner).toBeDefined() + await expect( + store.transaction(async (tx) => { + await tx.write(["value"], { count: 2 }) + await admin`SELECT pg_terminate_backend(${owner.pid})` + }), + ).rejects.toThrow() + expect(await store.read<{ count: number }>(["value"])).toEqual({ count: 1 }) + await expect(TransactionalStore.open({ backend: "postgres", namespace, url: url! })).rejects.toThrow( + "did not release ownership", + ) + replacement = await TransactionalStore.open({ backend: "postgres", namespace, url: url!, recover: true }) + await replacement.write(["value"], { count: 3 }) + await expect(store.write(["value"], { count: 4 })).rejects.toThrow() + expect(await replacement.read<{ count: number }>(["value"])).toEqual({ count: 3 }) + } finally { + await store.close() + await replacement?.close() + await admin.close() + } + }, + 30000, +) diff --git a/packages/harness/test/storage/released-upgrade.test.ts b/packages/harness/test/storage/released-upgrade.test.ts new file mode 100644 index 000000000..1ba0cb470 --- /dev/null +++ b/packages/harness/test/storage/released-upgrade.test.ts @@ -0,0 +1,47 @@ +import { expect, test } from "bun:test" +import fs from "node:fs/promises" +import path from "node:path" +import { StorageBootstrap } from "../../src/storage/bootstrap" +import { tmpdir } from "../support/fixture" + +for (const version of ["1.2.33", "2.4.4", "3.0.22"]) { + test(`upgrades the v${version} JSON writer shape through the actual startup migration runner`, async () => { + await using tmp = await tmpdir() + const root = path.join(tmp.path, ".synergy") + const fixture = (await Bun.file(new URL(`./fixtures/v${version}.json`, import.meta.url)).json()) as { + records: Array<{ key: string[]; value: unknown }> + } + for (const record of fixture.records) { + const file = path.join(root, "data", ...record.key) + ".json" + await fs.mkdir(path.dirname(file), { recursive: true }) + await Bun.write(file, JSON.stringify(record.value)) + } + const entry = new URL("../../src/storage/maintenance.ts", import.meta.url).pathname + const script = `import { StorageMaintenance } from ${JSON.stringify(entry)}; await using handle = await StorageMaintenance.open(); if (handle.manifest.phase !== "active") throw new Error("Upgrade did not activate");` + const child = Bun.spawn([process.execPath, "-e", script], { + env: { ...process.env, SYNERGY_HOME: tmp.path }, + stdout: "pipe", + stderr: "pipe", + }) + const [stderr, , code] = await Promise.all([ + new Response(child.stderr).text(), + new Response(child.stdout).text(), + child.exited, + ]) + expect(code, stderr).toBe(0) + const handle = await StorageBootstrap.inspect(root) + if (!handle) throw new Error("Upgraded dataset is absent") + try { + const session = await handle.store.read<{ id: string; futureOwner: { retained: boolean } }>( + fixture.records[0].key, + ) + expect(session.futureOwner).toEqual({ retained: true }) + expect(await handle.store.read(fixture.records[3].key)).toMatchObject({ text: "Keep the original transcript." }) + expect(await handle.store.read(["session_index", session.id])).toMatchObject({ scopeID: "home" }) + expect((await handle.store.verify()).issues).toEqual([]) + expect(await Bun.file(path.join(root, "data", ...fixture.records[0].key) + ".json").exists()).toBe(false) + } finally { + await handle.store.close() + } + }, 30000) +} diff --git a/packages/harness/test/storage/storage-retry.test.ts b/packages/harness/test/storage/storage-retry.test.ts index fca748f4e..7f1b8db50 100644 --- a/packages/harness/test/storage/storage-retry.test.ts +++ b/packages/harness/test/storage/storage-retry.test.ts @@ -2,7 +2,12 @@ import { describe, expect, spyOn, test } from "bun:test" import fs from "fs/promises" import path from "path" import { Global } from "../../src/global" -import { Storage } from "../../src/storage/storage" +import { AtomicFile } from "../../src/storage/atomic-file" +const FileRecords = { + write: (key: string[], value: unknown, options?: AtomicFile.WriteOptions) => + AtomicFile.writeJsonAtomic(path.join(Global.Path.data, ...key) + ".json", JSON.stringify(value), options), + read: (key: string[]) => Bun.file(path.join(Global.Path.data, ...key) + ".json").json() as Promise, +} function keyRoot() { return ["storage-retry-test", Math.random().toString(36).slice(2)] @@ -22,14 +27,14 @@ describe("Storage atomic write transient-failure retry", () => { test("durable write failure leaves the previous record intact", async () => { const root = keyRoot() const key = [...root, "durable"] - await Storage.write(key, { phase: "old" }) + await FileRecords.write(key, { phase: "old" }) const realOpen = fs.open.bind(fs) using _open = spyOn(fs, "open").mockImplementation((async (file, ...args) => { if (String(file).includes(".tmp-")) throw errnoError("ENOSPC") return realOpen(file, ...args) }) as typeof fs.open) - await expect(Storage.write(key, { phase: "new" }, { durable: true })).rejects.toMatchObject({ code: "ENOSPC" }) - expect(await Storage.read<{ phase: string }>(key)).toEqual({ phase: "old" }) + await expect(FileRecords.write(key, { phase: "new" }, { durable: true })).rejects.toMatchObject({ code: "ENOSPC" }) + expect(await FileRecords.read<{ phase: string }>(key)).toEqual({ phase: "old" }) expect(await tempFiles(root)).toEqual([]) }) test("retries a transient EPERM on rename and persists the payload", async () => { @@ -43,10 +48,10 @@ describe("Storage atomic write transient-failure retry", () => { }) as unknown as typeof fs.rename using _rename = spyOn(fs, "rename").mockImplementation(impl) - await Storage.write([...root, "item"], { value: 1 }) + await FileRecords.write([...root, "item"], { value: 1 }) expect(calls).toBe(3) - expect(await Storage.read<{ value: number }>([...root, "item"])).toEqual({ value: 1 }) + expect(await FileRecords.read<{ value: number }>([...root, "item"])).toEqual({ value: 1 }) expect(await tempFiles(root)).toEqual([]) }) @@ -61,10 +66,10 @@ describe("Storage atomic write transient-failure retry", () => { }) as unknown as typeof Bun.write using _write = spyOn(Bun, "write").mockImplementation(impl) - await Storage.write([...root, "item"], { value: 2 }) + await FileRecords.write([...root, "item"], { value: 2 }) expect(calls).toBe(2) - expect(await Storage.read<{ value: number }>([...root, "item"])).toEqual({ value: 2 }) + expect(await FileRecords.read<{ value: number }>([...root, "item"])).toEqual({ value: 2 }) expect(await tempFiles(root)).toEqual([]) }) @@ -77,7 +82,7 @@ describe("Storage atomic write transient-failure retry", () => { }) as unknown as typeof fs.rename using _rename = spyOn(fs, "rename").mockImplementation(impl) - await expect(Storage.write([...root, "item"], { value: 3 })).rejects.toMatchObject({ code: "EPERM" }) + await expect(FileRecords.write([...root, "item"], { value: 3 })).rejects.toMatchObject({ code: "EPERM" }) expect(calls).toBe(4) expect(await tempFiles(root)).toEqual([]) }) @@ -91,7 +96,7 @@ describe("Storage atomic write transient-failure retry", () => { }) as unknown as typeof fs.rename using _rename = spyOn(fs, "rename").mockImplementation(impl) - await expect(Storage.write([...root, "item"], { value: 4 })).rejects.toMatchObject({ code: "ENOSPC" }) + await expect(FileRecords.write([...root, "item"], { value: 4 })).rejects.toMatchObject({ code: "ENOSPC" }) expect(calls).toBe(1) expect(await tempFiles(root)).toEqual([]) }) @@ -113,7 +118,7 @@ describe("Storage atomic write transient-failure retry", () => { }) as unknown as typeof fs.unlink using _unlink = spyOn(fs, "unlink").mockImplementation(unlinkImpl) - await expect(Storage.write([...root, "item"], { value: 5 })).rejects.toMatchObject({ code: "EPERM" }) + await expect(FileRecords.write([...root, "item"], { value: 5 })).rejects.toMatchObject({ code: "EPERM" }) expect(renameCalls).toBe(4) expect(unlinkCalls).toBe(2) expect(await tempFiles(root)).toEqual([]) diff --git a/packages/harness/test/storage/storage.test.ts b/packages/harness/test/storage/storage.test.ts index bc2c0383d..7730bca7f 100644 --- a/packages/harness/test/storage/storage.test.ts +++ b/packages/harness/test/storage/storage.test.ts @@ -26,29 +26,15 @@ describe("Storage", () => { expect(await Storage.read<{ value: number }>([...root, "item"])).toEqual({ value: 2 }) }) - test("compact writes omit pretty-print indentation but read back identically", async () => { + test("keeps nested data in the database without creating a JSON mirror", async () => { const root = keyRoot() const content = { value: 1, nested: { a: [1, 2, 3] } } - - await Storage.write([...root, "pretty"], content) - await Storage.write([...root, "compact"], content, { compact: true }) - - const dir = path.join(Global.Path.data, ...root) - const prettyRaw = await fs.readFile(path.join(dir, "pretty.json"), "utf8") - const compactRaw = await fs.readFile(path.join(dir, "compact.json"), "utf8") - - expect(prettyRaw).toContain("\n") - expect(compactRaw).not.toContain("\n") - expect(compactRaw.length).toBeLessThan(prettyRaw.length) - - // Both forms parse to the same value. - expect(await Storage.read([...root, "pretty"])).toEqual(content) - expect(await Storage.read([...root, "compact"])).toEqual(content) - - // update honors the compact option too. - await Storage.update([...root, "compact"], (draft) => (draft.value = 2), { compact: true }) - const updatedRaw = await fs.readFile(path.join(dir, "compact.json"), "utf8") - expect(updatedRaw).not.toContain("\n") - expect(await Storage.read([...root, "compact"])).toEqual({ value: 2, nested: { a: [1, 2, 3] } }) + await Storage.write([...root, "record"], content) + expect(await Storage.read([...root, "record"])).toEqual(content) + expect(await Bun.file(path.join(Global.Path.data, ...root, "record.json")).exists()).toBe(false) + await Storage.update([...root, "record"], (draft) => { + draft.value = 2 + }) + expect(await Storage.read([...root, "record"])).toEqual({ ...content, value: 2 }) }) }) diff --git a/packages/harness/test/storage/transactional-store.test.ts b/packages/harness/test/storage/transactional-store.test.ts new file mode 100644 index 000000000..f8dd46cf6 --- /dev/null +++ b/packages/harness/test/storage/transactional-store.test.ts @@ -0,0 +1,163 @@ +import { afterAll, describe, expect, test } from "bun:test" +import fs from "node:fs/promises" +import path from "node:path" +import { TransactionalStore } from "../../src/storage/transactional-store" + +if (process.env.SYNERGY_REQUIRE_POSTGRES_TESTS === "1" && !process.env.SYNERGY_TEST_POSTGRES_URL) + throw new Error("PostgreSQL contract tests require a real database") + +const root = await fs.mkdtemp(path.join(process.env.SYNERGY_TEST_ROOT!, "transactional-store-")) +const stores: TransactionalStore[] = [] +async function failure(task: Promise) { + return task.then( + () => { + throw new Error("Expected storage operation to fail") + }, + (error: unknown) => error, + ) +} +afterAll(async () => { + await Promise.all(stores.map((store) => store.close())) + await fs.rm(root, { recursive: true, force: true }) +}) + +for (const backend of ["sqlite", ...(process.env.SYNERGY_TEST_POSTGRES_URL ? ["postgres"] : [])] as const) { + describe(`${backend} transactional records`, () => { + async function open() { + const namespace = crypto.randomUUID() + const store = await TransactionalStore.open( + backend === "sqlite" + ? { backend, namespace, filename: path.join(root, `${namespace}.sqlite`) } + : { backend: "postgres", namespace, url: process.env.SYNERGY_TEST_POSTGRES_URL! }, + ) + stores.push(store) + return store + } + + test("commits related records together and rolls back every mutation on failure", async () => { + const store = await open() + await store.transaction(async (tx) => { + await tx.write(["sessions", "scope", "session", "info"], { title: "before", optional: { future: true } }) + await tx.write(["session_index", "session"], { scopeID: "scope" }) + }) + expect( + await failure( + store.transaction(async (tx) => { + await tx.write(["sessions", "scope", "session", "info"], { title: "after" }) + await tx.remove(["session_index", "session"]) + throw new Error("abort operation") + }), + ), + ).toMatchObject({ message: "abort operation" }) + expect(await store.read>(["sessions", "scope", "session", "info"])).toEqual({ + title: "before", + optional: { future: true }, + }) + expect(await store.read>(["session_index", "session"])).toEqual({ scopeID: "scope" }) + }) + + test("propagates asynchronous transaction rejection", async () => { + const store = await open() + await expect( + store.transaction(async (tx) => { + await tx.write(["rollback"], { value: 1 }) + throw new Error("transaction aborted") + }), + ).rejects.toThrow("transaction aborted") + }) + + test("serializes concurrent read-modify-write and preserves unknown fields", async () => { + const store = await open() + await store.write(["counter"], { count: 0, unknownOwner: { future: "retained" } }) + await Promise.all( + Array.from({ length: 24 }, () => + store.transaction(async (tx) => { + await tx.update<{ count: number }>(["counter"], (value) => { + value.count++ + }) + }), + ), + ) + expect(await store.read>(["counter"])).toEqual({ + count: 24, + unknownOwner: { future: "retained" }, + }) + }) + + test("replays a committed operation receipt without running the command again", async () => { + const store = await open() + let calls = 0 + const execute = () => + store.transaction( + async (tx) => { + calls++ + await tx.write(["input"], { id: "message" }) + return { id: "message" } + }, + { operationID: "delivery", requestHash: "same-input" }, + ) + expect(await execute()).toEqual({ id: "message" }) + expect(await execute()).toEqual({ id: "message" }) + expect(calls).toBe(1) + expect( + await failure(store.transaction(async () => "wrong", { operationID: "delivery", requestHash: "other-input" })), + ).toMatchObject({ name: "StorageConflictError" }) + }) + + test("only reports missing records as absent and keeps ordered batched reads", async () => { + const store = await open() + await store.write(["a"], { value: 1 }) + expect(await store.readMany([["missing"], ["a"], ["a"]])).toEqual([undefined, { value: 1 }, { value: 1 }]) + expect(await failure(store.read>(["missing"]))).toMatchObject({ name: "NotFoundError" }) + await store.close() + expect(await failure(store.readMany([["a"]]))).toMatchObject({ name: "StorageClosedError" }) + }) + + test("enumerates logical keys without path collisions and deletes complete trees", async () => { + const store = await open() + await store.write(["a", "b/c"], { value: 1 }) + await store.write(["a", "b", "c"], { value: 2 }) + await store.write(["a", "other"], { value: 3 }) + expect(await store.scan(["a"])).toEqual(["b", "b/c", "other"]) + expect(await store.list(["a", "b"])).toEqual([["a", "b", "c"]]) + await store.removeTree(["a", "b"]) + expect(await store.scan(["a"])).toEqual(["b/c", "other"]) + expect(await store.read>(["a", "b/c"])).toEqual({ value: 1 }) + }) + + test("rejects stale revisions and never reuses the revision of deleted records", async () => { + const store = await open() + await store.write(["part"], { status: "running" }) + const before = await store.versioned(["part"]) + await store.write(["part"], { status: "completed" }) + expect( + await failure( + store.transaction((tx) => tx.write(["part"], { status: "running" }, { expectedRevision: before.revision })), + ), + ).toMatchObject({ name: "StorageConflictError" }) + await store.remove(["part"]) + await store.write(["part"], { status: "new" }) + expect((await store.versioned(["part"])).revision).toBeGreaterThan(before.revision) + }) + + test("records notifications in the same transaction and acknowledges them explicitly", async () => { + const store = await open() + expect( + await failure( + store.transaction(async (tx) => { + await tx.enqueue({ id: "rolled-back", scopeID: "scope", type: "changed", payload: {} }) + throw new Error("abort") + }), + ), + ).toMatchObject({ message: "abort" }) + expect(await store.pendingEvents()).toEqual([]) + await store.transaction(async (tx) => { + await tx.write(["record"], { value: 1 }) + await tx.enqueue({ id: "event", scopeID: "scope", type: "changed", payload: { value: 1 } }) + }) + expect((await store.pendingEvents()).map((event) => event.id)).toEqual(["event"]) + await store.acknowledgeEvents(["event"]) + expect(await store.pendingEvents()).toEqual([]) + }) + }) +} diff --git a/packages/harness/test/storage/verification.test.ts b/packages/harness/test/storage/verification.test.ts new file mode 100644 index 000000000..16b02fa46 --- /dev/null +++ b/packages/harness/test/storage/verification.test.ts @@ -0,0 +1,41 @@ +import { expect, test } from "bun:test" +import fs from "node:fs/promises" +import path from "node:path" +import { TransactionalStore } from "../../src/storage/transactional-store" + +async function fixture() { + const root = await fs.mkdtemp(path.join(process.env.SYNERGY_TEST_ROOT!, "verification-")) + const store = await TransactionalStore.open({ + backend: "sqlite", + namespace: "verify", + filename: path.join(root, "agent.sqlite"), + }) + return { + store, + async [Symbol.asyncDispose]() { + await store.close() + await fs.rm(root, { recursive: true, force: true }) + }, + } +} + +test("verification checks logical identities and reports missing parent records", async () => { + await using fixtureStore = await fixture() + const { store } = fixtureStore + await store.write(["sessions", "scope", "ses_one", "info"], { id: "ses_one", scope: { id: "scope" } }) + await store.write(["sessions", "scope", "ses_one", "messages", "msg_one", "parts", "part_one"], { + id: "part_one", + sessionID: "ses_one", + messageID: "msg_one", + }) + const broken = await store.verify() + expect(broken.records).toBe(2) + expect(broken.issues).toEqual([ + { key: ["sessions", "scope", "ses_one", "messages", "msg_one", "parts", "part_one"], reason: "missing_message" }, + ]) + await store.write(["sessions", "scope", "ses_one", "messages", "msg_one", "info"], { + id: "msg_one", + sessionID: "ses_one", + }) + expect((await store.verify()).issues).toEqual([]) +}) diff --git a/packages/harness/test/support/preload.ts b/packages/harness/test/support/preload.ts index cf0256466..bd99f71d3 100644 --- a/packages/harness/test/support/preload.ts +++ b/packages/harness/test/support/preload.ts @@ -8,3 +8,17 @@ const { runInProcessStream } = await import("../../src/session/agent-turn/in-pro Log.init({ print: false, dev: true, level: "DEBUG" }) AgentTurn.setInProcessStream(runInProcessStream) + +const { Storage } = await import("../../src/storage/storage") +const { TransactionalStore } = await import("../../src/storage/transactional-store") +const { beforeTestHomeDisposal } = await import("@ericsanchezok/synergy-testing/preload") +const storage = await TransactionalStore.open({ + backend: "sqlite", + namespace: "test", + filename: `${Global.Path.data}/storage/test.sqlite`, +}) +const uninstallStorage = Storage.install({ store: storage, artifactDirectory: Global.Path.data }) +beforeTestHomeDisposal(async () => { + await storage.close() + uninstallStorage() +}) diff --git a/packages/harness/test/tool/registry-observability.test.ts b/packages/harness/test/tool/registry-observability.test.ts index 0a10a4c7c..679984401 100644 --- a/packages/harness/test/tool/registry-observability.test.ts +++ b/packages/harness/test/tool/registry-observability.test.ts @@ -13,6 +13,8 @@ test("successful tool initialization does not emit per-tool info records", async import { ScopeContext } from "./src/scope/context.ts" import { ToolRegistry } from "./src/tool/registry.ts" + const { StorageMaintenance } = await import("./src/storage/maintenance.ts") + await using storage = await StorageMaintenance.open() await Log.init({ print: false, dev: true, level: "INFO" }) const { scope } = await Scope.fromDirectory(${JSON.stringify(project)}) const toolCount = await ScopeContext.provide({ diff --git a/packages/library/src/database.ts b/packages/library/src/database.ts index 000e94b17..146e512e0 100644 --- a/packages/library/src/database.ts +++ b/packages/library/src/database.ts @@ -1,3 +1,4 @@ +import { initializeSqliteEngine } from "@ericsanchezok/synergy-harness/storage/sqlite-engine" import { Database, type SQLQueryBindings } from "bun:sqlite" import * as sqliteVec from "sqlite-vec" import { Global } from "@ericsanchezok/synergy-harness/global" @@ -83,34 +84,6 @@ const MEMORY_RECALL_MODES = ["always", "contextual", "search_only"] as const type MemoryCategory = (typeof MEMORY_CATEGORIES)[number] type MemoryRecallMode = (typeof MEMORY_RECALL_MODES)[number] -const HOMEBREW_SQLITE_PATHS = [ - "/opt/homebrew/opt/sqlite/lib/libsqlite3.dylib", - "/usr/local/opt/sqlite/lib/libsqlite3.dylib", -] - -function setupCustomSQLite() { - if (process.platform !== "darwin") return - for (const p of HOMEBREW_SQLITE_PATHS) { - if (existsSync(p)) { - try { - Database.setCustomSQLite(p) - } catch (err) { - const message = err instanceof Error ? err.message : String(err) - if (message.includes("SQLite already loaded") || message.includes("exactly once")) { - log.debug("custom sqlite already initialized", { path: p }) - return - } - throw err - } - log.info("using custom sqlite", { path: p }) - return - } - } - log.warn("no homebrew sqlite found, extension loading may fail on macOS") -} - -setupCustomSQLite() - function loadSqliteVec(conn: Database) { const suffix = process.platform === "win32" ? "dll" : process.platform === "darwin" ? "dylib" : "so" @@ -134,6 +107,7 @@ function open(): Database { if (db) return db const dbPath = Global.Path.libraryDB log.info("open", { path: dbPath }) + initializeSqliteEngine() const conn = new Database(dbPath, { create: true }) try { loadSqliteVec(conn) diff --git a/packages/note/src/store.ts b/packages/note/src/store.ts index 0a29066db..588637313 100644 --- a/packages/note/src/store.ts +++ b/packages/note/src/store.ts @@ -173,24 +173,27 @@ export namespace NoteStore { .filter((entry): entry is Metadata => entry !== undefined) sortByPinAndTime(entries) return entries - } catch { - return rebuildIndex(scopeID) + } catch (error) { + if (error instanceof Storage.NotFoundError) return rebuildIndex(scopeID) + throw error } } async function rebuildIndex(scopeID: string): Promise { - const sid = Identifier.asScopeID(scopeID) - const ids = (await Storage.scan(StoragePath.notesRoot(sid))).filter((id) => !id.startsWith("_")) - if (ids.length === 0) return [] - const keys = ids.map((id) => StoragePath.note(sid, id)) - const results = await Storage.readMany>(keys) - const entries = results - .filter((n): n is z.infer => n !== undefined) - .map((n) => toMetadata(normalize(n))) - sortByPinAndTime(entries) - await Storage.write(indexPath(sid), entries) - log.info("index rebuilt", { scopeID, count: entries.length }) - return entries + return Storage.transaction(async () => { + const sid = Identifier.asScopeID(scopeID) + const ids = (await Storage.scan(StoragePath.notesRoot(sid))).filter((id) => !id.startsWith("_")) + if (ids.length === 0) return [] + const keys = ids.map((id) => StoragePath.note(sid, id)) + const results = await Storage.readMany>(keys) + const entries = results + .filter((n): n is z.infer => n !== undefined) + .map((n) => toMetadata(normalize(n))) + sortByPinAndTime(entries) + await Storage.write(indexPath(sid), entries) + log.info("index rebuilt", { scopeID, count: entries.length }) + return entries + }) } async function indexSet(scopeID: string, note: z.infer): Promise { @@ -237,8 +240,8 @@ export namespace NoteStore { silentNotFound: true, }) return { scopeID, note: normalize(note) } - } catch { - // fallthrough + } catch (error) { + if (!(error instanceof Storage.NotFoundError)) throw error } if (scopeID !== HOME_SCOPE_ID) { const globalSid = Identifier.asScopeID(HOME_SCOPE_ID) @@ -247,8 +250,8 @@ export namespace NoteStore { silentNotFound: true, }) return { scopeID: HOME_SCOPE_ID, note: normalize(note) } - } catch { - // fallthrough + } catch (error) { + if (!(error instanceof Storage.NotFoundError)) throw error } } const scopeIDs = await Storage.scan(["notes"]) @@ -309,10 +312,12 @@ export namespace NoteStore { version: 1, time: { created: now, updated: now }, } - await Storage.write(StoragePath.note(scopeID, id), note) - await indexSet(targetScopeID, note) - log.info("created", { id, title: note.title, global: isGlobal, scopeID: targetScopeID }) - await Bus.publish(NoteEvent.Created, { scopeID: targetScopeID, note, meta: toMetadata(note) }) + await Storage.transaction(async () => { + await Storage.write(StoragePath.note(scopeID, id), note) + await indexSet(targetScopeID, note) + log.info("created", { id, title: note.title, global: isGlobal, scopeID: targetScopeID }) + await Bus.publish(NoteEvent.Created, { scopeID: targetScopeID, note, meta: toMetadata(note) }) + }) await SessionPluginHooks.trigger( "note.create.after", { @@ -395,81 +400,84 @@ export namespace NoteStore { }, ) - let wasGlobal = false - const before = structuredClone(current) - const note = normalize( - await Storage.update>(sourcePath, (draft) => { - draft.global ??= false - draft.version ??= 1 - wasGlobal = draft.global - if (update.patch.expectedVersion !== undefined && update.patch.expectedVersion !== draft.version) { - throw new NoteError.Conflict({ - noteID, - expectedVersion: update.patch.expectedVersion, - note: normalize(draft), - }) - } - if (update.patch.title !== undefined) draft.title = update.patch.title - if (update.patch.content !== undefined) draft.content = NoteDocument.normalize(update.patch.content) - if (update.patch.pinned !== undefined) draft.pinned = update.patch.pinned - if (update.patch.tags !== undefined) draft.tags = update.patch.tags - if (update.patch.kind !== undefined) draft.kind = update.patch.kind - if (update.patch.blueprint === null) { - draft.blueprint = undefined - } else if (update.patch.blueprint !== undefined) { - const { activeLoopID, ...rest } = update.patch.blueprint as z.infer< - typeof NoteTypes.PatchInput - >["blueprint"] & { - status?: unknown + const committed = await Storage.transaction(async () => { + let wasGlobal = false + const before = structuredClone(current) + const note = normalize( + await Storage.update>(sourcePath, (draft) => { + draft.global ??= false + draft.version ??= 1 + wasGlobal = draft.global + if (update.patch.expectedVersion !== undefined && update.patch.expectedVersion !== draft.version) { + throw new NoteError.Conflict({ + noteID, + expectedVersion: update.patch.expectedVersion, + note: normalize(draft), + }) } - delete rest.status - const next = { ...(draft.blueprint ?? {}), ...rest } - if (activeLoopID !== undefined && activeLoopID !== null) next.activeLoopID = activeLoopID - if (activeLoopID === null) delete next.activeLoopID - draft.blueprint = next - } - if (update.patch.global !== undefined) draft.global = update.patch.global - if (update.patch.global === true && !wasGlobal) { - draft.originScope = sid as string - } - if (update.patch.archived !== undefined) { - draft.archived = update.patch.archived - } - draft.version += 1 - draft.time.updated = Date.now() - }), - ) - - const isNowGlobal = note.global ?? false - let finalScopeID = scopeID - if (!wasGlobal && isNowGlobal) { - const globalSid = Identifier.asScopeID(HOME_SCOPE_ID) - await Storage.write(StoragePath.note(globalSid, noteID), note) - await Storage.remove(sourcePath) - await indexRemove(scopeID, noteID) - await indexSet(HOME_SCOPE_ID, note) - finalScopeID = HOME_SCOPE_ID - log.info("promoted to global", { id: noteID, from: sid }) - } else if (wasGlobal && !isNowGlobal) { - const targetSid = Identifier.asScopeID(note.originScope || scopeID) - await Storage.write(StoragePath.note(targetSid, noteID), note) - await Storage.remove(sourcePath) - await indexRemove(scopeID, noteID) - await indexSet(note.originScope || scopeID, note) - finalScopeID = note.originScope || scopeID - log.info("demoted from global", { id: noteID, to: targetSid }) - } else { - await indexSet(scopeID, note) - } + if (update.patch.title !== undefined) draft.title = update.patch.title + if (update.patch.content !== undefined) draft.content = NoteDocument.normalize(update.patch.content) + if (update.patch.pinned !== undefined) draft.pinned = update.patch.pinned + if (update.patch.tags !== undefined) draft.tags = update.patch.tags + if (update.patch.kind !== undefined) draft.kind = update.patch.kind + if (update.patch.blueprint === null) { + draft.blueprint = undefined + } else if (update.patch.blueprint !== undefined) { + const { activeLoopID, ...rest } = update.patch.blueprint as z.infer< + typeof NoteTypes.PatchInput + >["blueprint"] & { + status?: unknown + } + delete rest.status + const next = { ...(draft.blueprint ?? {}), ...rest } + if (activeLoopID !== undefined && activeLoopID !== null) next.activeLoopID = activeLoopID + if (activeLoopID === null) delete next.activeLoopID + draft.blueprint = next + } + if (update.patch.global !== undefined) draft.global = update.patch.global + if (update.patch.global === true && !wasGlobal) { + draft.originScope = sid as string + } + if (update.patch.archived !== undefined) { + draft.archived = update.patch.archived + } + draft.version += 1 + draft.time.updated = Date.now() + }), + ) + + const isNowGlobal = note.global ?? false + let finalScopeID = scopeID + if (!wasGlobal && isNowGlobal) { + const globalSid = Identifier.asScopeID(HOME_SCOPE_ID) + await Storage.write(StoragePath.note(globalSid, noteID), note) + await Storage.remove(sourcePath) + await indexRemove(scopeID, noteID) + await indexSet(HOME_SCOPE_ID, note) + finalScopeID = HOME_SCOPE_ID + log.info("promoted to global", { id: noteID, from: sid }) + } else if (wasGlobal && !isNowGlobal) { + const targetSid = Identifier.asScopeID(note.originScope || scopeID) + await Storage.write(StoragePath.note(targetSid, noteID), note) + await Storage.remove(sourcePath) + await indexRemove(scopeID, noteID) + await indexSet(note.originScope || scopeID, note) + finalScopeID = note.originScope || scopeID + log.info("demoted from global", { id: noteID, to: targetSid }) + } else { + await indexSet(scopeID, note) + } - const meta = toMetadata(note) - log.info("updated", { id: noteID, version: note.version }) - await Bus.publish(NoteEvent.Updated, { scopeID: finalScopeID, note, meta, changed: changedFields(before, note) }) - if (patch.archived === true) { - await Bus.publish(NoteEvent.Archived, { ids: [noteID], scopeID: finalScopeID, metas: [meta] }) - } else if (patch.archived === false) { - await Bus.publish(NoteEvent.Unarchived, { ids: [noteID], scopeID: finalScopeID, metas: [meta] }) - } + const meta = toMetadata(note) + log.info("updated", { id: noteID, version: note.version }) + await Bus.publish(NoteEvent.Updated, { scopeID: finalScopeID, note, meta, changed: changedFields(before, note) }) + if (patch.archived === true) { + await Bus.publish(NoteEvent.Archived, { ids: [noteID], scopeID: finalScopeID, metas: [meta] }) + } else if (patch.archived === false) { + await Bus.publish(NoteEvent.Unarchived, { ids: [noteID], scopeID: finalScopeID, metas: [meta] }) + } + return note + }) await SessionPluginHooks.trigger( "note.update.after", { @@ -477,25 +485,60 @@ export namespace NoteStore { noteID, }, { - note, + note: committed, }, ) - return note + return committed + } + + export async function recordBlueprintRun(input: { + scopeID: string + noteID: string + loopID: string + started?: number + ended?: boolean + archive?: boolean + }): Promise { + await Storage.transaction(async () => { + const key = StoragePath.note(Identifier.asScopeID(input.scopeID), input.noteID) + const [stored] = await Storage.readMany>([key]) + if (!stored || stored.kind !== "blueprint") return + const before = structuredClone(normalize(stored)) + const note = structuredClone(before) + note.blueprint ??= {} + if (input.started !== undefined) { + note.blueprint.runCount = (note.blueprint.runCount ?? 0) + 1 + note.blueprint.lastRunAt = input.started + note.blueprint.activeLoopID = input.loopID + } + if (input.ended && note.blueprint.activeLoopID === input.loopID) delete note.blueprint.activeLoopID + if (input.archive) note.archived = true + note.version += 1 + note.time.updated = Date.now() + await Storage.write(key, note) + await indexSet(input.scopeID, note) + const meta = toMetadata(note) + await Bus.publish(NoteEvent.Updated, { scopeID: input.scopeID, note, meta, changed: changedFields(before, note) }) + if (input.archive) + await Bus.publish(NoteEvent.Archived, { ids: [note.id], scopeID: input.scopeID, metas: [meta] }) + }) } export async function remove(scopeID: string, noteID: string): Promise { - const sid = Identifier.asScopeID(scopeID) - const note = normalize(await Storage.read>(StoragePath.note(sid, noteID))) - if (!note.archived) { - throw new NoteError.NotArchived({ - noteID, - message: "Note must be archived before it can be deleted. Use note_archive first.", - }) - } - await Storage.remove(StoragePath.note(sid, noteID)) - await indexRemove(scopeID, noteID) - log.info("removed", { id: noteID }) - await Bus.publish(NoteEvent.Deleted, { id: noteID, scopeID }) + return Storage.transaction(async () => { + const sid = Identifier.asScopeID(scopeID) + const note = normalize(await Storage.read>(StoragePath.note(sid, noteID))) + if (!note.archived) { + throw new NoteError.NotArchived({ + noteID, + message: "Note must be archived before it can be deleted. Use note_archive first.", + }) + } + await Storage.remove(StoragePath.note(sid, noteID)) + await indexRemove(scopeID, noteID) + log.info("removed", { id: noteID }) + await Bus.publish(NoteEvent.Deleted, { id: noteID, scopeID }) + }) } async function groupResolvedNoteIDs(scopeID: string, noteIDs: string[]): Promise> { @@ -514,30 +557,32 @@ export namespace NoteStore { noteIDs: string[], archived: boolean, ): Promise[]> { - const grouped = await groupResolvedNoteIDs(scopeID, noteIDs) - const results: z.infer[] = [] - for (const [resolvedScopeID, ids] of grouped) { - const sid = Identifier.asScopeID(resolvedScopeID) - const scopedResults: z.infer[] = [] - for (const noteID of ids) { - const sourcePath = StoragePath.note(sid, noteID) - const note = normalize( - await Storage.update>(sourcePath, (draft) => { - draft.version ??= 1 - draft.archived = archived - draft.version += 1 - draft.time.updated = Date.now() - }), - ) - scopedResults.push(note) - results.push(note) + return Storage.transaction(async () => { + const grouped = await groupResolvedNoteIDs(scopeID, noteIDs) + const results: z.infer[] = [] + for (const [resolvedScopeID, ids] of grouped) { + const sid = Identifier.asScopeID(resolvedScopeID) + const scopedResults: z.infer[] = [] + for (const noteID of ids) { + const sourcePath = StoragePath.note(sid, noteID) + const note = normalize( + await Storage.update>(sourcePath, (draft) => { + draft.version ??= 1 + draft.archived = archived + draft.version += 1 + draft.time.updated = Date.now() + }), + ) + scopedResults.push(note) + results.push(note) + } + await indexUpdateMany(resolvedScopeID, scopedResults) + const payload = { ids, scopeID: resolvedScopeID, metas: scopedResults.map(toMetadata) } + await Bus.publish(archived ? NoteEvent.Archived : NoteEvent.Unarchived, payload) } - await indexUpdateMany(resolvedScopeID, scopedResults) - const payload = { ids, scopeID: resolvedScopeID, metas: scopedResults.map(toMetadata) } - await Bus.publish(archived ? NoteEvent.Archived : NoteEvent.Unarchived, payload) - } - log.info(archived ? "archived" : "unarchived", { ids: noteIDs, count: noteIDs.length }) - return results + log.info(archived ? "archived" : "unarchived", { ids: noteIDs, count: noteIDs.length }) + return results + }) } export async function archive(scopeID: string, noteIDs: string[]): Promise[]> { diff --git a/packages/note/test/storage-atomicity.test.ts b/packages/note/test/storage-atomicity.test.ts new file mode 100644 index 000000000..4923c5825 --- /dev/null +++ b/packages/note/test/storage-atomicity.test.ts @@ -0,0 +1,30 @@ +import { expect, spyOn, test } from "bun:test" +import { Scope } from "@ericsanchezok/synergy-harness/scope" +import { ScopeContext } from "@ericsanchezok/synergy-harness/scope/context" +import { Storage } from "@ericsanchezok/synergy-harness/storage/storage" +import { tmpdir } from "@ericsanchezok/synergy-harness/test/support/fixture" +import { NoteStore } from "../src" + +test("a failed global promotion preserves the original Note and both indexes", async () => { + await using tmp = await tmpdir({ git: true }) + const scope = (await Scope.fromDirectory(tmp.path)).scope + await ScopeContext.provide({ + scope, + fn: async () => { + const note = await NoteStore.create({ title: "Original" }) + const original = Storage.write + const write = spyOn(Storage, "write").mockImplementation(async (key, value) => { + if (key[0] === "notes" && key[1] === "home" && key[2] === "_index") throw new Error("index unavailable") + return original(key, value) + }) + try { + await expect(NoteStore.update(scope.id, note.id, { global: true })).rejects.toThrow("index unavailable") + } finally { + write.mockRestore() + } + expect(await NoteStore.get(scope.id, note.id)).toEqual(note) + await expect(NoteStore.get("home", note.id)).rejects.toBeInstanceOf(Storage.NotFoundError) + expect((await NoteStore.listMeta(scope.id)).some((entry) => entry.id === note.id)).toBe(true) + }, + }) +}) diff --git a/packages/plugin-host/src/plugin/audit.ts b/packages/plugin-host/src/plugin/audit.ts index 252718220..d4d5a534e 100644 --- a/packages/plugin-host/src/plugin/audit.ts +++ b/packages/plugin-host/src/plugin/audit.ts @@ -1,6 +1,4 @@ -import path from "path" -import fs from "fs/promises" -import { Global } from "@ericsanchezok/synergy-harness/global" +import { Storage } from "@ericsanchezok/synergy-harness/storage/storage" // --------------------------------------------------------------------------- // Types @@ -28,55 +26,28 @@ export interface PluginAuditEvent { details: Record } -// --------------------------------------------------------------------------- -// Storage path -// --------------------------------------------------------------------------- - -function auditPath(): string { - return path.join(Global.Path.data, "plugin-audit.json") -} - -// --------------------------------------------------------------------------- -// JSON read / write helpers -// --------------------------------------------------------------------------- - -async function readAll(): Promise { - try { - const text = await Bun.file(auditPath()).text() - return JSON.parse(text) - } catch { - return [] - } -} - -async function writeAll(events: PluginAuditEvent[]): Promise { - const p = auditPath() - await fs.mkdir(path.dirname(p), { recursive: true }) - await Bun.write(p, JSON.stringify(events, null, 2)) -} - -// --------------------------------------------------------------------------- -// Public API -// --------------------------------------------------------------------------- - export async function recordEvent(event: Omit): Promise { const full: PluginAuditEvent = { ...event, id: crypto.randomUUID(), time: Date.now(), } - const events = await readAll() - events.push(full) - await writeAll(events) + await Storage.write(["plugin-audit", "events", `${String(full.time).padStart(16, "0")}_${full.id}`], full) } export async function getEvents(pluginId?: string, limit?: number): Promise { - const events = await readAll() - const filtered = pluginId ? events.filter((e) => e.pluginId === pluginId) : events - if (limit != null && limit > 0) { - return filtered.slice(-limit) + const events: PluginAuditEvent[] = [] + const bounded = limit !== undefined && limit > 0 + for await (const record of Storage.records({ + kind: "plugin-audit", + descending: bounded, + limit: 128, + })) { + if (record.key[1] !== "events" || (pluginId && record.value.pluginId !== pluginId)) continue + events.push(record.value) + if (bounded && events.length >= limit) break } - return filtered + return bounded ? events.reverse() : events } export async function getRecentEvents(limit?: number): Promise { diff --git a/packages/plugin-host/src/plugin/cli-metadata.ts b/packages/plugin-host/src/plugin/cli-metadata.ts index 30dcad7b3..7a3514b49 100644 --- a/packages/plugin-host/src/plugin/cli-metadata.ts +++ b/packages/plugin-host/src/plugin/cli-metadata.ts @@ -1,9 +1,21 @@ +import { Storage } from "@ericsanchezok/synergy-harness/storage/storage" +import { StorageBootstrap } from "@ericsanchezok/synergy-harness/storage/bootstrap" +import { Global } from "@ericsanchezok/synergy-harness/global" import { computeManifestHash } from "@ericsanchezok/synergy-plugin/integrity" import { read } from "./lockfile" import { findPackageRoot, readPluginManifest } from "./spec-resolver" import type { PluginManifestType } from "@ericsanchezok/synergy-plugin" export async function installedPluginCliMetadata(): Promise> { + if (!Storage.available()) { + const handle = await StorageBootstrap.inspect(Global.Path.root) + if (!handle) return [] + try { + return await Storage.provide(handle, () => installedPluginCliMetadata()) + } finally { + await handle.store.close() + } + } const lock = await read().catch((error) => { console.error(`Plugin CLI metadata unavailable: ${error instanceof Error ? error.message : String(error)}`) return undefined diff --git a/packages/plugin-host/src/plugin/consent/approval-service.ts b/packages/plugin-host/src/plugin/consent/approval-service.ts index 577855d64..cba7aa90f 100644 --- a/packages/plugin-host/src/plugin/consent/approval-service.ts +++ b/packages/plugin-host/src/plugin/consent/approval-service.ts @@ -12,7 +12,7 @@ import { getDisabledPlugin, state as loaderState } from "../loader" import { resolvePluginSpec } from "../spec-resolver" import * as Lockfile from "../lockfile" import { PluginMarketplaceRegistry } from "../marketplace-registry" -import { localRegistryPath, resolveLocalRegistryInstallSpec } from "../local-registry-store" +import { readLocalRegistry, resolveLocalRegistryInstallSpec } from "../local-registry-store" import { pathToFileURL } from "url" import { ScopeContext } from "@ericsanchezok/synergy-harness/scope/context" @@ -131,10 +131,7 @@ export async function resolveRegistrySpec( official: true, } } - const registry = JSON.parse(await Bun.file(localRegistryPath()).text()) as { - plugins?: Array> - } - const entry = registry.plugins?.find((candidate) => candidate.id === id) + const entry = (await readLocalRegistry()).find((candidate) => candidate.id === id) if (!entry) throw new ApprovalPluginNotFoundError(`Local registry plugin not found: ${id}`) const versions = Array.isArray(entry.versions) ? entry.versions : [] const matched = versions.find( diff --git a/packages/plugin-host/src/plugin/consent/approval-store.ts b/packages/plugin-host/src/plugin/consent/approval-store.ts index 6b953ee0c..5c18935ad 100644 --- a/packages/plugin-host/src/plugin/consent/approval-store.ts +++ b/packages/plugin-host/src/plugin/consent/approval-store.ts @@ -1,13 +1,10 @@ -import path from "path" import { manifestHasTrustedUI, type PluginManifestType } from "@ericsanchezok/synergy-plugin" import { computePermissionsHash, permissionsHashPayload, type PluginGrantContract, } from "@ericsanchezok/synergy-plugin/integrity" -import { Global } from "@ericsanchezok/synergy-harness/global" import { Storage } from "@ericsanchezok/synergy-harness/storage/storage" -import { Lock } from "@ericsanchezok/synergy-harness/util/lock" import type { PluginSource, TrustTier } from "../trust.js" import { comparePluginAccess } from "./diff.js" @@ -24,40 +21,20 @@ export interface PluginApprovalRecord { approvedCapabilities: string[] } -function approvalPath() { - return path.join(Global.Path.data, "plugin-approvals.json") -} - async function readAll(): Promise { - try { - const value = JSON.parse(await Bun.file(approvalPath()).text()) - return Array.isArray(value) - ? value.filter( - (record): record is PluginApprovalRecord => - record?.schemaVersion === 2 && - typeof record.pluginId === "string" && - typeof record.grantHash === "string" && - record.grant && - typeof record.grant === "object", - ) - : [] - } catch { - return [] - } -} - -// Lock is not reentrant: writeAll must stay lock-free because saveApproval, -// removeApproval, and writeApprovals already hold Lock.write across their -// read-modify-write, which serializes concurrent approval mutations in-process. -async function writeAll(records: PluginApprovalRecord[]) { - await Storage.writeJsonAtomic(approvalPath(), `${JSON.stringify(records, null, 2)}\n`) + const keys = await Storage.list(["plugin-approvals", "records"]) + return (await Storage.readMany(keys)) + .filter((record): record is PluginApprovalRecord => record !== undefined) + .sort((a, b) => a.pluginId.localeCompare(b.pluginId)) } export const readApprovals = readAll export async function writeApprovals(records: PluginApprovalRecord[]) { - using _ = await Lock.write(approvalPath()) - await writeAll(records) + await Storage.transaction(async (tx) => { + await tx.removeTree(["plugin-approvals", "records"]) + for (const record of records) await tx.write(["plugin-approvals", "records", record.pluginId], record) + }) } export function createApprovalRecord(input: { @@ -84,22 +61,16 @@ export function createApprovalRecord(input: { } export async function getApproval(pluginId: string, manifest?: PluginManifestType) { - const records = (await readAll()) - .filter((record) => record.pluginId === pluginId) - .sort((left, right) => right.approvedAt - left.approvedAt) - return manifest ? records.find((record) => verifyApproval(record, manifest)) : records[0] + const [record] = await Storage.readMany([["plugin-approvals", "records", pluginId]]) + return record && (!manifest || verifyApproval(record, manifest)) ? record : undefined } export async function saveApproval(record: PluginApprovalRecord) { - using _ = await Lock.write(approvalPath()) - const records = (await readAll()).filter((item) => item.pluginId !== record.pluginId) - records.push(record) - await writeAll(records) + await Storage.write(["plugin-approvals", "records", record.pluginId], record) } export async function removeApproval(pluginId: string) { - using _ = await Lock.write(approvalPath()) - await writeAll((await readAll()).filter((record) => record.pluginId !== pluginId)) + await Storage.remove(["plugin-approvals", "records", pluginId]) } export function verifyApproval( diff --git a/packages/plugin-host/src/plugin/doctor.ts b/packages/plugin-host/src/plugin/doctor.ts index 256d548fc..61838ca97 100644 --- a/packages/plugin-host/src/plugin/doctor.ts +++ b/packages/plugin-host/src/plugin/doctor.ts @@ -1,3 +1,4 @@ +import { Storage } from "@ericsanchezok/synergy-harness/storage/storage" import fs from "fs/promises" import fsSync from "fs" import path from "path" @@ -54,18 +55,11 @@ async function listArchiveCacheDirs(): Promise { return entries.filter((entry) => entry.isDirectory()).map((entry) => path.join(root, entry.name)) } -function runtimeStatePath(): string { - return path.join(Global.Path.data, "plugin-runtime-state.json") -} - -async function readRawRuntimeState(): Promise { - try { - const text = await Bun.file(runtimeStatePath()).text() - const parsed = JSON.parse(text) - return Array.isArray(parsed) ? parsed : [] - } catch { - return [] - } +async function readRawRuntimeState(): Promise[]> { + const [value] = await Storage.readMany[]>([["plugin-runtime-state"]]) + if (value === undefined) return [] + if (!Array.isArray(value)) throw new Error("Plugin runtime state is not an array") + return value } function runtimeStateEntryUsable(entry: any): boolean { @@ -257,8 +251,7 @@ export async function doctor(options: { fix?: boolean } = {}): Promise -function filepath(data: string) { - return path.join(data, "plugin-incompatible.json") -} - export namespace IncompatiblePluginStore { - export async function read(data = Global.Path.data): Promise { - try { - const value = JSON.parse(await fs.readFile(filepath(data), "utf8")) - return z.array(IncompatiblePluginRecord).parse(value) - } catch (error: any) { - if (error?.code === "ENOENT") return [] - throw error - } + export async function read(): Promise { + const [value] = await Storage.readMany([["plugin-incompatible"]]) + return value === undefined ? [] : z.array(IncompatiblePluginRecord).parse(value) } - export async function write(records: IncompatiblePluginRecord[], data = Global.Path.data): Promise { - await Storage.writeJsonAtomic(filepath(data), `${JSON.stringify(records, null, 2)}\n`) + export async function write(records: IncompatiblePluginRecord[]): Promise { + await Storage.write(["plugin-incompatible"], records) } export function withoutPlugin(records: IncompatiblePluginRecord[], pluginId: string, specs: string[] = []) { diff --git a/packages/plugin-host/src/plugin/installation-recovery.ts b/packages/plugin-host/src/plugin/installation-recovery.ts new file mode 100644 index 000000000..ba85afb40 --- /dev/null +++ b/packages/plugin-host/src/plugin/installation-recovery.ts @@ -0,0 +1,196 @@ +import { isDeepStrictEqual } from "node:util" +import fs from "node:fs/promises" +import path from "node:path" +import { randomUUID } from "node:crypto" +import { z } from "zod" +import { Storage } from "@ericsanchezok/synergy-harness/storage/storage" +import { Config } from "@ericsanchezok/synergy-harness/config/config" +import { Lock } from "@ericsanchezok/synergy-harness/util/lock" +import * as Lockfile from "./lockfile" +import { readApprovals, removeApproval, saveApproval } from "./consent/approval-store" +import { IncompatiblePluginStore } from "./incompatible-store" +import type { ResolvedPluginSpec } from "./spec-resolver" + +const Intent = z + .object({ + version: z.literal(1), + id: z.uuid(), + status: z.enum(["pending", "complete"]), + sha256: z.string().regex(/^[a-f0-9]{64}$/), + }) + .strict() +type Snapshot = Awaited> +const root = () => path.join(Storage.current().artifactDirectory, "plugin-install-artifacts") +const snapshotPath = (id: string) => path.join(root(), `${z.uuid().parse(id)}.json`) + +async function capture(pluginId: string, nextDomain: Config.Info | undefined, resolved?: ResolvedPluginSpec) { + const id = randomUUID() + const finalDir = resolved?.stagingDir ? resolved.finalPluginDir : undefined + const promotion = + finalDir && resolved?.stagingDir + ? { + finalDir, + stagingDir: resolved.stagingDir, + backupDir: path.join(root(), "rollback", id), + hadOriginal: await exists(finalDir), + } + : undefined + return { + id, + pluginId, + previousDomain: await Config.domainGet("plugins"), + nextDomain, + lockfile: await Lockfile.read(), + approvals: await readApprovals(), + incompatible: await IncompatiblePluginStore.read(), + promotion, + } +} + +async function exists(filename: string) { + try { + await fs.lstat(filename) + return true + } catch (error) { + if (error && typeof error === "object" && "code" in error && error.code === "ENOENT") return false + throw error + } +} + +async function syncDirectories(...directories: string[]) { + if (process.platform === "win32") return + for (const directory of new Set(directories)) { + const handle = await fs.open(directory, "r") + try { + await handle.sync() + } finally { + await handle.close() + } + } +} + +async function promote(snapshot: Snapshot) { + const p = snapshot.promotion + if (!p) return + await fs.mkdir(path.dirname(p.backupDir), { recursive: true, mode: 0o700 }) + await fs.mkdir(path.dirname(p.finalDir), { recursive: true, mode: 0o700 }) + if (p.hadOriginal) await fs.rename(p.finalDir, p.backupDir) + await fs.rename(p.stagingDir, p.finalDir) + await syncDirectories(path.dirname(p.finalDir), path.dirname(p.stagingDir), path.dirname(p.backupDir)) +} + +async function finish(snapshot: Snapshot, sha256: string) { + await Storage.write(["plugin-install-intents", snapshot.id], { + version: 1, + id: snapshot.id, + status: "complete", + sha256, + }) + await cleanup(snapshot) +} + +async function cleanup(snapshot: Snapshot) { + if (snapshot.promotion) await fs.rm(snapshot.promotion.backupDir, { recursive: true, force: true }) + await Storage.remove(["plugin-install-intents", snapshot.id]) + await fs.rm(snapshotPath(snapshot.id), { force: true }) +} + +async function rollback(snapshot: Snapshot, sha256: string) { + const [state] = await Storage.readMany([["plugin-install-intents", snapshot.id]]) + if (!state) return + if (Intent.parse(state).status === "complete") { + await cleanup(snapshot) + return + } + const current = await Config.domainGet("plugins") + if ( + snapshot.nextDomain && + !isDeepStrictEqual(current, snapshot.previousDomain) && + !isDeepStrictEqual(current, snapshot.nextDomain) + ) + throw new Error( + "Plugin configuration changed during installation; the recovery snapshot is retained for reconciliation", + ) + const promotion = snapshot.promotion + if (promotion) { + if (await exists(promotion.backupDir)) { + await fs.rm(promotion.finalDir, { recursive: true, force: true }) + await fs.rename(promotion.backupDir, promotion.finalDir) + await syncDirectories(path.dirname(promotion.backupDir), path.dirname(promotion.finalDir)) + } else if (!promotion.hadOriginal && !(await exists(promotion.stagingDir))) + await fs.rm(promotion.finalDir, { recursive: true, force: true }) + } + if (snapshot.nextDomain) await Config.domainUpdate("plugins", snapshot.previousDomain, { mode: "replace-domain" }) + await Storage.transaction(async () => { + const currentLock = await Lockfile.read() + for (const [id, entry] of Object.entries(currentLock.plugins)) + if (id === snapshot.pluginId || entry.approvalId === snapshot.pluginId) delete currentLock.plugins[id] + for (const [id, entry] of Object.entries(snapshot.lockfile.plugins)) + if (id === snapshot.pluginId || entry.approvalId === snapshot.pluginId) currentLock.plugins[id] = entry + await Lockfile.write(currentLock) + const approval = snapshot.approvals.find((entry) => entry.pluginId === snapshot.pluginId) + if (approval) await saveApproval(approval) + else await removeApproval(snapshot.pluginId) + await IncompatiblePluginStore.write([ + ...(await IncompatiblePluginStore.read()).filter((entry) => entry.pluginId !== snapshot.pluginId), + ...snapshot.incompatible.filter((entry) => entry.pluginId === snapshot.pluginId), + ]) + await Storage.write(["plugin-install-intents", snapshot.id], { + version: 1, + id: snapshot.id, + status: "complete", + sha256, + }) + }) + await cleanup(snapshot) +} + +export namespace PluginInstallationRecovery { + export async function begin(pluginId: string, nextDomain?: Config.Info, resolved?: ResolvedPluginSpec) { + const snapshot = await capture(pluginId, nextDomain, resolved) + const text = JSON.stringify(snapshot) + const sha256 = new Bun.CryptoHasher("sha256").update(text).digest("hex") + await Storage.writeJsonAtomic(snapshotPath(snapshot.id), text, { private: true, durable: true }) + await Storage.write(["plugin-install-intents", snapshot.id], { + version: 1, + id: snapshot.id, + status: "pending", + sha256, + }) + return { + promote: () => promote(snapshot), + finish: () => finish(snapshot, sha256), + rollback: () => rollback(snapshot, sha256), + } + } + + export async function recoverUnlocked() { + for (const key of await Storage.list(["plugin-install-intents"])) { + const intent = Intent.parse(await Storage.read(key)) + const text = await Bun.file(snapshotPath(intent.id)).text() + if (new Bun.CryptoHasher("sha256").update(text).digest("hex") !== intent.sha256) + throw new Error("Plugin recovery snapshot checksum mismatch") + const snapshot = JSON.parse(text) as Snapshot + if (snapshot.id !== intent.id || typeof snapshot.pluginId !== "string" || !snapshot.pluginId) + throw new Error("Plugin recovery identity mismatch") + if (snapshot.promotion) { + const p = snapshot.promotion + if ( + p.backupDir !== path.join(root(), "rollback", intent.id) || + !path.isAbsolute(p.finalDir) || + !path.isAbsolute(p.stagingDir) || + p.finalDir === p.stagingDir || + path.dirname(p.finalDir) === p.finalDir + ) + throw new Error("Invalid plugin recovery directory ownership") + } + if (intent.status === "complete") await cleanup(snapshot) + else await rollback(snapshot, intent.sha256) + } + } + + export async function recover() { + using lock = await Lock.write("plugin-installation") + await recoverUnlocked() + } +} diff --git a/packages/plugin-host/src/plugin/installation-transaction.ts b/packages/plugin-host/src/plugin/installation-transaction.ts index 185d937ea..641ded180 100644 --- a/packages/plugin-host/src/plugin/installation-transaction.ts +++ b/packages/plugin-host/src/plugin/installation-transaction.ts @@ -1,14 +1,15 @@ +import { Storage } from "@ericsanchezok/synergy-harness/storage/storage" +import { Lock } from "@ericsanchezok/synergy-harness/util/lock" +import { PluginInstallationRecovery } from "./installation-recovery" import type { LoadedPlugin } from "./loader" import type { PluginLockEntry } from "./lockfile-schema" import type { PluginApprovalRecord } from "./consent/approval-store" import fs from "fs/promises" -import fsSync from "fs" import path from "path" -import { Global } from "@ericsanchezok/synergy-harness/global" import { Config } from "@ericsanchezok/synergy-harness/config/config" import * as Lockfile from "./lockfile" import { addEntry, removePluginEntries } from "./lockfile" -import { readApprovals, removeApproval, saveApproval, writeApprovals } from "./consent/approval-store" +import { removeApproval, saveApproval } from "./consent/approval-store" import type { ResolvedPluginSpec } from "./spec-resolver" import { Log } from "@ericsanchezok/synergy-harness/util/log" import { recordEvent } from "./audit" @@ -16,10 +17,6 @@ import { IncompatiblePluginStore } from "./incompatible-store" const log = Log.create({ service: "plugin.install.transaction" }) -const LOCK_STALE_MS = 120_000 -const LOCK_POLL_MS = 50 -const LOCK_TIMEOUT_MS = 30_000 - export interface CanonicalizePluginSpecsInput { specs: string[] pluginId: string @@ -88,47 +85,10 @@ export interface PluginDoctorResult { changed: boolean } -function sleep(ms: number) { - return new Promise((resolve) => setTimeout(resolve, ms)) -} - -async function acquireLock(): Promise<() => Promise> { - const lockDir = path.join(Global.Path.state, "plugin-install", "transaction.lock") - const deadline = Date.now() + LOCK_TIMEOUT_MS - await fs.mkdir(path.dirname(lockDir), { recursive: true }) - - while (true) { - try { - await fs.mkdir(lockDir) - await Bun.write(path.join(lockDir, "owner.json"), JSON.stringify({ pid: process.pid, createdAt: Date.now() })) - let released = false - return async () => { - if (released) return - released = true - await fs.rm(lockDir, { recursive: true, force: true }).catch(() => {}) - } - } catch (err: any) { - if (err?.code !== "EEXIST") throw err - const stat = await fs.stat(lockDir).catch(() => null) - if (stat && Date.now() - stat.mtimeMs > LOCK_STALE_MS) { - await fs.rm(lockDir, { recursive: true, force: true }).catch(() => {}) - continue - } - if (Date.now() > deadline) { - throw new Error("Timed out waiting for plugin installation lock") - } - await sleep(LOCK_POLL_MS) - } - } -} - export async function withPluginInstallationLock(fn: () => Promise): Promise { - const release = await acquireLock() - try { - return await fn() - } finally { - await release() - } + using lock = await Lock.write("plugin-installation") + await PluginInstallationRecovery.recoverUnlocked() + return fn() } export async function canonicalizePluginSpecs( @@ -220,52 +180,6 @@ function resolvedPathAfterPromotion(resolved: ResolvedPluginSpec): string { return path.join(resolved.finalPluginDir, path.relative(resolved.stagingDir, resolvedFile)) } -async function promoteStagingDir(resolved: ResolvedPluginSpec): Promise<{ - finalDir?: string - backupDir?: string - restore: () => Promise - cleanup: () => Promise -}> { - if (!resolved.stagingDir || !resolved.finalPluginDir) { - return { restore: async () => {}, cleanup: async () => {} } - } - - const finalDir = resolved.finalPluginDir - const backupDir = path.join( - Global.Path.state, - "plugin-install", - "rollback", - `${path.basename(finalDir)}-${process.pid}-${Date.now()}`, - ) - await fs.mkdir(path.dirname(backupDir), { recursive: true }) - await fs.mkdir(path.dirname(finalDir), { recursive: true }) - - let hasBackup = false - if (fsSync.existsSync(finalDir)) { - await fs.rm(backupDir, { recursive: true, force: true }) - await fs.rename(finalDir, backupDir) - hasBackup = true - } - - await fs.rename(resolved.stagingDir, finalDir) - - return { - finalDir, - backupDir: hasBackup ? backupDir : undefined, - restore: async () => { - await fs.rm(finalDir, { recursive: true, force: true }).catch(() => {}) - if (hasBackup) { - await fs.rename(backupDir, finalDir).catch(async () => { - await fs.rm(backupDir, { recursive: true, force: true }).catch(() => {}) - }) - } - }, - cleanup: async () => { - if (hasBackup) await fs.rm(backupDir, { recursive: true, force: true }).catch(() => {}) - }, - } -} - function assertSingleLoadedPlugin(pluginId: string, loaded: LoadedPlugin[]) { const matches = loaded.filter((plugin) => plugin.id === pluginId) if (matches.length > 1) { @@ -282,7 +196,6 @@ export namespace PluginInstallationTransaction { return withPluginInstallationLock(async () => { const previousDomain = await Config.domainGet("plugins") const previousLockfile = await Lockfile.read() - const previousApprovals = await readApprovals() const previousIncompatible = await IncompatiblePluginStore.read() const currentPlugins = previousDomain.plugin ?? [] const resolveConfiguredPluginId = async (spec: string): Promise => { @@ -299,29 +212,31 @@ export namespace PluginInstallationTransaction { resolvePluginId: resolveConfiguredPluginId, }) - const promoted = await promoteStagingDir(input.resolved) + const nextDomain = { ...previousDomain, plugin: nextConfig.plugins } + const recovery = await PluginInstallationRecovery.begin(input.pluginId, nextDomain, input.resolved) const lockEntry: PluginLockEntry = { ...input.lockEntry, resolved: resolvedPathAfterPromotion(input.resolved), } try { - await Lockfile.write(addEntry(previousLockfile, input.pluginId, lockEntry)) - await Config.domainUpdate("plugins", { ...previousDomain, plugin: nextConfig.plugins } as any, { - mode: "replace-domain", + await recovery.promote() + await Config.domainUpdate("plugins", nextDomain, { mode: "replace-domain" }) + await Storage.transaction(async () => { + await Lockfile.write(addEntry(await Lockfile.read(), input.pluginId, lockEntry)) + if (input.approval) await saveApproval(input.approval) + await IncompatiblePluginStore.write( + IncompatiblePluginStore.withoutPlugin(previousIncompatible, input.pluginId, [ + input.spec, + ...nextConfig.removed, + ]), + ) }) - if (input.approval) await saveApproval(input.approval) - await IncompatiblePluginStore.write( - IncompatiblePluginStore.withoutPlugin(previousIncompatible, input.pluginId, [ - input.spec, - ...nextConfig.removed, - ]), - ) if (input.autoReload !== false) await input.reload() const loaded = await input.getLoaded() const plugin = assertSingleLoadedPlugin(input.pluginId, loaded) - await promoted.cleanup() + await recovery.finish() return plugin } catch (err) { const previousEntry = previousLockfile.plugins[input.pluginId] @@ -329,11 +244,7 @@ export namespace PluginInstallationTransaction { pluginId: input.pluginId, error: err instanceof Error ? err.message : String(err), }) - await Lockfile.write(previousLockfile).catch(() => {}) - await Config.domainUpdate("plugins", previousDomain as any, { mode: "replace-domain" }).catch(() => {}) - await writeApprovals(previousApprovals).catch(() => {}) - await IncompatiblePluginStore.write(previousIncompatible).catch(() => {}) - await promoted.restore().catch(() => {}) + await recovery.rollback() if (input.autoReload !== false) await input.reload().catch(() => {}) if (previousEntry) { await recordEvent({ @@ -361,7 +272,6 @@ export namespace PluginInstallationTransaction { await withPluginInstallationLock(async () => { const previousDomain = await Config.domainGet("plugins") const previousLockfile = await Lockfile.read() - const previousApprovals = await readApprovals() const previousIncompatible = await IncompatiblePluginStore.read() const currentPlugins = previousDomain.plugin ?? [] const recordedIds = new Map() @@ -387,19 +297,20 @@ export namespace PluginInstallationTransaction { } await input.beforeCommit?.() + const recovery = await PluginInstallationRecovery.begin(input.pluginId, nextDomain) try { await Config.domainUpdate("plugins", nextDomain, { mode: "replace-domain" }) - await Lockfile.write(removePluginEntries(previousLockfile, input.pluginId, selected.removed)) - await removeApproval(input.pluginId) - await IncompatiblePluginStore.write( - IncompatiblePluginStore.withoutPlugin(previousIncompatible, input.pluginId, selected.removed), - ) + await Storage.transaction(async () => { + await Lockfile.write(removePluginEntries(await Lockfile.read(), input.pluginId, selected.removed)) + await removeApproval(input.pluginId) + await IncompatiblePluginStore.write( + IncompatiblePluginStore.withoutPlugin(previousIncompatible, input.pluginId, selected.removed), + ) + }) await input.reload() + await recovery.finish() } catch (err) { - await Config.domainUpdate("plugins", previousDomain as any, { mode: "replace-domain" }).catch(() => {}) - await Lockfile.write(previousLockfile).catch(() => {}) - await writeApprovals(previousApprovals).catch(() => {}) - await IncompatiblePluginStore.write(previousIncompatible).catch(() => {}) + await recovery.rollback() await input.reload().catch(() => {}) throw err } @@ -413,20 +324,21 @@ export namespace PluginInstallationTransaction { getLoaded: () => Promise }): Promise { return withPluginInstallationLock(async () => { - const previousApprovals = await readApprovals() const approval = typeof input.approval === "function" ? await input.approval() : input.approval - + const recovery = await PluginInstallationRecovery.begin(input.pluginId) try { await saveApproval(approval) await input.reload() const loaded = await input.getLoaded() - return assertSingleLoadedPlugin(input.pluginId, loaded) + const plugin = assertSingleLoadedPlugin(input.pluginId, loaded) + await recovery.finish() + return plugin } catch (err) { log.warn("plugin approval transaction failed; rolling back", { pluginId: input.pluginId, error: err instanceof Error ? err.message : String(err), }) - await writeApprovals(previousApprovals).catch(() => {}) + await recovery.rollback() await input.reload().catch(() => {}) throw err } diff --git a/packages/plugin-host/src/plugin/local-registry-store.ts b/packages/plugin-host/src/plugin/local-registry-store.ts index ca55e8312..a907fbac1 100644 --- a/packages/plugin-host/src/plugin/local-registry-store.ts +++ b/packages/plugin-host/src/plugin/local-registry-store.ts @@ -1,12 +1,25 @@ +import { Storage } from "@ericsanchezok/synergy-harness/storage/storage" import path from "path" import { Global } from "@ericsanchezok/synergy-harness/global" -export function localRegistryPath(): string { - return path.join(Global.Path.data, "registry", "plugins.json") +export function localRegistryStoreDir(): string { + return path.join(Global.Path.data, "registry") } -export function localRegistryStoreDir(): string { - return path.dirname(localRegistryPath()) +export async function readLocalRegistry(): Promise[]> { + const keys = await Storage.list(["registry", "entries"]) + return (await Storage.readMany>(keys)).filter( + (entry): entry is Record => entry !== undefined, + ) +} + +export async function writeLocalRegistry(entries: Array<{ id: string }>): Promise { + await Storage.transaction(async (tx) => { + const existing = await tx.scan(["registry", "entries"]) + const ids = new Set(entries.map((entry) => entry.id)) + for (const id of existing) if (!ids.has(id)) await tx.remove(["registry", "entries", id]) + for (const entry of entries) await tx.write(["registry", "entries", entry.id], entry) + }) } export function localRegistryArtifactDir(pluginId: string, version: string): string { diff --git a/packages/plugin-host/src/plugin/lockfile.ts b/packages/plugin-host/src/plugin/lockfile.ts index db5820d82..025b22662 100644 --- a/packages/plugin-host/src/plugin/lockfile.ts +++ b/packages/plugin-host/src/plugin/lockfile.ts @@ -1,46 +1,27 @@ -import path from "path" -import fs from "fs/promises" -import os from "os" -import { Global } from "@ericsanchezok/synergy-harness/global" +import { Storage } from "@ericsanchezok/synergy-harness/storage/storage" import { PluginLockfile } from "./lockfile-schema" import type { PluginLockEntry } from "./lockfile-schema" -const EMPTY_LOCKFILE: PluginLockfile = { - version: 2 as const, - plugins: {}, -} - -function lockfilePath() { - return path.join(Global.Path.root, "plugin.lock") -} - -/** - * Read and parse the plugin lockfile. - * Returns an empty lockfile if the file doesn't exist. - */ export async function read(): Promise { - try { - const text = await Bun.file(lockfilePath()).text() - if (!text.trim()) return { ...EMPTY_LOCKFILE } - const parsed = PluginLockfile.parse(JSON.parse(text)) - return parsed - } catch (err: any) { - if (err.code === "ENOENT") return { ...EMPTY_LOCKFILE } - throw err - } + return Storage.snapshot(async (tx) => { + const [metadata] = await tx.readMany>([["plugin-lock", "info"]]) + const plugins: Record = {} + for (const key of await tx.list(["plugin-lock", "entries"])) + plugins[key.at(-1)!] = await tx.read(key) + const document = { version: 2, ...metadata, plugins } + PluginLockfile.loose().parse(document) + return document as PluginLockfile + }) } -/** - * Write the lockfile atomically (write to temp + rename). - */ export async function write(lockfile: PluginLockfile): Promise { - const targetPath = lockfilePath() - // Unique temp name (pid + timestamp + random) so concurrent writers in the same - // millisecond cannot collide on one temp path and fail the rename. - const tmpPath = path.join(os.tmpdir(), `.synergy-plugin-lock-${process.pid}-${Date.now()}-${crypto.randomUUID()}.tmp`) - await Bun.write(tmpPath, JSON.stringify(lockfile, null, 2) + "\n") - await fs.mkdir(path.dirname(targetPath), { recursive: true }) - await fs.rename(tmpPath, targetPath) + const { plugins, ...metadata } = lockfile + await Storage.transaction(async (tx) => { + await tx.write(["plugin-lock", "info"], metadata) + const existing = await tx.scan(["plugin-lock", "entries"]) + for (const id of existing) if (!(id in plugins)) await tx.remove(["plugin-lock", "entries", id]) + for (const [id, entry] of Object.entries(plugins)) await tx.write(["plugin-lock", "entries", id], entry) + }) } /** diff --git a/packages/plugin-host/src/plugin/migration.ts b/packages/plugin-host/src/plugin/migration.ts index 173a8b942..a4ebec1ce 100644 --- a/packages/plugin-host/src/plugin/migration.ts +++ b/packages/plugin-host/src/plugin/migration.ts @@ -1,3 +1,7 @@ +import { Storage } from "@ericsanchezok/synergy-harness/storage/storage" +import { writeLocalRegistry } from "./local-registry-store" +import * as Lockfile from "./lockfile" +import { writeApprovals } from "./consent/approval-store" import fs from "fs/promises" import path from "path" import { pathToFileURL } from "url" @@ -55,7 +59,7 @@ async function migrateApprovalFile( data: string, manifests: Map, ) { - const value = await readJson(path.join(data, "plugin-approvals.json")) + const [value] = await Storage.readMany([["plugin-approvals"]]) const oldApprovals = Array.isArray(value) ? value : Object.values(record(value)) const approvals: PluginApprovalRecord[] = [] for (const value of oldApprovals) { @@ -85,12 +89,11 @@ async function migrateApprovalFile( approvedAt: Number.isFinite(Number(old.approvedAt)) ? Number(old.approvedAt) : Date.now(), }) } - await fs.mkdir(data, { recursive: true }) - await Bun.write(path.join(data, "plugin-approvals.json"), `${JSON.stringify(approvals, null, 2)}\n`) + await Storage.write(["plugin-approvals"], approvals) } export async function migratePluginApprovalsV2(input: { root: string; data: string; cache: string }) { - const rawLock = record(await readJson(path.join(input.root, "plugin.lock"))) + const rawLock = record((await Storage.readMany([["plugin-lock"]]))[0]) const rawPlugins = record(rawLock.plugins) const manifests = new Map() for (const [pluginId, value] of Object.entries(rawPlugins)) { @@ -110,8 +113,7 @@ export async function migratePluginCatalog(input: { progress?: (current: number, total: number) => void }) { const progress = input.progress ?? (() => undefined) - const lockPath = path.join(input.root, "plugin.lock") - const rawLock = record(await readJson(lockPath)) + const rawLock = record((await Storage.readMany([["plugin-lock"]]))[0]) const rawPlugins = record(rawLock.plugins) const next: PluginLockfile = { version: 2, plugins: {} } const incompatible: IncompatiblePluginRecord[] = [] @@ -147,9 +149,8 @@ export async function migratePluginCatalog(input: { } progress(++current, Math.max(1, entries.length)) } - await fs.mkdir(path.dirname(lockPath), { recursive: true }) - await Bun.write(lockPath, `${JSON.stringify(next, null, 2)}\n`) - await IncompatiblePluginStore.write(incompatible, input.data) + await Storage.write(["plugin-lock"], next) + await IncompatiblePluginStore.write(incompatible) await migrateApprovalFile(input.data, manifests) await fs.rm(path.join(input.cache, "plugin"), { recursive: true, force: true }).catch(() => undefined) @@ -175,6 +176,45 @@ const migrations: Migration[] = [ progress(1, 1) }, }, + { + id: "20260914-plugin-transactional-records", + description: "Separate plugin installation, approval, and audit records in authoritative storage", + async up(progress) { + await Storage.transaction(async (tx) => { + const [rawLock, approvals, audit, registry] = await tx.readMany([ + ["plugin-lock"], + ["plugin-approvals"], + ["plugin-audit"], + ["registry", "plugins"], + ]) + if (rawLock !== undefined) await Lockfile.write(rawLock as PluginLockfile) + if (approvals !== undefined) { + if (!Array.isArray(approvals)) throw new Error("Plugin approvals must be an array") + await writeApprovals(approvals as PluginApprovalRecord[]) + } + if (audit !== undefined) { + if (!Array.isArray(audit)) throw new Error("Plugin audit history must be an array") + for (const entry of audit) { + const event = record(entry) + if (typeof event.id !== "string" || typeof event.time !== "number") + throw new Error("Plugin audit event has no stable identity") + await tx.write(["plugin-audit", "events", `${String(event.time).padStart(16, "0")}_${event.id}`], event) + } + } + if (registry !== undefined) { + const entries = Array.isArray(registry) ? registry : record(registry).plugins + if (!Array.isArray(entries) || entries.some((entry) => typeof record(entry).id !== "string")) + throw new Error("Plugin registry entries have no stable identities") + await writeLocalRegistry(entries as Array<{ id: string }>) + await tx.remove(["registry", "plugins"]) + } + await tx.remove(["plugin-lock"]) + await tx.remove(["plugin-approvals"]) + await tx.remove(["plugin-audit"]) + }) + progress(1, 1) + }, + }, ] function pathToFileSpec(directory: string) { diff --git a/packages/plugin-host/src/plugin/routes/plugin-registry-routes.ts b/packages/plugin-host/src/plugin/routes/plugin-registry-routes.ts index 1422536a2..4f3113541 100644 --- a/packages/plugin-host/src/plugin/routes/plugin-registry-routes.ts +++ b/packages/plugin-host/src/plugin/routes/plugin-registry-routes.ts @@ -8,7 +8,7 @@ import { errors } from "@ericsanchezok/synergy-server/server/error" import { checkPathContainment } from "@ericsanchezok/synergy-harness/util/path-contain" import { PluginMarketplaceRegistry } from "../marketplace-registry" import { Log } from "@ericsanchezok/synergy-harness/util/log" -import { localRegistryPath, localRegistryStoreDir } from "../local-registry-store" +import { readLocalRegistry, writeLocalRegistry, localRegistryStoreDir } from "../local-registry-store" const log = Log.create({ service: "plugin.registry.route" }) const OFFICIAL_REGISTRY_UNAVAILABLE_MESSAGE = "Official plugin registry temporarily unavailable" @@ -172,28 +172,8 @@ const PublishInput = RegistryPluginEntry.omit({ createdAt: true, updatedAt: true // ── Helpers ── -function registryPath(): string { - return localRegistryPath() -} - -function missingFileError(err: unknown): boolean { - if (!(err instanceof Error)) return false - return (err as NodeJS.ErrnoException).code === "ENOENT" -} - async function loadRegistry(): Promise { - const file = Bun.file(registryPath()) - try { - const exists = await file.exists() - if (!exists) return [] - } catch (err) { - if (missingFileError(err)) return [] - throw err - } - const text = await file.text() - const parsed = JSON.parse(text) - const plugins = Array.isArray(parsed) ? parsed : parsed && Array.isArray(parsed.plugins) ? parsed.plugins : [] - return z.array(RegistryPluginEntry).parse(plugins.map(normalizeLegacyRegistryEntry)) + return z.array(RegistryPluginEntry).parse((await readLocalRegistry()).map(normalizeLegacyRegistryEntry)) } function normalizeLegacyRegistryEntry(value: unknown): unknown { @@ -300,12 +280,7 @@ function mergeSummaries( } async function saveRegistry(plugins: RegistryPluginEntry[]): Promise { - const realPath = registryPath() - const tmpPath = realPath + ".tmp" - const dir = path.dirname(realPath) - fs.mkdirSync(dir, { recursive: true }) - await Bun.write(tmpPath, JSON.stringify({ plugins }, null, 2)) - fs.renameSync(tmpPath, realPath) + await writeLocalRegistry(plugins) } function isLoopbackHost(input: string): boolean { @@ -718,3 +693,7 @@ export const RegistryRoute = new Hono() return c.json(created) }, ) + +function missingFileError(error: unknown) { + return error !== null && typeof error === "object" && "code" in error && error.code === "ENOENT" +} diff --git a/packages/plugin-host/src/plugin/startup.ts b/packages/plugin-host/src/plugin/startup.ts index 757c75ab8..4949275de 100644 --- a/packages/plugin-host/src/plugin/startup.ts +++ b/packages/plugin-host/src/plugin/startup.ts @@ -1,3 +1,4 @@ +import { StorageRecovery } from "@ericsanchezok/synergy-harness/storage/recovery" import { ScopeStartup } from "@ericsanchezok/synergy-harness/scope/startup" import { Plugin } from "." @@ -8,7 +9,13 @@ import { Plugin } from "." * right after the listeners, before session recovery. Registered through * src/product-registration.ts. */ +async function recoverInstallations() { + const { PluginInstallationRecovery } = await import("./installation-recovery") + await PluginInstallationRecovery.recover() +} + export function registerPluginStartup() { + StorageRecovery.register("plugin-installation", recoverInstallations) ScopeStartup.register({ name: "plugin-activate", phase: "core", diff --git a/packages/plugin-host/src/plugin/trust.ts b/packages/plugin-host/src/plugin/trust.ts index 5ec42a458..c189d11f6 100644 --- a/packages/plugin-host/src/plugin/trust.ts +++ b/packages/plugin-host/src/plugin/trust.ts @@ -1,5 +1,4 @@ import path from "path" -import fs from "fs" import type { PluginManifest } from "@ericsanchezok/synergy-plugin" import { defaultPluginTrustDecision, @@ -29,10 +28,9 @@ export { } from "@ericsanchezok/synergy-util/plugin-policy" export type { PluginSource, PluginTrustDecision, TrustTier } from "@ericsanchezok/synergy-util/plugin-policy" -function sourceFromLockfile(pluginDir: string): PluginSource | undefined { +async function sourceFromLockfile(pluginDir: string): Promise { try { - const lockfilePath = path.join(Global.Path.root, "plugin.lock") - const parsed = JSON.parse(fs.readFileSync(lockfilePath, "utf-8")) + const parsed = await Lockfile.read() const entries = Object.values(parsed?.plugins ?? {}) as Array<{ spec?: string source?: PluginSource @@ -53,8 +51,8 @@ function sourceFromLockfile(pluginDir: string): PluginSource | undefined { * Derive the plugin source classification from its lockfile entry and directory path. * Lockfile specs win because cache paths alone cannot distinguish npm from git/url archives. */ -export function derivePluginSource(pluginDir: string): PluginSource { - const fromLockfile = sourceFromLockfile(pluginDir) +export async function derivePluginSource(pluginDir: string): Promise { + const fromLockfile = await sourceFromLockfile(pluginDir) if (fromLockfile) return fromLockfile const cacheRoot = Global.Path.cache @@ -121,7 +119,7 @@ export interface InstalledPluginPolicyDecision extends PluginPolicyDecision { export async function resolveInstalledPluginPolicy( input: InstalledPluginPolicyInput, ): Promise { - const source = input.source ?? derivePluginSource(input.pluginDir) + const source = input.source ?? (await derivePluginSource(input.pluginDir)) const [approval, integrity] = await Promise.all([ input.approval === undefined ? getApproval(input.pluginId, input.manifest) : Promise.resolve(input.approval), input.verifiedIntegrity === undefined ? resolvePluginIntegrity(input.pluginDir) : Promise.resolve(undefined), diff --git a/packages/plugin-host/test/plugin/approval-store.test.ts b/packages/plugin-host/test/plugin/approval-store.test.ts index 863a81879..aa351ea89 100644 --- a/packages/plugin-host/test/plugin/approval-store.test.ts +++ b/packages/plugin-host/test/plugin/approval-store.test.ts @@ -1,3 +1,4 @@ +import { Storage } from "@ericsanchezok/synergy-harness/storage/storage" import { describe, expect, test } from "bun:test" import fs from "fs/promises" import path from "path" @@ -47,7 +48,8 @@ describe("approval store concurrent read-modify-write", () => { for (const id of ids) expect(present.has(id)).toBe(true) expect(present.has(removedId)).toBe(false) - JSON.parse(await Bun.file(storeFile()).text()) + expect(await Storage.read(["plugin-approvals", "records", ids[0]!])).toMatchObject({ pluginId: ids[0] }) + expect(await Bun.file(storeFile()).exists()).toBe(false) expect(await tempResidue()).toEqual([]) }) }) diff --git a/packages/plugin-host/test/plugin/doctor.test.ts b/packages/plugin-host/test/plugin/doctor.test.ts index 3e7d5164c..09e2502dd 100644 --- a/packages/plugin-host/test/plugin/doctor.test.ts +++ b/packages/plugin-host/test/plugin/doctor.test.ts @@ -1,3 +1,4 @@ +import { Storage } from "@ericsanchezok/synergy-harness/storage/storage" import { expect, test } from "bun:test" import fs from "node:fs/promises" import path from "node:path" @@ -12,10 +13,8 @@ test("doctor reports without mutation, then repairs duplicate config, stale lock await using tmp = await tmpdir({}) const domain = await Config.domainGet("plugins") const lock = await Lockfile.read() - const statePath = path.join(Global.Path.data, "plugin-runtime-state.json") - const state = await Bun.file(statePath) - .text() - .catch(() => undefined) + const statePath = ["plugin-runtime-state"] + const [state] = await Storage.readMany([statePath]) const orphan = path.join(Global.Path.cache, "plugin-archives", `doctor-${crypto.randomUUID()}`) const a = path.join(tmp.path, "a") const b = path.join(tmp.path, "b") @@ -39,7 +38,7 @@ test("doctor reports without mutation, then repairs duplicate config, stale lock manifestHash: "hash", } await Lockfile.write({ version: 2, plugins: { duplicate: entry, stale: { ...entry, spec: "file:///unused" } } }) - await Bun.write(statePath, JSON.stringify([runtime, { pluginId: "missing-entry", pluginDir: b }, null])) + await Storage.write(statePath, [runtime, { pluginId: "missing-entry", pluginDir: b }, null]) const observed = await doctor() expect(observed.changed).toBe(false) expect(observed.issues.map((issue) => issue.type)).toEqual( @@ -58,14 +57,14 @@ test("doctor reports without mutation, then repairs duplicate config, stale lock expect(repaired.changed).toBe(true) expect((await Config.domainGet("plugins")).plugin).toEqual([specB, missingSpec]) expect(Object.keys((await Lockfile.read()).plugins)).toEqual(["duplicate"]) - expect(await Bun.file(statePath).json()).toEqual([runtime]) + expect(await Storage.read>(statePath)).toEqual([runtime]) expect(await fs.stat(orphan).catch(() => undefined)).toBeUndefined() expect((await doctor()).issues.map((issue) => issue.type)).toEqual(["unresolved_config_spec"]) } finally { await Config.domainUpdate("plugins", domain, { mode: "replace-domain" }) await Lockfile.write(lock) - if (state === undefined) await fs.rm(statePath, { force: true }) - else await Bun.write(statePath, state) + if (state === undefined) await Storage.remove(statePath) + else await Storage.write(statePath, state) await fs.rm(orphan, { recursive: true, force: true }) } }) diff --git a/packages/plugin-host/test/plugin/incompatible-store.test.ts b/packages/plugin-host/test/plugin/incompatible-store.test.ts index eb7e0fb4f..e54f630b7 100644 --- a/packages/plugin-host/test/plugin/incompatible-store.test.ts +++ b/packages/plugin-host/test/plugin/incompatible-store.test.ts @@ -1,10 +1,12 @@ -import { describe, expect, test } from "bun:test" +import { Storage } from "@ericsanchezok/synergy-harness/storage/storage" +import { beforeEach, describe, expect, test } from "bun:test" import fs from "fs/promises" import path from "path" import { IncompatiblePluginStore } from "../../src/plugin/incompatible-store" import { tmpdir } from "@ericsanchezok/synergy-harness/test/support/fixture" describe("incompatible plugin records", () => { + beforeEach(() => Storage.remove(["plugin-incompatible"])) test("round-trips records and removes all records owned by a plugin", async () => { await using tmp = await tmpdir() const data = path.join(tmp.path, "data") @@ -14,8 +16,8 @@ describe("incompatible plugin records", () => { { pluginId: "other", spec: "file:///other.tgz", reason: "reinstallRequired" as const }, ] - await IncompatiblePluginStore.write(records, data) - expect(await IncompatiblePluginStore.read(data)).toEqual(records) + await IncompatiblePluginStore.write(records) + expect(await IncompatiblePluginStore.read()).toEqual(records) expect(IncompatiblePluginStore.withoutPlugin(records, "focus")).toEqual([records[2]]) expect(IncompatiblePluginStore.withoutPlugin(records, "unknown", ["file:///other.tgz"])).toEqual([ records[0], @@ -26,9 +28,9 @@ describe("incompatible plugin records", () => { test("returns an empty catalog only when the file is missing and rejects corrupt data", async () => { await using tmp = await tmpdir() const data = path.join(tmp.path, "data") - expect(await IncompatiblePluginStore.read(data)).toEqual([]) - await Bun.write(path.join(data, "plugin-incompatible.json"), "not json") - await expect(IncompatiblePluginStore.read(data)).rejects.toThrow() + expect(await IncompatiblePluginStore.read()).toEqual([]) + await Storage.write(["plugin-incompatible"], { invalid: true }) + await expect(IncompatiblePluginStore.read()).rejects.toThrow() }) test("concurrent writes land exactly one complete batch without temp residue", async () => { @@ -38,12 +40,11 @@ describe("incompatible plugin records", () => { { pluginId: `race-${index}`, reason: "reinstallRequired" as const }, ]) - await Promise.all(batches.map((batch) => IncompatiblePluginStore.write(batch, data))) + await Promise.all(batches.map((batch) => IncompatiblePluginStore.write(batch))) - const final = await IncompatiblePluginStore.read(data) + const final = await IncompatiblePluginStore.read() expect(final).toHaveLength(1) expect(batches.some((batch) => batch[0]!.pluginId === final[0]?.pluginId)).toBe(true) - const entries = await fs.readdir(data) - expect(entries.filter((name) => name.includes(".tmp"))).toEqual([]) + expect(await Storage.list(["plugin-incompatible"])).toEqual([]) }) }) diff --git a/packages/plugin-host/test/plugin/installation-recovery.test.ts b/packages/plugin-host/test/plugin/installation-recovery.test.ts new file mode 100644 index 000000000..8394bc8ca --- /dev/null +++ b/packages/plugin-host/test/plugin/installation-recovery.test.ts @@ -0,0 +1,48 @@ +import { expect, spyOn, test } from "bun:test" +import { Storage } from "@ericsanchezok/synergy-harness/storage/storage" +import { Config } from "@ericsanchezok/synergy-harness/config/config" +import { PluginInstallationRecovery } from "../../src/plugin/installation-recovery" +import * as Lockfile from "../../src/plugin/lockfile" + +test("restart recovery restores an interrupted installation without discarding unrelated metadata", async () => { + const before = await Config.domainGet("plugins") + const lockfile = await Lockfile.read() + const next = { ...before, plugin: [...(before.plugin ?? []), "npm:interrupted"] } + try { + await PluginInstallationRecovery.begin("interrupted", next) + await Config.domainUpdate("plugins", next, { mode: "replace-domain" }) + await Storage.write(["plugin-lock", "info"], { version: 2, unrelatedOwner: { retained: true } }) + await PluginInstallationRecovery.recover() + expect(await Config.domainGet("plugins")).toEqual(before) + expect(await Storage.read<{ unrelatedOwner: { retained: boolean } }>(["plugin-lock", "info"])).toMatchObject({ + unrelatedOwner: { retained: true }, + }) + expect(await Storage.list(["plugin-install-intents"])).toEqual([]) + await PluginInstallationRecovery.recover() + } finally { + await Config.domainUpdate("plugins", before, { mode: "replace-domain" }) + await Lockfile.write(lockfile) + } +}) + +test("completed installation cleanup resumes without rolling back committed configuration", async () => { + const before = await Config.domainGet("plugins") + const next = { ...before, plugin: [...(before.plugin ?? []), "npm:completed"] } + try { + const intent = await PluginInstallationRecovery.begin("completed", next) + await Config.domainUpdate("plugins", next, { mode: "replace-domain" }) + const remove = Storage.remove + { + using failure = spyOn(Storage, "remove").mockImplementation(async (key) => { + if (key[0] === "plugin-install-intents") throw new Error("cleanup interrupted") + return remove(key) + }) + await expect(intent.finish()).rejects.toThrow("cleanup interrupted") + } + await PluginInstallationRecovery.recover() + expect(await Config.domainGet("plugins")).toEqual(next) + expect(await Storage.list(["plugin-install-intents"])).toEqual([]) + } finally { + await Config.domainUpdate("plugins", before, { mode: "replace-domain" }) + } +}) diff --git a/packages/plugin-host/test/plugin/installation-transaction.test.ts b/packages/plugin-host/test/plugin/installation-transaction.test.ts index 562f98aa8..7d8b3dba1 100644 --- a/packages/plugin-host/test/plugin/installation-transaction.test.ts +++ b/packages/plugin-host/test/plugin/installation-transaction.test.ts @@ -179,7 +179,7 @@ describe("plugin uninstall transaction", () => { expect(await Config.domainGet("plugins")).toEqual(configured) expect(await Lockfile.read()).toEqual(locked) - expect(await readApprovals()).toEqual(approvals) + expect(await readApprovals()).toEqual(approvals.toSorted((a, b) => a.pluginId.localeCompare(b.pluginId))) expect(await IncompatiblePluginStore.read()).toEqual(incompatible) } finally { await Config.domainUpdate("plugins", previousDomain, { mode: "replace-domain" }) diff --git a/packages/plugin-host/test/plugin/migration.test.ts b/packages/plugin-host/test/plugin/migration.test.ts index 87de911e5..86a23f39d 100644 --- a/packages/plugin-host/test/plugin/migration.test.ts +++ b/packages/plugin-host/test/plugin/migration.test.ts @@ -1,3 +1,5 @@ +import { Storage } from "@ericsanchezok/synergy-harness/storage/storage" +import { StorageBootstrap } from "@ericsanchezok/synergy-harness/storage/bootstrap" import { describe, expect, test } from "bun:test" import fs from "fs/promises" import path from "path" @@ -68,31 +70,42 @@ describe("plugin catalog migration", () => { await fs.mkdir(path.join(cache, "plugin-market"), { recursive: true }) await Bun.write(path.join(cache, "plugin-market", "registry.json"), "{}") - await migratePluginCatalog({ root, data, cache }) + const prepared = await StorageBootstrap.prepare({ root }) + try { + await Storage.provide({ store: prepared.store, artifactDirectory: data }, async () => { + await migratePluginCatalog({ root, data, cache }) - const lock = JSON.parse(await Bun.file(path.join(root, "plugin.lock")).text()) - expect(lock.version).toBe(2) - expect(lock.plugins["migrated-plugin"]).toMatchObject({ - version: "2.0.0", - apiVersion: "4.0", - generation: "migrated-generation", - }) - const incompatible = JSON.parse(await Bun.file(path.join(data, "plugin-incompatible.json")).text()) - expect(incompatible).toEqual([{ pluginId: "incompatible", spec: "file:old", reason: "reinstallRequired" }]) - const approvals = JSON.parse(await Bun.file(path.join(data, "plugin-approvals.json")).text()) - expect(approvals[0]).toMatchObject({ - schemaVersion: 2, - pluginId: "migrated-plugin", - approvedCapabilities: [], - }) - expect(approvals).toHaveLength(1) - expect(verifyApproval(approvals[0], manifest)).toBe(true) - expect(JSON.parse(await Bun.file(settingsPath).text())).toEqual({ "migrated-plugin": { enabled: true } }) - expect(await Bun.file(path.join(cache, "plugin", "temporary")).exists()).toBe(false) - expect(await Bun.file(path.join(cache, "plugin-market", "registry.json")).exists()).toBe(false) + const lock = await Storage.read<{ version: number; plugins: Record }>(["plugin-lock"]) + expect(lock.version).toBe(2) + expect(lock.plugins["migrated-plugin"]).toMatchObject({ + version: "2.0.0", + apiVersion: "4.0", + generation: "migrated-generation", + }) + const incompatible = await Storage.read(["plugin-incompatible"]) + expect(incompatible).toEqual([{ pluginId: "incompatible", spec: "file:old", reason: "reinstallRequired" }]) + const approvals = await Storage.read([ + "plugin-approvals", + ]) + expect(approvals[0]).toMatchObject({ + schemaVersion: 2, + pluginId: "migrated-plugin", + approvedCapabilities: [], + }) + expect(approvals).toHaveLength(1) + expect(verifyApproval(approvals[0], manifest)).toBe(true) + expect(JSON.parse(await Bun.file(settingsPath).text())).toEqual({ "migrated-plugin": { enabled: true } }) + expect(await Bun.file(path.join(cache, "plugin", "temporary")).exists()).toBe(false) + expect(await Bun.file(path.join(cache, "plugin-market", "registry.json")).exists()).toBe(false) - await migratePluginCatalog({ root, data, cache }) - const rerunApprovals = JSON.parse(await Bun.file(path.join(data, "plugin-approvals.json")).text()) - expect(rerunApprovals).toEqual(approvals) + await migratePluginCatalog({ root, data, cache }) + const rerunApprovals = await Storage.read< + import("../../src/plugin/consent/approval-store").PluginApprovalRecord[] + >(["plugin-approvals"]) + expect(rerunApprovals).toEqual(approvals) + }) + } finally { + await prepared.store.close() + } }) }) diff --git a/packages/plugin-host/test/plugin/tool-invocation.test.ts b/packages/plugin-host/test/plugin/tool-invocation.test.ts index 29a389d6c..2d36cdefc 100644 --- a/packages/plugin-host/test/plugin/tool-invocation.test.ts +++ b/packages/plugin-host/test/plugin/tool-invocation.test.ts @@ -166,7 +166,7 @@ test("plugin host cancellation reaches an active tool and persists interrupted e test("plugin host preserves recording failures when durable output cannot be written", async () => { await withSession(async ({ session, assistantID }) => { - const artifacts = path.join(Global.Path.data, "sessions", session.scope.id, session.id, "rollout", "artifacts") + const artifacts = path.join(Global.Path.data, "sessions", session.scope.id, session.id, "rollout", "blobs") const backup = artifacts + "-preserved" let blocked = false await ToolRegistry.register( diff --git a/packages/product-runtime/schema/config.schema.json b/packages/product-runtime/schema/config.schema.json index dcf3c7e99..9af058aba 100644 --- a/packages/product-runtime/schema/config.schema.json +++ b/packages/product-runtime/schema/config.schema.json @@ -10,6 +10,32 @@ "type": "string", "enum": ["DEBUG", "INFO", "WARN", "ERROR"] }, + "storage": { + "description": "Global authoritative storage; backend changes require an explicit storage migration", + "anyOf": [ + { + "type": "object", + "properties": { + "backend": { "type": "string", "const": "sqlite" }, + "namespace": { "type": "string", "pattern": "^[a-zA-Z0-9_-]{1,128}$" }, + "filename": { "type": "string", "minLength": 1 } + }, + "required": ["backend"], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "backend": { "type": "string", "const": "postgres" }, + "namespace": { "type": "string", "pattern": "^[a-zA-Z0-9_-]{1,128}$" }, + "connectionEnv": { "type": "string", "pattern": "^[a-zA-Z_][a-zA-Z0-9_]*$" }, + "maxConnections": { "type": "integer", "minimum": 2, "maximum": 64 } + }, + "required": ["backend", "namespace", "connectionEnv"], + "additionalProperties": false + } + ] + }, "server": { "description": "Server configuration for synergy serve and web commands", "ref": "ServerConfig", diff --git a/packages/product-runtime/src/cli-commands.ts b/packages/product-runtime/src/cli-commands.ts index 21ba3b81b..febeb7f9a 100644 --- a/packages/product-runtime/src/cli-commands.ts +++ b/packages/product-runtime/src/cli-commands.ts @@ -19,6 +19,7 @@ export const productCommands: CommandEntry[] = [ }, { command: "debug", + storage: "maintenance", describe: "debugging and troubleshooting tools", load: async () => (await import("@ericsanchezok/synergy-cli/cli/cmd/debug")).createDebugCommand([ @@ -28,12 +29,14 @@ export const productCommands: CommandEntry[] = [ }, { command: "stats", + storage: "maintenance", describe: "show token usage and cost statistics", load: async () => (await import("@ericsanchezok/synergy-workbench/stats/cli/stats")).StatsCommand as unknown as CommandModule, }, { command: "mcp", + storage: "maintenance", describe: "manage MCP (Model Context Protocol) servers", load: async () => (await import("@ericsanchezok/synergy-agent-integrations/mcp/cli/mcp")).McpCommand as unknown as CommandModule, @@ -42,7 +45,9 @@ export const productCommands: CommandEntry[] = [ command: "acp", describe: "start ACP (Agent Client Protocol) server", load: async () => - (await import("@ericsanchezok/synergy-agent-integrations/acp/cli/acp")).AcpCommand as unknown as CommandModule, + (await import("@ericsanchezok/synergy-agent-integrations/acp/cli/acp")).createAcpCommand( + (await import("./server/runtime-handle")).ProductRuntimeHandle.open, + ) as unknown as CommandModule, }, { command: "web", @@ -51,6 +56,7 @@ export const productCommands: CommandEntry[] = [ }, { command: "channel", + storage: "maintenance", describe: "manage messaging channels", load: async () => (await import("@ericsanchezok/synergy-connections/channel/cli/channel")) @@ -58,18 +64,21 @@ export const productCommands: CommandEntry[] = [ }, { command: "holos", + storage: "maintenance", describe: "manage Holos identity and runtime", load: async () => (await import("@ericsanchezok/synergy-connections/holos/cli/holos")).HolosCommand as unknown as CommandModule, }, { command: "library", + storage: "maintenance", describe: "manage library memory and learning", load: async () => (await import("@ericsanchezok/synergy-library/cli/library")).LibraryCommand as unknown as CommandModule, }, { command: "embed", + storage: "maintenance", describe: "manage the local embedding model", load: async () => (await import("@ericsanchezok/synergy-library/cli/embed")).EmbedCommand as unknown as CommandModule, @@ -98,12 +107,14 @@ export const productCommands: CommandEntry[] = [ }, { command: "browser", + storage: "maintenance", describe: "diagnose and install Chromium for Browser tools", load: async () => (await import("@ericsanchezok/synergy-browser-runtime/cli/browser")).BrowserCommand as unknown as CommandModule, }, { command: "plugin", + storage: "maintenance", describe: "install, remove, update, and inspect plugins", load: async () => (await import("@ericsanchezok/synergy-plugin-host/plugin/cli/plugin")).PluginCommand as unknown as CommandModule, diff --git a/packages/product-runtime/src/cli/data/merge.ts b/packages/product-runtime/src/cli/data/merge.ts index 57119c361..cd4b0c9e5 100644 --- a/packages/product-runtime/src/cli/data/merge.ts +++ b/packages/product-runtime/src/cli/data/merge.ts @@ -4,7 +4,7 @@ import { mergeLibraryDB, type LibraryConflictStrategy, } from "@ericsanchezok/synergy-library/cli/data" -import { SnapshotArchive } from "@ericsanchezok/synergy-harness/session/snapshot-archive" +import { DataTransfer } from "./transfer" import fs from "fs/promises" import path from "path" import os from "os" @@ -19,7 +19,6 @@ import { shortenPath, dirExists, copyDirSkipExisting, - invalidatePendingRolloutLedger, dataRoot, } from "@ericsanchezok/synergy-cli/cli/cmd/data/shared" @@ -174,7 +173,7 @@ export const DataMergeCommand = cmd({ } } - await using homes = await SnapshotArchive.lockHomes([source.dataDir, targetRoot]) + await using homes = await DataTransfer.lockHomes([source.dataDir, targetRoot]) // Step 4: Execute merge UI.empty() @@ -213,18 +212,20 @@ export const DataMergeCommand = cmd({ spinner.start(`Merging ${subdir}/...`) try { - if (subdir === "data") await SnapshotArchive.merge(src, dst) - const result = await copyDirSkipExisting( - src, - dst, - (p) => { - const pct = Math.round(((p.copied + p.skipped) / p.total) * 100) - spinner.message(`Merging ${subdir}/ ${pct}% — ${shortenPath(p.currentFile)}`) - }, - undefined, - undefined, - archiveExclusions(subdir), - ) + const result = + subdir === "data" + ? await DataTransfer.merge(source.dataDir, targetRoot) + : await copyDirSkipExisting( + src, + dst, + (p) => { + const pct = Math.round(((p.copied + p.skipped) / p.total) * 100) + spinner.message(`Merging ${subdir}/ ${pct}% — ${shortenPath(p.currentFile)}`) + }, + undefined, + undefined, + archiveExclusions(subdir), + ) const skippedNote = result.skipped > 0 ? ` (${result.skipped} existing files kept)` : "" spinner.stop(`Merged ${subdir}/${skippedNote}`) } catch (e) { @@ -233,13 +234,11 @@ export const DataMergeCommand = cmd({ } } } - // Imported owner trees can hold journals the target ledger never - // listed, so force one exhaustive recovery on the next startup. - await invalidatePendingRolloutLedger(path.join(targetRoot, "data")) // Report UI.empty() if (errors.length > 0) { + process.exitCode = 1 prompts.log.warn("Merge completed with errors:") for (const err of errors) prompts.log.error(` ${err}`) } else { diff --git a/packages/product-runtime/src/cli/data/move.ts b/packages/product-runtime/src/cli/data/move.ts index d2d61a79f..083dd3054 100644 --- a/packages/product-runtime/src/cli/data/move.ts +++ b/packages/product-runtime/src/cli/data/move.ts @@ -4,7 +4,7 @@ import { mergeLibraryDB, type LibraryConflictStrategy, } from "@ericsanchezok/synergy-library/cli/data" -import { SnapshotArchive } from "@ericsanchezok/synergy-harness/session/snapshot-archive" +import { DataTransfer } from "./transfer" import fs from "fs/promises" import path from "path" import os from "os" @@ -22,7 +22,6 @@ import { checkDiskSpace, isDirEmpty, copyDirSkipExisting, - invalidatePendingRolloutLedger, updateShellProfile, dataRoot, } from "@ericsanchezok/synergy-cli/cli/cmd/data/shared" @@ -158,7 +157,7 @@ export async function executeMove(opts: MoveOptions) { return } - await using homes = await SnapshotArchive.lockHomes([sourceRoot, targetPath]) + await using homes = await DataTransfer.lockHomes([sourceRoot, targetPath]) // Step 5: Handle library.db if core is selected let libraryStrategy: LibraryConflictStrategy = "skip" @@ -248,18 +247,20 @@ export async function executeMove(opts: MoveOptions) { spinner.start(`Moving ${subdir}/ (${formatSize(catSize)})...`) try { - if (subdir === "data") await SnapshotArchive.merge(src, dst) - const result = await copyDirSkipExisting( - src, - dst, - (p) => { - const pct = Math.round(((p.copied + p.skipped) / p.total) * 100) - spinner.message(`Moving ${subdir}/ ${pct}% — ${shortenPath(p.currentFile)}`) - }, - undefined, - undefined, - archiveExclusions(subdir), - ) + const result = + subdir === "data" + ? await DataTransfer.merge(sourceRoot, targetPath) + : await copyDirSkipExisting( + src, + dst, + (p) => { + const pct = Math.round(((p.copied + p.skipped) / p.total) * 100) + spinner.message(`Moving ${subdir}/ ${pct}% — ${shortenPath(p.currentFile)}`) + }, + undefined, + undefined, + archiveExclusions(subdir), + ) const skippedNote = result.skipped > 0 ? ` (${result.skipped} existing files kept)` : "" spinner.stop(`Moved ${subdir}/${skippedNote}`) } catch (e) { @@ -268,9 +269,6 @@ export async function executeMove(opts: MoveOptions) { } } } - // Copied owner trees can hold journals the target ledger never listed, - // so force one exhaustive recovery on the next startup. - await invalidatePendingRolloutLedger(path.join(targetPath, "data")) // Step 7: Write marker if (errors.length === 0) { @@ -294,6 +292,7 @@ export async function executeMove(opts: MoveOptions) { // Report UI.empty() if (errors.length > 0) { + process.exitCode = 1 prompts.log.warn("Move completed with errors:") for (const err of errors) prompts.log.error(` ${err}`) prompts.log.info("Original data preserved at " + shortenPath(sourceRoot)) diff --git a/packages/product-runtime/src/cli/data/pack.ts b/packages/product-runtime/src/cli/data/pack.ts index eab0599fc..172e2eeb2 100644 --- a/packages/product-runtime/src/cli/data/pack.ts +++ b/packages/product-runtime/src/cli/data/pack.ts @@ -1,5 +1,5 @@ import { getLibraryInfo, resolveLibraryDB } from "@ericsanchezok/synergy-library/cli/data" -import { SnapshotArchive } from "@ericsanchezok/synergy-harness/session/snapshot-archive" +import { DataTransfer } from "./transfer" import fs from "fs/promises" import path from "path" import os from "os" @@ -99,6 +99,7 @@ export const DataPackCommand = cmd({ spinner.stop("Packing failed", 1) prompts.log.error(`Failed to pack: ${e instanceof Error ? e.message : String(e)}`) prompts.outro("Failed") + process.exitCode = 1 return } @@ -107,7 +108,7 @@ export const DataPackCommand = cmd({ }) export async function createDataArchive(root: string, output: string, directories: string[], manifest: unknown) { - await using homes = await SnapshotArchive.lockHomes([root]) + await using homes = await DataTransfer.lockHomes([root]) const stage = await fs.mkdtemp(path.join(os.tmpdir(), "synergy-pack-")) try { await Bun.write(path.join(stage, "manifest.json"), JSON.stringify(manifest, null, 2)) @@ -123,8 +124,8 @@ export async function createDataArchive(root: string, output: string, directorie ) if (!exists) continue const destination = path.join(stage, directory) - if (directory === "data") await SnapshotArchive.merge(source, destination) - await copyDirSkipExisting(source, destination, undefined, undefined, undefined, archiveExclusions(directory)) + if (directory === "data") await DataTransfer.pack(root, destination) + else await copyDirSkipExisting(source, destination, undefined, undefined, undefined, archiveExclusions(directory)) included.push(directory) } const zip = Bun.which("zip") @@ -135,7 +136,7 @@ export async function createDataArchive(root: string, output: string, directorie `.synergy-pack-${crypto.randomUUID()}${zip ? ".zip" : ".tar.gz"}`, ) try { - const command = zip ? [zip, "-q", "-r", temporary, ...included] : ["tar", "-czf", temporary, ...included] + const command = zip ? [zip, "-q", "-r", "-y", temporary, ...included] : ["tar", "-czf", temporary, ...included] const child = Bun.spawn(command, { cwd: stage, stdout: "ignore", stderr: "pipe" }) const errors = new Response(child.stderr).text() if ((await child.exited) !== 0) throw new Error(`Archive creation failed: ${await errors}`) diff --git a/packages/product-runtime/src/cli/data/transfer.ts b/packages/product-runtime/src/cli/data/transfer.ts new file mode 100644 index 000000000..3a76fe2b4 --- /dev/null +++ b/packages/product-runtime/src/cli/data/transfer.ts @@ -0,0 +1,134 @@ +import fs from "node:fs/promises" +import path from "node:path" +import { fileURLToPath } from "node:url" +import { existsSync } from "node:fs" +import { randomUUID } from "node:crypto" +import { StorageBootstrap } from "@ericsanchezok/synergy-harness/storage/bootstrap" +import { StoragePortable, type StorageEntry } from "@ericsanchezok/synergy-harness/storage/portable" +import { legacyRecordKey } from "@ericsanchezok/synergy-harness/storage/legacy-import" +import { Storage } from "@ericsanchezok/synergy-harness/storage/storage" +import { Session } from "@ericsanchezok/synergy-harness/session" +import { SnapshotArchive } from "@ericsanchezok/synergy-harness/session/snapshot-archive" +import { + archiveExclusions, + copyDirSkipExisting, + type CopyProgress, +} from "@ericsanchezok/synergy-cli/cli/cmd/data/shared" + +const derived = new Set([ + "session_index", + "endpoint_session", + "sessions_page_index", + "session_child_index", + "session_nav_v2", +]) +const local = new Set(["storage_meta", "storage_import", "storage_import_files", "storage_staging", "storage_transfer"]) + +export namespace DataTransfer { + export async function lockHomes(roots: string[]) { + for (const root of [...new Set(roots.map((root) => path.resolve(root)))].sort()) { + const entry = fileURLToPath(new URL("../../index.ts", import.meta.url)) + const child = Bun.spawn({ + cmd: existsSync(entry) + ? [process.execPath, "run", entry, "__storage-maintenance-runner"] + : [process.execPath, "__storage-maintenance-runner"], + env: { ...process.env, SYNERGY_HOME: path.dirname(root), SYNERGY_MAINTENANCE_ROOT: root }, + stdout: "ignore", + stderr: "pipe", + }) + const error = new Response(child.stderr).text() + const code = await child.exited + if (code !== 0) throw new Error(`Data upgrade failed: ${await error}`) + await error + } + return SnapshotArchive.lockHomes(roots) + } + + function exclude(relative: string, skipped: ReadonlySet = new Set()) { + const segments = relative.split(path.sep) + if (archiveExclusions("data").includes(segments[0])) return true + if (segments[0] === "sessions" && skipped.has(segments[2] ?? "")) return true + return Boolean(legacyRecordKey(segments.join("/"))) + } + + export async function pack(sourceRoot: string, destination: string) { + const handle = await StorageBootstrap.inspect(sourceRoot) + if (!handle) throw new Error("Source storage has not been initialized") + try { + await StoragePortable.exportFile(handle.store, path.join(destination, "agent-records.ndjson")) + await SnapshotArchive.merge(handle.artifactDirectory, destination, { metadata: false }) + return await copyDirSkipExisting(handle.artifactDirectory, destination, undefined, undefined, undefined, exclude) + } finally { + await handle.store.close() + } + } + + export async function merge(sourceRoot: string, targetRoot: string, progress?: (progress: CopyProgress) => void) { + const source = await StorageBootstrap.inspect(sourceRoot) + if (!source) throw new Error("Source storage has not been initialized") + let target: StorageBootstrap.Prepared | undefined + try { + target = await StorageBootstrap.prepare({ root: targetRoot }) + const skipped = new Set() + for (const scopeID of await target.store.scan(["sessions"])) + for (const id of await target.store.scan(["sessions", scopeID])) skipped.add(id) + for (const scopeID of await target.store.scan(["snapshot-v2"])) + for (const id of await target.store.scan(["snapshot-v2", scopeID, "owners"])) skipped.add(id) + const id = randomUUID() + const backup = path.join(targetRoot, "data", "storage", "transfers", id) + // Retain the entire source, including skipped aggregates, before move can remove its home. + await pack(sourceRoot, path.join(backup, "data")) + const copied = await copyDirSkipExisting( + source.artifactDirectory, + path.join(targetRoot, "data"), + progress, + undefined, + undefined, + (relative) => exclude(relative, skipped), + ) + await SnapshotArchive.merge(source.artifactDirectory, path.join(targetRoot, "data"), { metadata: false, skipped }) + const conflicts = new Set() + const result = await StoragePortable.importFile(target.store, path.join(backup, "data", "agent-records.ndjson"), { + operationID: id, + accept: async (entry, tx) => { + if (entry.type === "event") return false + if (entry.type === "receipt") return true + if (local.has(entry.key[0]) || derived.has(entry.key[0])) return false + const owner = sessionOwner(entry) + if (owner && skipped.has(owner)) { + conflicts.add(owner) + return false + } + return (await tx.readMany([entry.key]))[0] === undefined + }, + afterImport: async (tx) => { + await Session.rebuildStorageIndexes(tx) + await tx.write(["storage_transfer", id], { + version: 1, + complete: true, + skippedSessionIDs: [...conflicts], + backup: path.relative(targetRoot, backup), + created: Date.now(), + }) + }, + }) + await Storage.writeJsonAtomic( + path.join(backup, "report.json"), + JSON.stringify({ version: 1, ...result, skippedSessionIDs: [...conflicts] }), + { private: true, durable: true }, + ) + return { ...copied, skipped: copied.skipped + conflicts.size, skippedSessions: conflicts.size } + } finally { + await target?.store.close() + await source.store.close() + } + } +} + +function sessionOwner(entry: Extract): string | undefined { + const key = entry.key + if (key[0] === "sessions" || key[0].startsWith("session_search_") || key[0] === "session_message_order_v1") + return key[2] + if (key[0] === "snapshot-v2" && ["owners", "migrations", "deletions"].includes(key[2])) return key[3] + if (key[0] === "storage_recovery" && key[1] === "sessions") return key[2] +} diff --git a/packages/product-runtime/src/cli/server.ts b/packages/product-runtime/src/cli/server.ts index 7be416e8f..56aa94aec 100644 --- a/packages/product-runtime/src/cli/server.ts +++ b/packages/product-runtime/src/cli/server.ts @@ -7,7 +7,6 @@ import { FormatError, FormatUnknownError } from "@ericsanchezok/synergy-cli/cli/ import { Log } from "@ericsanchezok/synergy-harness/util/log" import { ServerProcessLock } from "@ericsanchezok/synergy-harness/util/server-process-lock" import { Server } from "@ericsanchezok/synergy-server/server/server" -import type { RuntimeOptions } from "../server/runtime" export const ServerCommand = cmd({ command: ["$0", "server"], @@ -30,21 +29,22 @@ export const ServerCommand = cmd({ }), describe: "start synergy server", handler: async (args) => { - let network: RuntimeOptions["network"] | undefined + let network: Awaited> | undefined try { const managed = process.env.SYNERGY_DESKTOP_STARTUP_PROGRESS === "1" - network = await resolveNetworkOptions(args, { - output: managed ? "silent" : "interactive", - reporter: managed ? createManagedMigrationReporter() : undefined, - }) const managedService = args.managedService await runServerRuntime({ + migrationReporter: managed ? createManagedMigrationReporter() : undefined, + migrationOutput: managed ? "silent" : "interactive", recoveryReporter: managed ? createManagedRecoveryReporter() : undefined, interactive: !(managedService || args.nonInteractive), printBanner: args.banner, printChannelStatus: !managedService, - network, + network: async () => { + network = await resolveNetworkOptions(args) + return network + }, }) } catch (error) { Log.Default.error("server startup failed", { diff --git a/packages/product-runtime/src/daemon-entry.ts b/packages/product-runtime/src/daemon-entry.ts index 7fae65ee3..b9a30ebcf 100644 --- a/packages/product-runtime/src/daemon-entry.ts +++ b/packages/product-runtime/src/daemon-entry.ts @@ -2,7 +2,6 @@ import { run as runServerRuntime } from "./server/runtime" import { Installation } from "@ericsanchezok/synergy-harness/global/installation" import { Log } from "@ericsanchezok/synergy-harness/util/log" import { DaemonSpec } from "@ericsanchezok/synergy-cli/daemon/spec" -import { ensureMigrations } from "@ericsanchezok/synergy-harness/migration" async function main() { await Log.init({ @@ -11,14 +10,11 @@ async function main() { level: Installation.isLocal() ? "DEBUG" : "INFO", }) - await ensureMigrations() - const network = await DaemonSpec.resolveNetwork({ argv: process.argv }) - await runServerRuntime({ interactive: false, printBanner: false, printChannelStatus: false, - network, + network: () => DaemonSpec.resolveNetwork({ argv: process.argv }), }) } diff --git a/packages/product-runtime/src/index.ts b/packages/product-runtime/src/index.ts index 001084578..57fba33f0 100644 --- a/packages/product-runtime/src/index.ts +++ b/packages/product-runtime/src/index.ts @@ -1,4 +1,15 @@ async function bootstrap(): Promise { + if (process.argv.includes("__storage-maintenance-runner")) { + await import("./product-registration") + const { StorageMaintenance } = await import("@ericsanchezok/synergy-harness/storage/maintenance") + await using handle = await StorageMaintenance.open({ recover: true }) + return + } + if (process.argv.includes("__storage-worker-runner")) { + await import("@ericsanchezok/synergy-harness/storage/sqlite-worker") + await new Promise(() => {}) + return + } if (process.argv.some((arg) => arg.startsWith("__") && arg.endsWith("-runner"))) { const { Global } = await import("@ericsanchezok/synergy-harness/global") await Global.initialize({ cache: false }) diff --git a/packages/product-runtime/src/server/runtime.ts b/packages/product-runtime/src/server/runtime.ts index 850559406..7e2f310f5 100644 --- a/packages/product-runtime/src/server/runtime.ts +++ b/packages/product-runtime/src/server/runtime.ts @@ -24,24 +24,29 @@ const log = Log.create({ service: "server-runtime" }) const CHANNEL_CONNECT_TIMEOUT = 15_000 const STATUS_POLL_INTERVAL = 320 +type Network = import("@ericsanchezok/synergy-harness/lifecycle").RuntimeNetwork + export interface RuntimeOptions { + migrationReporter?: Parameters[0]["reporter"] + migrationOutput?: Parameters[0]["migrationOutput"] recoveryReporter?: Parameters[0]["recoveryReporter"] interactive: boolean printBanner: boolean printChannelStatus: boolean - network: { - hostname: string - port: number - mdns?: boolean - cors?: string[] - } + network: Network | (() => Promise) } export async function run(options: RuntimeOptions) { + let network: Network = { hostname: "127.0.0.1", port: 0 } const reporter = options.printBanner ? StartupReporter.create() : undefined await using handle = await ProductRuntimeHandle.open({ mode: "server", - network: options.network, - reporter: reporter ? { summary: (summary) => reporter.migration(summary) } : undefined, + network: async () => { + network = typeof options.network === "function" ? await options.network() : options.network + return network + }, + reporter: + options.migrationReporter ?? (reporter ? { summary: (summary) => reporter.migration(summary) } : undefined), + migrationOutput: options.migrationOutput, recoveryReporter: options.recoveryReporter, }) const server = handle.server @@ -54,7 +59,7 @@ export async function run(options: RuntimeOptions) { cwd: process.cwd(), launchCwd: startupScopeLabel(), mode: process.env.SYNERGY_DAEMON === "1" ? "daemon" : "server", - network: options.network, + network, }, }) @@ -105,7 +110,7 @@ export async function run(options: RuntimeOptions) { const location = issue.quarantinedPath ?? issue.path reporter?.warning(`Configuration issue (${issue.code}): ${issue.error}${location ? ` — ${location}` : ""}`) } - renderBanner({ server, network: options.network, reporter: reporter ?? StartupReporter.create(), statuses }) + renderBanner({ server, network, reporter: reporter ?? StartupReporter.create(), statuses }) } if (process.env.SYNERGY_DAEMON === "1") { @@ -117,7 +122,7 @@ export async function run(options: RuntimeOptions) { function renderBanner(input: { server: { hostname?: string; port?: number } - network: RuntimeOptions["network"] + network: Network reporter: StartupReporter.Reporter statuses: StartupReporter.StatusRow[] }) { diff --git a/packages/product-runtime/test/cli/data-transfer.test.ts b/packages/product-runtime/test/cli/data-transfer.test.ts new file mode 100644 index 000000000..32dc28552 --- /dev/null +++ b/packages/product-runtime/test/cli/data-transfer.test.ts @@ -0,0 +1,94 @@ +import { expect, test } from "bun:test" +import fs from "node:fs/promises" +import path from "node:path" +import { DataTransfer } from "../../src/cli/data/transfer" +import { StorageBootstrap } from "@ericsanchezok/synergy-harness/storage/bootstrap" +import { Storage } from "@ericsanchezok/synergy-harness/storage/storage" +import { Session } from "@ericsanchezok/synergy-harness/session" +import { Identifier } from "@ericsanchezok/synergy-harness/id/id" +import { ScopeContext } from "@ericsanchezok/synergy-harness/scope/context" +import { SnapshotArchive } from "@ericsanchezok/synergy-harness/session/snapshot-archive" +import { tmpdir } from "@ericsanchezok/synergy-harness/test/support/fixture" + +test("merge keeps the target session aggregate and retains skipped source evidence", async () => { + await using tmp = await tmpdir({ git: true }) + const scope = await tmp.scope() + const id = Identifier.descending("session") + const addedID = Identifier.descending("session") + const sourceRoot = path.join(tmp.path, "source") + const targetRoot = path.join(tmp.path, "target") + for (const [root, title] of [ + [sourceRoot, "source"], + [targetRoot, "target"], + ]) { + const prepared = await StorageBootstrap.prepare({ root }) + try { + await Storage.provide({ store: prepared.store, artifactDirectory: path.join(root, "data") }, () => + ScopeContext.provide({ + scope, + fn: async () => { + await Storage.write(["projects", scope.id], scope) + await Session.create({ id, title }) + await Storage.write(["sessions", scope.id, id, "owner-extension"], { title }) + if (root === sourceRoot) { + await Session.create({ id: addedID, title: "new session" }) + await Storage.write(["sessions", scope.id, id, "source-only"], { preserve: true }) + await Bun.write(path.join(root, "data", "sessions", scope.id, id, "private.bin"), "source evidence") + } + }, + }), + ) + await prepared.activate() + } finally { + await prepared.store.close() + } + } + await using locks = await SnapshotArchive.lockHomes([sourceRoot, targetRoot]) + const result = await DataTransfer.merge(sourceRoot, targetRoot) + expect(result.skippedSessions).toBe(1) + const target = await StorageBootstrap.inspect(targetRoot) + if (!target) throw new Error("missing target") + try { + expect(await target.store.read<{ title: string }>(["sessions", scope.id, id, "owner-extension"])).toEqual({ + title: "target", + }) + expect((await target.store.readMany([["sessions", scope.id, id, "source-only"]]))[0]).toBeUndefined() + expect(await Bun.file(path.join(targetRoot, "data", "sessions", scope.id, id, "private.bin")).exists()).toBe(false) + expect(await target.store.read(["session_index", addedID])).toMatchObject({ scopeID: scope.id }) + const [transfer] = await target.store.query<{ backup: string }>({ kind: "storage_transfer" }) + expect( + await Bun.file( + path.join(targetRoot, transfer.value.backup, "data", "sessions", scope.id, id, "private.bin"), + ).text(), + ).toBe("source evidence") + expect((await target.store.verify()).issues).toEqual([]) + } finally { + await target.store.close() + } +}) + +test("portable pack restores authority without copying the source database identity", async () => { + await using tmp = await tmpdir() + const sourceRoot = path.join(tmp.path, "source") + const restoredRoot = path.join(tmp.path, "restored") + const source = await StorageBootstrap.prepare({ root: sourceRoot }) + await source.store.write(["future-owner", "record"], { nested: { unknown: 42 } }) + await source.activate() + await source.store.close() + await DataTransfer.pack(sourceRoot, path.join(restoredRoot, "data")) + expect(await Bun.file(path.join(restoredRoot, "data", "storage", "manifest.json")).exists()).toBe(false) + const restored = await StorageBootstrap.prepare({ root: restoredRoot }) + try { + expect(await restored.store.read<{ nested: { unknown: number } }>(["future-owner", "record"])).toEqual({ + nested: { unknown: 42 }, + }) + expect(restored.manifest.storeID).not.toBe(source.manifest.storeID) + await restored.activate() + await fs.rm(sourceRoot, { recursive: true }) + expect(await restored.store.read<{ nested: { unknown: number } }>(["future-owner", "record"])).toEqual({ + nested: { unknown: 42 }, + }) + } finally { + await restored.store.close() + } +}) diff --git a/packages/product-runtime/test/config/domain.test.ts b/packages/product-runtime/test/config/domain.test.ts index 6b6d262bb..809c2698c 100644 --- a/packages/product-runtime/test/config/domain.test.ts +++ b/packages/product-runtime/test/config/domain.test.ts @@ -31,6 +31,7 @@ test("config domain filenames are stable and ordered", () => { "115-github.jsonc", "120-runtime.jsonc", "125-voice.jsonc", + "130-storage.jsonc", ]) }) diff --git a/packages/product-runtime/test/daemon/config-migration.test.ts b/packages/product-runtime/test/daemon/config-migration.test.ts index 046d29f1c..a429613d1 100644 --- a/packages/product-runtime/test/daemon/config-migration.test.ts +++ b/packages/product-runtime/test/daemon/config-migration.test.ts @@ -3,6 +3,7 @@ import { afterEach, beforeEach, describe, expect, test } from "bun:test" import fs from "fs/promises" import os from "os" import path from "path" +import { Storage } from "@ericsanchezok/synergy-harness/storage/storage" import { Config } from "@ericsanchezok/synergy-harness/config/config" import { DaemonSpec } from "@ericsanchezok/synergy-cli/daemon/spec" import { ObservabilityStore } from "@ericsanchezok/synergy-harness/observability" @@ -31,6 +32,7 @@ describe("daemon.spec", () => { await fs.mkdir(path.join(home, ".synergy", "config"), { recursive: true }) Config.global.reset() resetMigrations() + await Storage.removeTree(["meta", "migration"]) }) afterEach(async () => { diff --git a/packages/product-runtime/test/provider/fixtures/models-runtime-offline.ts b/packages/product-runtime/test/provider/fixtures/models-runtime-offline.ts index 53fb21a1a..ebe26a357 100644 --- a/packages/product-runtime/test/provider/fixtures/models-runtime-offline.ts +++ b/packages/product-runtime/test/provider/fixtures/models-runtime-offline.ts @@ -1,6 +1,8 @@ import path from "path" import fs from "fs/promises" +import { StorageMaintenance } from "@ericsanchezok/synergy-harness/storage/maintenance" const action = process.argv[2] +await using storage = await StorageMaintenance.open() let requests = 0 function refreshedCatalog() { @@ -100,7 +102,9 @@ if (action === "refresh") { : [], requests, }), - () => process.exit(0), + () => { + void storage.close().then(() => process.exit(0)) + }, ) } else if (action === "invalid-refresh") { globalThis.fetch = (() => Promise.resolve(Response.json(refreshedCatalog()))) as unknown as typeof fetch @@ -132,7 +136,9 @@ if (action === "refresh") { providerCatalogProviders: provider.catalogProviders, bootstrapCatalogProviders: bootstrap.provider.catalogProviders, }), - () => process.exit(0), + () => { + void storage.close().then(() => process.exit(0)) + }, ) } else if (action === "refresh-routes") { const waiters: Array<() => void> = [] @@ -188,7 +194,9 @@ if (action === "refresh") { bootstrapConnected: bootstrap.provider.connected, runtimeReloads, }), - () => process.exit(0), + () => { + void storage.close().then(() => process.exit(0)) + }, ) } else if (action === "refresh-during-discovery") { let modelsRefreshStarted!: () => void @@ -244,7 +252,9 @@ if (action === "refresh") { const snapshot = persisted.snapshots.find((candidate: { providerID: string }) => candidate.providerID === providerID) process.stdout.write( JSON.stringify({ activeModels: snapshot?.activeModels.map((model: { id: string }) => model.id) ?? [] }), - () => process.exit(0), + () => { + void storage.close().then(() => process.exit(0)) + }, ) } else if (action === "refresh-after-discovery") { let modelsRefreshStarted!: () => void diff --git a/packages/product-runtime/test/server/plugin-approval-routes.test.ts b/packages/product-runtime/test/server/plugin-approval-routes.test.ts index 77c692d1c..3bafd9129 100644 --- a/packages/product-runtime/test/server/plugin-approval-routes.test.ts +++ b/packages/product-runtime/test/server/plugin-approval-routes.test.ts @@ -19,7 +19,7 @@ import { tmpdir } from "@ericsanchezok/synergy-harness/test/support/fixture" import { compilePluginManifest, definePlugin, operation, capability } from "@ericsanchezok/synergy-plugin" import z from "zod" import { sha256File } from "@ericsanchezok/synergy-harness/util/crypto" -import { localRegistryPath } from "@ericsanchezok/synergy-plugin-host/plugin/local-registry-store" +import { Storage } from "@ericsanchezok/synergy-harness/storage/storage" Log.init({ print: false }) @@ -610,8 +610,6 @@ describe("plugin approval routes", () => { const registryVersion = "1.0.0" try { // Build a local registry that resolves the fixture - const registryPath = localRegistryPath() - fs.mkdirSync(path.dirname(registryPath), { recursive: true }) const registry = { plugins: [ { @@ -625,7 +623,7 @@ describe("plugin approval routes", () => { }, ], } - fs.writeFileSync(registryPath, JSON.stringify(registry, null, 2)) + for (const entry of registry.plugins) await Storage.write(["registry", "entries", entry.id], entry) await ScopeContext.provide({ scope, @@ -665,7 +663,7 @@ describe("plugin approval routes", () => { } finally { // Clean up registry file try { - fs.unlinkSync(localRegistryPath()) + await Storage.removeTree(["registry", "entries"]) } catch {} await restoreState() } @@ -679,8 +677,6 @@ describe("plugin approval routes", () => { const fixture = createPluginFixture(tmp.path, "reg-stale-test") const registryVersion = "1.0.0" try { - const registryPath = localRegistryPath() - fs.mkdirSync(path.dirname(registryPath), { recursive: true }) const registry = { plugins: [ { @@ -689,7 +685,7 @@ describe("plugin approval routes", () => { }, ], } - fs.writeFileSync(registryPath, JSON.stringify(registry, null, 2)) + for (const entry of registry.plugins) await Storage.write(["registry", "entries", entry.id], entry) await ScopeContext.provide({ scope, @@ -719,7 +715,7 @@ describe("plugin approval routes", () => { }) } finally { try { - fs.unlinkSync(localRegistryPath()) + await Storage.removeTree(["registry", "entries"]) } catch {} await restoreState() } @@ -731,23 +727,10 @@ describe("plugin approval routes", () => { const targetPluginId = "reg-manifest-target" const registryVersion = "1.0.0" try { - const registryPath = localRegistryPath() - fs.mkdirSync(path.dirname(registryPath), { recursive: true }) - fs.writeFileSync( - registryPath, - JSON.stringify( - { - plugins: [ - { - id: targetPluginId, - versions: [{ version: registryVersion, downloadUrl: fixture.spec }], - }, - ], - }, - null, - 2, - ), - ) + await Storage.write(["registry", "entries", targetPluginId], { + id: targetPluginId, + versions: [{ version: registryVersion, downloadUrl: fixture.spec }], + }) await ScopeContext.provide({ scope, @@ -776,7 +759,7 @@ describe("plugin approval routes", () => { }) } finally { try { - fs.unlinkSync(localRegistryPath()) + await Storage.removeTree(["registry", "entries"]) } catch {} await restoreState() } diff --git a/packages/product-runtime/test/session/invoke.test.ts b/packages/product-runtime/test/session/invoke.test.ts index cd9a90198..a8055cb5c 100644 --- a/packages/product-runtime/test/session/invoke.test.ts +++ b/packages/product-runtime/test/session/invoke.test.ts @@ -2587,7 +2587,7 @@ for (const phase of ["materializing", "persisted-terminal", "startup-without-tas parts: [{ type: "text", text: "Child task completed" }], }) if (phase !== "materializing") - for (const item of await SessionInbox.drainSteer(session.id)) + for (const item of await SessionInbox.peekSteer(session.id)) await SessionInbox.materializeItem(item, rootID, { guiding: true }) const queued = phase.startsWith("startup") ? undefined diff --git a/packages/runtime-local/test/session/migration.test.ts b/packages/runtime-local/test/session/migration.test.ts index 628b32a71..73440f835 100644 --- a/packages/runtime-local/test/session/migration.test.ts +++ b/packages/runtime-local/test/session/migration.test.ts @@ -264,11 +264,11 @@ describe("session migrations", () => { const target = StoragePath.sessionNavIndex(Identifier.asScopeID(tmpScope.id)) const originalWrite = Storage.write { - using _write = spyOn(Storage, "write").mockImplementation(async (key, content, options) => { + using _write = spyOn(Storage, "write").mockImplementation(async (key, content) => { if (key.length === target.length && key.every((part, index) => part === target[index])) { throw new Error("nav index write failed") } - return originalWrite(key, content, options) + return originalWrite(key, content) }) const migration = migrations.find((entry) => entry.id === "20260730-session-nav-channel-provider-fields") diff --git a/packages/runtime-local/test/session/search-index.test.ts b/packages/runtime-local/test/session/search-index.test.ts index 5c6ac8822..ace9f0490 100644 --- a/packages/runtime-local/test/session/search-index.test.ts +++ b/packages/runtime-local/test/session/search-index.test.ts @@ -333,18 +333,14 @@ describe("session.search-index", () => { const { Storage } = await import("@ericsanchezok/synergy-harness/storage/storage") const { StoragePath } = await import("@ericsanchezok/synergy-harness/storage/path") - await Storage.write( - StoragePath.sessionSearchIndex(scopeID, sessionID), - { - version: 1, - tokenizerVersion: 1, - scopeID, - sessionID, - updatedAt: Date.now(), - messages: [], - }, - { compact: true }, - ) + await Storage.write(StoragePath.sessionSearchIndex(scopeID, sessionID), { + version: 1, + tokenizerVersion: 1, + scopeID, + sessionID, + updatedAt: Date.now(), + messages: [], + }) // readRecord ignores the stale version, so the query rescans and finds // the content, then persists a current-version record. diff --git a/packages/runtime-local/test/tool/bash-github-token.test.ts b/packages/runtime-local/test/tool/bash-github-token.test.ts index 92c46b61b..310a99ebb 100644 --- a/packages/runtime-local/test/tool/bash-github-token.test.ts +++ b/packages/runtime-local/test/tool/bash-github-token.test.ts @@ -28,7 +28,8 @@ async function reset() { } beforeEach(async () => { - delete process.env.SHELL + if (process.platform !== "win32") process.env.SHELL = "/bin/bash" + else delete process.env.SHELL Shell.preferred.reset() Shell.acceptable.reset() }) diff --git a/packages/sdk/js/src/gen/sdk.gen.ts b/packages/sdk/js/src/gen/sdk.gen.ts index 5610c37cc..8f75e94e7 100644 --- a/packages/sdk/js/src/gen/sdk.gen.ts +++ b/packages/sdk/js/src/gen/sdk.gen.ts @@ -6379,6 +6379,7 @@ export class Domain extends HeyApiClient { | "commands" | "permissions" | "runtime" + | "storage" | "plugins" | "channels" | "holos" @@ -6427,6 +6428,7 @@ export class Domain extends HeyApiClient { | "commands" | "permissions" | "runtime" + | "storage" | "plugins" | "channels" | "holos" @@ -6482,6 +6484,7 @@ export class Domain extends HeyApiClient { | "commands" | "permissions" | "runtime" + | "storage" | "plugins" | "channels" | "holos" @@ -6738,6 +6741,7 @@ export class Config extends HeyApiClient { | "commands" | "permissions" | "runtime" + | "storage" | "plugins" | "channels" | "holos" @@ -6755,6 +6759,7 @@ export class Config extends HeyApiClient { | "commands" | "permissions" | "runtime" + | "storage" | "plugins" | "channels" | "holos" diff --git a/packages/sdk/js/src/gen/types.gen.ts b/packages/sdk/js/src/gen/types.gen.ts index 11798969d..28c5b34f8 100644 --- a/packages/sdk/js/src/gen/types.gen.ts +++ b/packages/sdk/js/src/gen/types.gen.ts @@ -4199,6 +4199,21 @@ export type Config = { */ $schema?: string logLevel?: LogLevel + /** + * Global authoritative storage; backend changes require an explicit storage migration + */ + storage?: + | { + backend: "sqlite" + namespace?: string + filename?: string + } + | { + backend: "postgres" + namespace: string + connectionEnv: string + maxConnections?: number + } server?: ServerConfig /** * Command configuration @@ -5320,6 +5335,7 @@ export type ConfigDomainSummary = { | "commands" | "permissions" | "runtime" + | "storage" | "plugins" | "channels" | "holos" @@ -5376,6 +5392,7 @@ export type ConfigExportResult = { | "commands" | "permissions" | "runtime" + | "storage" | "plugins" | "channels" | "holos" @@ -5434,6 +5451,7 @@ export type ConfigDomainImportDomainPlan = { | "commands" | "permissions" | "runtime" + | "storage" | "plugins" | "channels" | "holos" @@ -5487,6 +5505,7 @@ export type ConfigDomainImportPlanInput = { | "commands" | "permissions" | "runtime" + | "storage" | "plugins" | "channels" | "holos" @@ -5570,6 +5589,7 @@ export type ConfigImportRevisionConflictError = { | "commands" | "permissions" | "runtime" + | "storage" | "plugins" | "channels" | "holos" @@ -5601,6 +5621,7 @@ export type ConfigDomainImportApplyInput = { | "commands" | "permissions" | "runtime" + | "storage" | "plugins" | "channels" | "holos" @@ -12900,6 +12921,7 @@ export type ConfigDomainGetData = { | "commands" | "permissions" | "runtime" + | "storage" | "plugins" | "channels" | "holos" @@ -12950,6 +12972,7 @@ export type ConfigDomainUpdateData = { | "commands" | "permissions" | "runtime" + | "storage" | "plugins" | "channels" | "holos" @@ -13000,6 +13023,7 @@ export type ConfigDomainOpenData = { | "commands" | "permissions" | "runtime" + | "storage" | "plugins" | "channels" | "holos" @@ -13058,6 +13082,7 @@ export type ConfigExportData = { | "commands" | "permissions" | "runtime" + | "storage" | "plugins" | "channels" | "holos" @@ -13075,6 +13100,7 @@ export type ConfigExportData = { | "commands" | "permissions" | "runtime" + | "storage" | "plugins" | "channels" | "holos" diff --git a/packages/sdk/openapi.json b/packages/sdk/openapi.json index 26d639577..c5fa353ba 100644 --- a/packages/sdk/openapi.json +++ b/packages/sdk/openapi.json @@ -4676,6 +4676,7 @@ "commands", "permissions", "runtime", + "storage", "plugins", "channels", "holos", @@ -4761,6 +4762,7 @@ "commands", "permissions", "runtime", + "storage", "plugins", "channels", "holos", @@ -4857,6 +4859,7 @@ "commands", "permissions", "runtime", + "storage", "plugins", "channels", "holos", @@ -4963,6 +4966,7 @@ "commands", "permissions", "runtime", + "storage", "plugins", "channels", "holos", @@ -4986,6 +4990,7 @@ "commands", "permissions", "runtime", + "storage", "plugins", "channels", "holos", @@ -36981,6 +36986,54 @@ "logLevel": { "$ref": "#/components/schemas/LogLevel" }, + "storage": { + "description": "Global authoritative storage; backend changes require an explicit storage migration", + "anyOf": [ + { + "type": "object", + "properties": { + "backend": { + "type": "string", + "const": "sqlite" + }, + "namespace": { + "type": "string", + "pattern": "^[a-zA-Z0-9_-]{1,128}$" + }, + "filename": { + "type": "string", + "minLength": 1 + } + }, + "required": ["backend"], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "backend": { + "type": "string", + "const": "postgres" + }, + "namespace": { + "type": "string", + "pattern": "^[a-zA-Z0-9_-]{1,128}$" + }, + "connectionEnv": { + "type": "string", + "pattern": "^[a-zA-Z_][a-zA-Z0-9_]*$" + }, + "maxConnections": { + "type": "integer", + "minimum": 2, + "maximum": 64 + } + }, + "required": ["backend", "namespace", "connectionEnv"], + "additionalProperties": false + } + ] + }, "server": { "$ref": "#/components/schemas/ServerConfig" }, @@ -39802,6 +39855,7 @@ "commands", "permissions", "runtime", + "storage", "plugins", "channels", "holos", @@ -39948,6 +40002,7 @@ "commands", "permissions", "runtime", + "storage", "plugins", "channels", "holos", @@ -40079,6 +40134,7 @@ "commands", "permissions", "runtime", + "storage", "plugins", "channels", "holos", @@ -40211,6 +40267,7 @@ "commands", "permissions", "runtime", + "storage", "plugins", "channels", "holos", @@ -40431,6 +40488,7 @@ "commands", "permissions", "runtime", + "storage", "plugins", "channels", "holos", @@ -40489,6 +40547,7 @@ "commands", "permissions", "runtime", + "storage", "plugins", "channels", "holos", diff --git a/packages/testing/src/preload.ts b/packages/testing/src/preload.ts index a05b5872a..14b53813f 100644 --- a/packages/testing/src/preload.ts +++ b/packages/testing/src/preload.ts @@ -8,7 +8,22 @@ delete process.env["SYNERGY_HOME"] process.env["SYNERGY_TEST_HOME"] = isolated.env.SYNERGY_TEST_HOME process.env["SYNERGY_TEST_ROOT"] = isolated.env.SYNERGY_TEST_ROOT process.env["LC_ALL"] = isolated.env.LC_ALL -afterAll(() => isolated.dispose()) +const cleanups: Array<() => Promise | void> = [] +export function beforeTestHomeDisposal(cleanup: () => Promise | void) { + cleanups.push(cleanup) +} +afterAll(async () => { + const errors: unknown[] = [] + for (const cleanup of cleanups.toReversed()) { + try { + await cleanup() + } catch (error) { + errors.push(error) + } + } + if (errors.length) throw new AggregateError(errors, "Test resources did not settle before home disposal") + await isolated.dispose() +}) const testHome = isolated.env.SYNERGY_TEST_HOME! // Existing observability/performance tests exercise the store contract with diff --git a/packages/workbench/src/stats/engine.ts b/packages/workbench/src/stats/engine.ts index 077e1810d..7a013d71e 100644 --- a/packages/workbench/src/stats/engine.ts +++ b/packages/workbench/src/stats/engine.ts @@ -29,8 +29,8 @@ export namespace Engine { async function operationDigests() { const result: OperationDigest[] = [] const retained = new Set() - for (const scopeID of await Storage.scan(["operations"], { strict: true })) { - for (const operationID of await Storage.scan(["operations", scopeID], { strict: true })) { + for (const scopeID of await Storage.scan(["operations"])) { + for (const operationID of await Storage.scan(["operations", scopeID])) { const owner = { kind: "operation" as const, scopeID, operationID } const revision = await readRolloutRevision(owner) if (!revision) continue @@ -50,8 +50,8 @@ export namespace Engine { result.push(digest) } } - for (const scopeID of await Storage.scan(StoragePath.statsOperations(), { strict: true })) - for (const id of await Storage.scan([...StoragePath.statsOperations(), scopeID], { strict: true })) { + for (const scopeID of await Storage.scan(StoragePath.statsOperations())) + for (const id of await Storage.scan([...StoragePath.statsOperations(), scopeID])) { const key = StoragePath.statsOperation(scopeID, id) if (!retained.has(key.join("/"))) await Storage.remove(key) } diff --git a/packages/workbench/test/cortex/manager.test.ts b/packages/workbench/test/cortex/manager.test.ts index bf4d7d7fd..a280c68f3 100644 --- a/packages/workbench/test/cortex/manager.test.ts +++ b/packages/workbench/test/cortex/manager.test.ts @@ -2072,7 +2072,7 @@ describe.serial("Cortex", () => { expect(await waitForNotification(parentSession.id, first.id)).toBeDefined() expect(await waitForNotification(parentSession.id, second.id)).toBeDefined() - const drained = await SessionInbox.drainSteer(parentSession.id) + const drained = await SessionInbox.peekSteer(parentSession.id) expect(drained).toHaveLength(2) expect(new Set(drained.map((item) => item.deliveryKey))).toEqual( new Set([`cortex:taskNotification:${first.id}`, `cortex:taskNotification:${second.id}`]), diff --git a/packages/workflows/src/agenda/store.ts b/packages/workflows/src/agenda/store.ts index 2c3310e32..b3e7ce54b 100644 --- a/packages/workflows/src/agenda/store.ts +++ b/packages/workflows/src/agenda/store.ts @@ -23,7 +23,10 @@ export namespace AgendaStore { type IndexedRun = RunIndexEntry & { scopeID: Identifier.ScopeID } async function readRunIndex(scopeID: Identifier.ScopeID): Promise { - return Storage.read(StoragePath.agendaRunIndex(scopeID)).catch(() => ({ entries: [] })) + return Storage.read(StoragePath.agendaRunIndex(scopeID)).catch((error) => { + if (error instanceof Storage.NotFoundError) return { entries: [] } + throw error + }) } async function writeRunIndex(scopeID: Identifier.ScopeID, index: RunIndex): Promise { @@ -95,50 +98,52 @@ export namespace AgendaStore { input: InternalCreateInput, id: string = Identifier.ascending("agenda"), ): Promise { - const scope = ScopeContext.current.scope - const now = Date.now() - const triggers = input.triggers ?? [] - - for (const trigger of triggers) { - if (trigger.type === "webhook" && !trigger.token) { - trigger.token = randomUUID() + return Storage.transaction(async () => { + const scope = ScopeContext.current.scope + const now = Date.now() + const triggers = input.triggers ?? [] + + for (const trigger of triggers) { + if (trigger.type === "webhook" && !trigger.token) { + trigger.token = randomUUID() + } } - } - const item: AgendaTypes.Item = { - id, - status: triggers.length > 0 ? "active" : "pending", - title: input.title, - description: input.description, - tags: input.tags, - global: input.global ?? false, - triggers, - prompt: input.prompt, - deliveryMode: input.deliveryMode, - agent: input.agent, - model: input.model, - controlProfile: input.controlProfile, - sessionMode: input.sessionMode, - sessionRefs: input.sessionRefs, - timeout: input.timeout, - wake: input.wake ?? true, - silent: input.silent ?? false, - autoDone: input.autoDone ?? false, - origin: { scope, sessionID: input.sessionID, endpoint: input.endpoint }, - createdBy: input.createdBy ?? "user", - state: { - consecutiveErrors: 0, - runCount: 0, - nextRunAt: computeNextRunAt(triggers, now), - }, - time: { created: now, updated: now }, - } + const item: AgendaTypes.Item = { + id, + status: triggers.length > 0 ? "active" : "pending", + title: input.title, + description: input.description, + tags: input.tags, + global: input.global ?? false, + triggers, + prompt: input.prompt, + deliveryMode: input.deliveryMode, + agent: input.agent, + model: input.model, + controlProfile: input.controlProfile, + sessionMode: input.sessionMode, + sessionRefs: input.sessionRefs, + timeout: input.timeout, + wake: input.wake ?? true, + silent: input.silent ?? false, + autoDone: input.autoDone ?? false, + origin: { scope, sessionID: input.sessionID, endpoint: input.endpoint }, + createdBy: input.createdBy ?? "user", + state: { + consecutiveErrors: 0, + runCount: 0, + nextRunAt: computeNextRunAt(triggers, now), + }, + time: { created: now, updated: now }, + } - const scopeID = Identifier.asScopeID(item.global ? HOME_SCOPE_ID : scope.id) - await Storage.write(StoragePath.agendaItem(scopeID, id), item) - log.info("created", { id, title: input.title, global: item.global }) - await Bus.publish(AgendaEvent.ItemCreated, { item }) - return item + const scopeID = Identifier.asScopeID(item.global ? HOME_SCOPE_ID : scope.id) + await Storage.write(StoragePath.agendaItem(scopeID, id), item) + log.info("created", { id, title: input.title, global: item.global }) + await Bus.publish(AgendaEvent.ItemCreated, { item }) + return item + }) } export async function get(scopeID: string, itemID: string): Promise { @@ -168,40 +173,42 @@ export namespace AgendaStore { patch: AgendaTypes.PatchInput, options?: { recomputeNextRunAt?: boolean }, ): Promise { - const sid = Identifier.asScopeID(scopeID) - const now = Date.now() - const item = await Storage.update(StoragePath.agendaItem(sid, itemID), (draft) => { - if (patch.title !== undefined) draft.title = patch.title - if (patch.description !== undefined) draft.description = patch.description - if (patch.status !== undefined) draft.status = patch.status - if (patch.tags !== undefined) draft.tags = patch.tags - if (patch.triggers !== undefined) { - for (const trigger of patch.triggers) { - if (trigger.type === "webhook" && !trigger.token) { - trigger.token = randomUUID() + return Storage.transaction(async () => { + const sid = Identifier.asScopeID(scopeID) + const now = Date.now() + const item = await Storage.update(StoragePath.agendaItem(sid, itemID), (draft) => { + if (patch.title !== undefined) draft.title = patch.title + if (patch.description !== undefined) draft.description = patch.description + if (patch.status !== undefined) draft.status = patch.status + if (patch.tags !== undefined) draft.tags = patch.tags + if (patch.triggers !== undefined) { + for (const trigger of patch.triggers) { + if (trigger.type === "webhook" && !trigger.token) { + trigger.token = randomUUID() + } } + draft.triggers = patch.triggers + draft.state.nextRunAt = computeNextRunAt(patch.triggers, now) } - draft.triggers = patch.triggers - draft.state.nextRunAt = computeNextRunAt(patch.triggers, now) - } - if (patch.prompt !== undefined) draft.prompt = patch.prompt - if (patch.global !== undefined) draft.global = patch.global - if (patch.wake !== undefined) draft.wake = patch.wake - if (patch.silent !== undefined) draft.silent = patch.silent - if (patch.agent !== undefined) draft.agent = patch.agent - if (patch.model !== undefined) draft.model = patch.model - if (patch.controlProfile !== undefined) draft.controlProfile = patch.controlProfile - if (patch.sessionMode !== undefined) draft.sessionMode = patch.sessionMode - if (patch.sessionRefs !== undefined) draft.sessionRefs = patch.sessionRefs - if (patch.timeout !== undefined) draft.timeout = patch.timeout - if (options?.recomputeNextRunAt) { - draft.state.nextRunAt = computeNextRunAt(draft.triggers, now) - } - draft.time.updated = now + if (patch.prompt !== undefined) draft.prompt = patch.prompt + if (patch.global !== undefined) draft.global = patch.global + if (patch.wake !== undefined) draft.wake = patch.wake + if (patch.silent !== undefined) draft.silent = patch.silent + if (patch.agent !== undefined) draft.agent = patch.agent + if (patch.model !== undefined) draft.model = patch.model + if (patch.controlProfile !== undefined) draft.controlProfile = patch.controlProfile + if (patch.sessionMode !== undefined) draft.sessionMode = patch.sessionMode + if (patch.sessionRefs !== undefined) draft.sessionRefs = patch.sessionRefs + if (patch.timeout !== undefined) draft.timeout = patch.timeout + if (options?.recomputeNextRunAt) { + draft.state.nextRunAt = computeNextRunAt(draft.triggers, now) + } + draft.time.updated = now + }) + log.info("updated", { id: itemID }) + await Bus.publish(AgendaEvent.ItemUpdated, { item }) + return item }) - log.info("updated", { id: itemID }) - await Bus.publish(AgendaEvent.ItemUpdated, { item }) - return item } export async function updateRunState( @@ -219,67 +226,74 @@ export namespace AgendaStore { triggers: AgendaTypes.Trigger[], signalType: string, ): Promise<{ item: AgendaTypes.Item; nextRunAt: number | undefined }> { - const sid = Identifier.asScopeID(scopeID) - const newNextRunAt = computeNextRunAt(triggers) - const item = await Storage.update(StoragePath.agendaItem(sid, itemID), (draft) => { - draft.state.lastRunAt = result.startTime - draft.state.lastRunStatus = result.status - draft.state.lastRunError = result.error - draft.state.lastRunDuration = result.duration - draft.state.lastRunSessionID = result.sessionID - draft.state.runCount++ - - if (result.status === "error") { - draft.state.consecutiveErrors++ - } else { - draft.state.consecutiveErrors = 0 - } + return Storage.transaction(async () => { + const sid = Identifier.asScopeID(scopeID) + const newNextRunAt = computeNextRunAt(triggers) + const item = await Storage.update(StoragePath.agendaItem(sid, itemID), (draft) => { + draft.state.lastRunAt = result.startTime + draft.state.lastRunStatus = result.status + draft.state.lastRunError = result.error + draft.state.lastRunDuration = result.duration + draft.state.lastRunSessionID = result.sessionID + draft.state.runCount++ + + if (result.status === "error") { + draft.state.consecutiveErrors++ + } else { + draft.state.consecutiveErrors = 0 + } - draft.state.nextRunAt = newNextRunAt - - if (result.autoDone && result.status !== "error") { - draft.status = "done" - } else { - const hasNonTimeTriggers = triggers.some( - (t) => t.type === "watch" || t.type === "webhook" || t.type === "github", - ) - const hasRecurringSessionTrigger = triggers.some((t) => t.type === "session" && t.once === false) - if ( - newNextRunAt === undefined && - signalType !== "manual" && - !hasNonTimeTriggers && - !hasRecurringSessionTrigger - ) { + draft.state.nextRunAt = newNextRunAt + + if (result.autoDone && result.status !== "error") { draft.status = "done" + } else { + const hasNonTimeTriggers = triggers.some( + (t) => t.type === "watch" || t.type === "webhook" || t.type === "github", + ) + const hasRecurringSessionTrigger = triggers.some((t) => t.type === "session" && t.once === false) + if ( + newNextRunAt === undefined && + signalType !== "manual" && + !hasNonTimeTriggers && + !hasRecurringSessionTrigger + ) { + draft.status = "done" + } } - } - draft.time.updated = Date.now() + draft.time.updated = Date.now() + }) + await Bus.publish(AgendaEvent.ItemUpdated, { item }) + return { item, nextRunAt: newNextRunAt } }) - await Bus.publish(AgendaEvent.ItemUpdated, { item }) - return { item, nextRunAt: newNextRunAt } } export async function remove(scopeID: string, itemID: string): Promise { - const sid = Identifier.asScopeID(scopeID) - await Storage.remove(StoragePath.agendaItem(sid, itemID)) - await Storage.removeTree(StoragePath.agendaRunsRoot(sid, itemID)) - const index = await readRunIndex(sid) - if (index.entries.length > 0) { - index.entries = index.entries.filter((e) => e.itemID !== itemID) - await writeRunIndex(sid, index) - } - log.info("removed", { id: itemID }) - await Bus.publish(AgendaEvent.ItemDeleted, { id: itemID, scopeID }) + return Storage.transaction(async () => { + const sid = Identifier.asScopeID(scopeID) + await Storage.remove(StoragePath.agendaItem(sid, itemID)) + await Storage.removeTree(StoragePath.agendaRunsRoot(sid, itemID)) + const index = await readRunIndex(sid) + if (index.entries.length > 0) { + index.entries = index.entries.filter((e) => e.itemID !== itemID) + await writeRunIndex(sid, index) + } + log.info("removed", { id: itemID }) + await Bus.publish(AgendaEvent.ItemDeleted, { id: itemID, scopeID }) + }) } export async function appendRun(scopeID: string, run: AgendaTypes.RunLog): Promise { - const sid = Identifier.asScopeID(scopeID) - await Storage.write(StoragePath.agendaRun(sid, run.itemID, run.id), run) - const index = await readRunIndex(sid) - // New runs always have the latest started time, so unshift preserves descending order - index.entries.unshift({ id: run.id, itemID: run.itemID, started: run.time.started }) - await writeRunIndex(sid, index) + return Storage.transaction(async () => { + const sid = Identifier.asScopeID(scopeID) + await Storage.write(StoragePath.agendaRun(sid, run.itemID, run.id), run) + const index = await readRunIndex(sid) + index.entries = index.entries.filter((entry) => entry.id !== run.id) + index.entries.push({ id: run.id, itemID: run.itemID, started: run.time.started }) + index.entries.sort((a, b) => b.started - a.started || b.id.localeCompare(a.id)) + await writeRunIndex(sid, index) + }) } export async function listRuns(scopeID: string, itemID: string): Promise { diff --git a/packages/workflows/src/blueprint/loop-store.ts b/packages/workflows/src/blueprint/loop-store.ts index 30bac359d..3cfdf4c06 100644 --- a/packages/workflows/src/blueprint/loop-store.ts +++ b/packages/workflows/src/blueprint/loop-store.ts @@ -57,66 +57,53 @@ export namespace BlueprintLoopStore { executionTools?: Info["executionTools"] auditTools?: Info["auditTools"] }): Promise { - const scopeID = ScopeContext.current.scope.id - const sid = Identifier.asScopeID(scopeID) - const activeLoop = (await list(scopeID)).find( - (loop) => loop.noteID === input.noteID && isActiveLoopStatus(loop.status), - ) - if (activeLoop) { - throw new LoopError.AlreadyActive({ - noteID: input.noteID, - loopID: activeLoop.id, - sessionID: activeLoop.sessionID, - status: activeLoop.status, - }) - } - - const now = Date.now() - const id = Identifier.ascending("blueprint_loop") - const loop: Info = { - id, - noteID: input.noteID, - noteVersion: input.noteVersion, - title: input.title, - description: input.description, - sessionID: input.sessionID, - executionAgent: input.executionAgent, - auditAgent: input.auditAgent?.trim() || "supervisor", - scopeID, - status: "armed", - runMode: input.runMode, - parentSessionID: input.parentSessionID, - firstPrompt: input.firstPrompt, - source: input.source ?? "user", - sourceDigest: input.sourceDigest, - budget: input.budget, - pluginOwner: input.pluginOwner, - model: input.model, - executionTools: input.executionTools, - auditTools: input.auditTools, - time: { created: now, updated: now }, - } - await Storage.write(StoragePath.blueprintLoop(sid, id), loop) - - // Link to note: increment runCount, set lastRunAt, activeLoopID - try { - const note = await NoteStore.get(scopeID, loop.noteID) - if (note.kind === "blueprint") { - const bp = note.blueprint ?? {} - await NoteStore.update(scopeID, loop.noteID, { - blueprint: { - runCount: (bp.runCount ?? 0) + 1, - lastRunAt: now, - activeLoopID: id, - }, + return Storage.transaction(async () => { + const scopeID = ScopeContext.current.scope.id + const sid = Identifier.asScopeID(scopeID) + const activeLoop = (await list(scopeID)).find( + (loop) => loop.noteID === input.noteID && isActiveLoopStatus(loop.status), + ) + if (activeLoop) { + throw new LoopError.AlreadyActive({ + noteID: input.noteID, + loopID: activeLoop.id, + sessionID: activeLoop.sessionID, + status: activeLoop.status, }) } - } catch { - // Note may not exist or not be a blueprint — best effort - } - await Bus.publish(LoopEvent.Created, { loop }) - return loop + const now = Date.now() + const id = Identifier.ascending("blueprint_loop") + const loop: Info = { + id, + noteID: input.noteID, + noteVersion: input.noteVersion, + title: input.title, + description: input.description, + sessionID: input.sessionID, + executionAgent: input.executionAgent, + auditAgent: input.auditAgent?.trim() || "supervisor", + scopeID, + status: "armed", + runMode: input.runMode, + parentSessionID: input.parentSessionID, + firstPrompt: input.firstPrompt, + source: input.source ?? "user", + sourceDigest: input.sourceDigest, + budget: input.budget, + pluginOwner: input.pluginOwner, + model: input.model, + executionTools: input.executionTools, + auditTools: input.auditTools, + time: { created: now, updated: now }, + } + await Storage.write(StoragePath.blueprintLoop(sid, id), loop) + + await NoteStore.recordBlueprintRun({ scopeID, noteID: loop.noteID, loopID: id, started: now }) + + await Bus.publish(LoopEvent.Created, { loop }) + return loop + }) } export async function get(scopeID: string, id: string): Promise { @@ -137,17 +124,19 @@ export namespace BlueprintLoopStore { id: string, stopRequest: NonNullable, ): Promise { - const sid = Identifier.asScopeID(scopeID) - const updated = await Storage.update(StoragePath.blueprintLoop(sid, id), (draft) => { - if (draft.status !== "running") { - throw new Error(`Cannot request review for BlueprintLoop ${draft.id} while its status is "${draft.status}"`) - } - if (draft.stopRequest) return - draft.stopRequest = stopRequest - draft.time.updated = Date.now() + return Storage.transaction(async () => { + const sid = Identifier.asScopeID(scopeID) + const updated = await Storage.update(StoragePath.blueprintLoop(sid, id), (draft) => { + if (draft.status !== "running") { + throw new Error(`Cannot request review for BlueprintLoop ${draft.id} while its status is "${draft.status}"`) + } + if (draft.stopRequest) return + draft.stopRequest = stopRequest + draft.time.updated = Date.now() + }) + await Bus.publish(LoopEvent.Updated, { loop: updated }) + return updated }) - await Bus.publish(LoopEvent.Updated, { loop: updated }) - return updated } export async function recordAuditToolRecovery( @@ -160,22 +149,24 @@ export namespace BlueprintLoopStore { attempts: number }, ): Promise { - const sid = Identifier.asScopeID(scopeID) - const updated = await Storage.update(StoragePath.blueprintLoop(sid, id), (draft) => { - if ( - draft.status !== "auditing" || - !draft.stopRequest || - draft.auditSessionID !== input.auditSessionID || - draft.auditTaskID !== input.expectedAuditTaskID - ) { - throw new Error(`BlueprintLoop ${draft.id} review binding changed before recovery`) - } - draft.auditTaskID = input.auditTaskID - draft.stopRequest.reviewToolRecoveryAttempts = input.attempts - draft.time.updated = Date.now() + return Storage.transaction(async () => { + const sid = Identifier.asScopeID(scopeID) + const updated = await Storage.update(StoragePath.blueprintLoop(sid, id), (draft) => { + if ( + draft.status !== "auditing" || + !draft.stopRequest || + draft.auditSessionID !== input.auditSessionID || + draft.auditTaskID !== input.expectedAuditTaskID + ) { + throw new Error(`BlueprintLoop ${draft.id} review binding changed before recovery`) + } + draft.auditTaskID = input.auditTaskID + draft.stopRequest.reviewToolRecoveryAttempts = input.attempts + draft.time.updated = Date.now() + }) + await Bus.publish(LoopEvent.Updated, { loop: updated }) + return updated }) - await Bus.publish(LoopEvent.Updated, { loop: updated }) - return updated } export async function updateStatus( @@ -192,97 +183,78 @@ export namespace BlueprintLoopStore { stopRequest?: Info["stopRequest"] | null }, ): Promise { - const sid = Identifier.asScopeID(scopeID) - const current = await Storage.read(StoragePath.blueprintLoop(sid, id)) + const updated = await Storage.transaction(async () => { + const sid = Identifier.asScopeID(scopeID) + const current = await Storage.read(StoragePath.blueprintLoop(sid, id)) - if (!isValidTransition(current.status, patch.status)) { - throw new LoopError.InvalidTransition({ - from: current.status, - to: patch.status, - }) - } - - const isTerminal = patch.status === "completed" || patch.status === "failed" || patch.status === "cancelled" - if (isTerminal) { - cancelDeadline(scopeID, id) - } - - const updated = await Storage.update(StoragePath.blueprintLoop(sid, id), (draft) => { - draft.status = patch.status - draft.time.updated = Date.now() - if (isTerminal) { - draft.time.completed = Date.now() - draft.auditTaskID = undefined - draft.stopRequest = undefined - } - if (patch.status === "running" && !draft.time.started) { - draft.time.started = Date.now() - } - if (patch.status === "running" && current.status === "auditing" && patch.auditSessionID === undefined) { - draft.auditSessionID = undefined - draft.auditTaskID = undefined - draft.stopRequest = undefined + if (!isValidTransition(current.status, patch.status)) { + throw new LoopError.InvalidTransition({ + from: current.status, + to: patch.status, + }) } - if (patch.audit) draft.audit = patch.audit - if (patch.auditSessionID !== undefined) draft.auditSessionID = patch.auditSessionID ?? undefined - if (patch.auditTaskID !== undefined) draft.auditTaskID = patch.auditTaskID ?? undefined - if (patch.userPrompt !== undefined) draft.userPrompt = patch.userPrompt ?? undefined - if (patch.summary !== undefined) draft.summary = patch.summary - if (patch.stopRequest !== undefined) draft.stopRequest = patch.stopRequest ?? undefined - if (patch.error !== undefined) draft.error = patch.error - }) - if (isTerminal || (patch.status === "running" && current.status === "auditing")) { - try { - if (current.auditSessionID) { - await Session.update(current.auditSessionID, (draft) => { - draft.blueprint = { ...draft.blueprint, loopID: undefined, loopRole: undefined } - }) - } - } catch { - // best effort - } - } + const isTerminal = patch.status === "completed" || patch.status === "failed" || patch.status === "cancelled" + if (isTerminal) Storage.afterCommit(() => cancelDeadline(scopeID, id)) - if (isTerminal) { - // Clear activeLoopID from note - try { - const note = await NoteStore.get(scopeID, updated.noteID) - if (note.kind === "blueprint" && note.blueprint?.activeLoopID === id) { - await NoteStore.update(scopeID, updated.noteID, { - blueprint: { activeLoopID: null }, - }) + const updated = await Storage.update(StoragePath.blueprintLoop(sid, id), (draft) => { + draft.status = patch.status + draft.time.updated = Date.now() + if (isTerminal) { + draft.time.completed = Date.now() + draft.auditTaskID = undefined + draft.stopRequest = undefined } - } catch { - // best effort - } + if (patch.status === "running" && !draft.time.started) { + draft.time.started = Date.now() + } + if (patch.status === "running" && current.status === "auditing" && patch.auditSessionID === undefined) { + draft.auditSessionID = undefined + draft.auditTaskID = undefined + draft.stopRequest = undefined + } + if (patch.audit) draft.audit = patch.audit + if (patch.auditSessionID !== undefined) draft.auditSessionID = patch.auditSessionID ?? undefined + if (patch.auditTaskID !== undefined) draft.auditTaskID = patch.auditTaskID ?? undefined + if (patch.userPrompt !== undefined) draft.userPrompt = patch.userPrompt ?? undefined + if (patch.summary !== undefined) draft.summary = patch.summary + if (patch.stopRequest !== undefined) draft.stopRequest = patch.stopRequest ?? undefined + if (patch.error !== undefined) draft.error = patch.error + }) - // Unbind execution session - try { - if (updated.sessionID) { - await Session.update(updated.sessionID, (draft) => { + async function unbind(sessionID: string | undefined, archive = false) { + if (!sessionID) return + try { + await Session.update(sessionID, (draft) => { draft.blueprint = { ...draft.blueprint, loopID: undefined, loopRole: undefined } + if (archive) draft.time.archived = Date.now() }) + } catch (error) { + if (!(error instanceof Storage.NotFoundError)) throw error } - } catch { - // best effort } - - // Archive plugin-owned generated resources (Note + execution Session) - // User/lattice-owned loops archive via their own lifecycle paths. - if (updated.source === "plugin") { - void NoteStore.update(scopeID, updated.noteID, { archived: true }).catch(() => {}) - void Session.update(updated.sessionID, (draft) => { - draft.time.archived = Date.now() - }).catch(() => {}) + if (isTerminal || (patch.status === "running" && current.status === "auditing")) + await unbind(current.auditSessionID) + if (isTerminal) { + await NoteStore.recordBlueprintRun({ + scopeID, + noteID: updated.noteID, + loopID: id, + ended: true, + archive: updated.source === "plugin", + }) + await unbind(updated.sessionID, updated.source === "plugin") } - } - if (isTerminal && updated.source === "plugin" && updated.pluginOwner) { + await Bus.publish(LoopEvent.Updated, { loop: updated }) + return updated + }) + if ( + ["completed", "failed", "cancelled"].includes(updated.status) && + updated.source === "plugin" && + updated.pluginOwner + ) await deliverTerminalHook(scopeID, id) - } - - await Bus.publish(LoopEvent.Updated, { loop: updated }) return updated } diff --git a/packages/workflows/src/lattice/store.ts b/packages/workflows/src/lattice/store.ts index 14f708351..a8f1579c5 100644 --- a/packages/workflows/src/lattice/store.ts +++ b/packages/workflows/src/lattice/store.ts @@ -5,7 +5,6 @@ import { Identifier } from "@ericsanchezok/synergy-harness/id/id" import { ScopeContext } from "@ericsanchezok/synergy-harness/scope/context" import { StoragePath } from "@ericsanchezok/synergy-harness/storage/path" import { Storage } from "@ericsanchezok/synergy-harness/storage/storage" -import { Lock } from "@ericsanchezok/synergy-harness/util/lock" import { LatticeError } from "./error" import { LatticeEvent } from "./event" import { LatticeMachine } from "./machine" @@ -27,10 +26,6 @@ export namespace LatticeStore { changed: boolean } - function sessionLock(scopeID: string, sessionID: string): string { - return `lattice:${scopeID}:${sessionID}` - } - async function readOptional(key: string[]): Promise { try { return await Storage.read(key) @@ -65,54 +60,55 @@ export namespace LatticeStore { } export async function create(input: CreateInput): Promise { - const scopeID = ScopeContext.current.scope.id - const sid = Identifier.asScopeID(scopeID) - let run!: LatticeTypes.Run - { - using _ = await Lock.write(sessionLock(scopeID, input.sessionID)) - const existing = newest( - (await listBySession(scopeID, input.sessionID)).filter( - (candidate) => !LatticeTypes.isTerminalRun(candidate.status), - ), - ) - if (existing) { - throw new LatticeError.StateConflict({ - state: existing.state, - reason: `session already has an active Lattice Run ${existing.id}`, + return Storage.transaction(async () => { + const scopeID = ScopeContext.current.scope.id + const sid = Identifier.asScopeID(scopeID) + let run!: LatticeTypes.Run + { + const existing = newest( + (await listBySession(scopeID, input.sessionID)).filter( + (candidate) => !LatticeTypes.isTerminalRun(candidate.status), + ), + ) + if (existing) { + throw new LatticeError.StateConflict({ + state: existing.state, + reason: `session already has an active Lattice Run ${existing.id}`, + }) + } + + const now = Date.now() + run = LatticeTypes.Run.parse({ + schemaVersion: LatticeTypes.SCHEMA_VERSION, + id: Identifier.ascending("lattice_run"), + scopeID, + sessionID: input.sessionID, + mode: input.mode, + maxModelCalls: input.maxModelCalls ?? 0, + modelCallCount: 0, + status: "active", + state: "clarifying", + goalSeed: input.goal, + revision: 0, + stateRevision: 0, + pathwayRevision: 0, + pathway: [], + time: { created: now, updated: now }, }) + if (input.promptOnCreate) run = LatticeMachine.setPromptEffect(run, { promptType: "state_entry" }, now) + await Storage.write(StoragePath.latticeRun(sid, run.id), run) + await writePointer(scopeID, input.sessionID, run.id, now) } - const now = Date.now() - run = LatticeTypes.Run.parse({ - schemaVersion: LatticeTypes.SCHEMA_VERSION, - id: Identifier.ascending("lattice_run"), - scopeID, - sessionID: input.sessionID, - mode: input.mode, - maxModelCalls: input.maxModelCalls ?? 0, - modelCallCount: 0, - status: "active", - state: "clarifying", - goalSeed: input.goal, - revision: 0, - stateRevision: 0, - pathwayRevision: 0, - pathway: [], - time: { created: now, updated: now }, + const view = LatticeTypes.toRunView(run) + await Bus.publish(LatticeEvent.Created, { run: view }) + await appendEvent(scopeID, run, { + kind: "run_created", + state: run.state, + message: `Lattice Run created (${run.mode})`, }) - if (input.promptOnCreate) run = LatticeMachine.setPromptEffect(run, { promptType: "state_entry" }, now) - await Storage.write(StoragePath.latticeRun(sid, run.id), run) - await writePointer(scopeID, input.sessionID, run.id, now) - } - - const view = LatticeTypes.toRunView(run) - await Bus.publish(LatticeEvent.Created, { run: view }) - await appendEvent(scopeID, run, { - kind: "run_created", - state: run.state, - message: `Lattice Run created (${run.mode})`, - }).catch(() => undefined) - return run + return run + }) } /** @deprecated v2 never overwrites history; reset creates a new Run when the current one is terminal. */ @@ -227,42 +223,44 @@ export namespace LatticeStore { } export async function updateByRunID(scopeID: string, runID: string, editor: Editor): Promise { - const before = await getByRunID(scopeID, runID) - if (!before) throw new LatticeError.NotFound({ runID }) - let result!: UpdateResult - { - using _ = await Lock.write(sessionLock(scopeID, before.sessionID)) - result = await updateRecordUnlocked(scopeID, runID, editor) - } - if (result.changed) await Bus.publish(LatticeEvent.Updated, { run: LatticeTypes.toRunView(result.run) }) - return result.run + return Storage.transaction(async () => { + const before = await getByRunID(scopeID, runID) + if (!before) throw new LatticeError.NotFound({ runID }) + let result!: UpdateResult + { + result = await updateRecordUnlocked(scopeID, runID, editor) + } + if (result.changed) await Bus.publish(LatticeEvent.Updated, { run: LatticeTypes.toRunView(result.run) }) + return result.run + }) } export async function update(scopeID: string, sessionID: string, editor: Editor): Promise { - let result!: UpdateResult - { - using _ = await Lock.write(sessionLock(scopeID, sessionID)) - const pointer = await readPointer(scopeID, sessionID) - let current = pointer ? await getByRunID(scopeID, pointer.runID) : undefined - if (current?.sessionID !== sessionID) current = undefined - if (!current) { - const candidates = await listBySession(scopeID, sessionID) - const nonTerminal = candidates.filter((run) => !LatticeTypes.isTerminalRun(run.status)) - if (nonTerminal.length > 1) { - const conflict = newest(nonTerminal)! - throw new LatticeError.StateConflict({ - state: conflict.state, - reason: "multiple non-terminal Runs require pointer reconciliation before update", - }) + return Storage.transaction(async () => { + let result!: UpdateResult + { + const pointer = await readPointer(scopeID, sessionID) + let current = pointer ? await getByRunID(scopeID, pointer.runID) : undefined + if (current?.sessionID !== sessionID) current = undefined + if (!current) { + const candidates = await listBySession(scopeID, sessionID) + const nonTerminal = candidates.filter((run) => !LatticeTypes.isTerminalRun(run.status)) + if (nonTerminal.length > 1) { + const conflict = newest(nonTerminal)! + throw new LatticeError.StateConflict({ + state: conflict.state, + reason: "multiple non-terminal Runs require pointer reconciliation before update", + }) + } + current = nonTerminal[0] ?? newest(candidates) + if (!current) throw new LatticeError.NotFound({ sessionID }) + await writePointer(scopeID, sessionID, current.id) } - current = nonTerminal[0] ?? newest(candidates) - if (!current) throw new LatticeError.NotFound({ sessionID }) - await writePointer(scopeID, sessionID, current.id) + result = await updateRecordUnlocked(scopeID, current.id, editor) } - result = await updateRecordUnlocked(scopeID, current.id, editor) - } - if (result.changed) await Bus.publish(LatticeEvent.Updated, { run: LatticeTypes.toRunView(result.run) }) - return result.run + if (result.changed) await Bus.publish(LatticeEvent.Updated, { run: LatticeTypes.toRunView(result.run) }) + return result.run + }) } export const updateCurrent = update @@ -289,48 +287,47 @@ export namespace LatticeStore { sessionID: string, runIDs: string[], ): Promise { - const changed: LatticeTypes.Run[] = [] - let selected: LatticeTypes.Run | undefined - { - using _ = await Lock.write(sessionLock(scopeID, sessionID)) - const pointer = await readPointer(scopeID, sessionID) - const candidateIDs = new Set(runIDs) - if (pointer) candidateIDs.add(pointer.runID) - const candidates = (await Promise.all([...candidateIDs].map((runID) => getByRunID(scopeID, runID)))).filter( - (run): run is LatticeTypes.Run => run?.sessionID === sessionID, - ) - if (candidates.length === 0) { - if (pointer) await Storage.remove(StoragePath.latticeCurrent(Identifier.asScopeID(scopeID), sessionID)) - } else { - const nonTerminal = candidates.filter((run) => !LatticeTypes.isTerminalRun(run.status)) - if (nonTerminal.length > 1) { - const ordered = [...nonTerminal].sort(compareRuns) - const newestActive = ordered.at(-1)! - // Quarantine the selected Run first. If the process dies between - // record writes, no later recovery can execute its stale effect. - for (const conflict of [newestActive, ...ordered.slice(0, -1)]) { - const result = await updateRecordUnlocked(scopeID, conflict.id, (draft) => - LatticeMachine.quarantineDuplicate(draft, conflict.id === newestActive.id), - ) - if (result.changed) changed.push(result.run) - if (result.run.id === newestActive.id) selected = result.run - } + return Storage.transaction(async () => { + const changed: LatticeTypes.Run[] = [] + let selected: LatticeTypes.Run | undefined + { + const pointer = await readPointer(scopeID, sessionID) + const candidateIDs = new Set(runIDs) + if (pointer) candidateIDs.add(pointer.runID) + const candidates = (await Promise.all([...candidateIDs].map((runID) => getByRunID(scopeID, runID)))).filter( + (run): run is LatticeTypes.Run => run?.sessionID === sessionID, + ) + if (candidates.length === 0) { + if (pointer) await Storage.remove(StoragePath.latticeCurrent(Identifier.asScopeID(scopeID), sessionID)) } else { - selected = nonTerminal[0] ?? newest(candidates) - } + const nonTerminal = candidates.filter((run) => !LatticeTypes.isTerminalRun(run.status)) + if (nonTerminal.length > 1) { + const ordered = [...nonTerminal].sort(compareRuns) + const newestActive = ordered.at(-1)! + for (const conflict of [newestActive, ...ordered.slice(0, -1)]) { + const result = await updateRecordUnlocked(scopeID, conflict.id, (draft) => + LatticeMachine.quarantineDuplicate(draft, conflict.id === newestActive.id), + ) + if (result.changed) changed.push(result.run) + if (result.run.id === newestActive.id) selected = result.run + } + } else { + selected = nonTerminal[0] ?? newest(candidates) + } - if ( - selected && - (pointer?.scopeID !== scopeID || pointer.sessionID !== sessionID || pointer.runID !== selected.id) - ) { - await writePointer(scopeID, sessionID, selected.id) + if ( + selected && + (pointer?.scopeID !== scopeID || pointer.sessionID !== sessionID || pointer.runID !== selected.id) + ) { + await writePointer(scopeID, sessionID, selected.id) + } } } - } - for (const run of changed) { - await Bus.publish(LatticeEvent.Updated, { run: LatticeTypes.toRunView(run) }) - } - return selected + for (const run of changed) { + await Bus.publish(LatticeEvent.Updated, { run: LatticeTypes.toRunView(run) }) + } + return selected + }) } export async function appendEvent( @@ -345,40 +342,42 @@ export namespace LatticeStore { data?: Record }, ): Promise { - const sid = Identifier.asScopeID(scopeID) - const eventID = `lte_${createHash("sha256") - .update( - JSON.stringify({ - runID: run.id, - kind: input.kind, - stepID: input.stepID, - state: input.state, - stateRevision: run.stateRevision, - pathwayRevision: run.pathwayRevision, - message: input.message, - data: input.data, - }), - ) - .digest("hex") - .slice(0, 26)}` - using _ = await Lock.write(`lattice-event:${scopeID}:${run.id}:${eventID}`) - const existing = await readOptional(StoragePath.latticeEvent(sid, run.id, eventID)) - if (existing !== undefined) return LatticeTypes.EventInfo.parse(existing) - const event = LatticeTypes.EventInfo.parse({ - id: eventID, - runID: run.id, - scopeID, - sessionID: run.sessionID, - kind: input.kind, - stepID: input.stepID, - state: input.state, - message: input.message, - data: input.data, - time: { created: Date.now() }, + return Storage.transaction(async () => { + const sid = Identifier.asScopeID(scopeID) + const eventID = `lte_${createHash("sha256") + .update( + JSON.stringify({ + runID: run.id, + kind: input.kind, + stepID: input.stepID, + state: input.state, + stateRevision: run.stateRevision, + pathwayRevision: run.pathwayRevision, + message: input.message, + data: input.data, + }), + ) + .digest("hex") + .slice(0, 26)}` + + const existing = await readOptional(StoragePath.latticeEvent(sid, run.id, eventID)) + if (existing !== undefined) return LatticeTypes.EventInfo.parse(existing) + const event = LatticeTypes.EventInfo.parse({ + id: eventID, + runID: run.id, + scopeID, + sessionID: run.sessionID, + kind: input.kind, + stepID: input.stepID, + state: input.state, + message: input.message, + data: input.data, + time: { created: Date.now() }, + }) + await Storage.write(StoragePath.latticeEvent(sid, run.id, event.id), event) + await Bus.publish(LatticeEvent.EventAppended, { event }) + return event }) - await Storage.write(StoragePath.latticeEvent(sid, run.id, event.id), event) - await Bus.publish(LatticeEvent.EventAppended, { event }) - return event } export async function listEvents(scopeID: string, runID: string): Promise { diff --git a/packages/workflows/src/superplan/store.ts b/packages/workflows/src/superplan/store.ts index e633c231b..aae969c12 100644 --- a/packages/workflows/src/superplan/store.ts +++ b/packages/workflows/src/superplan/store.ts @@ -33,61 +33,63 @@ export namespace SuperPlanStore { nodes?: NodeCreateInput[] merges?: MergeCreateInput[] }): Promise { - const scopeID = ScopeContext.current.scope.id - const sid = Identifier.asScopeID(scopeID) - const now = Date.now() - const runID = Identifier.ascending("superplan_run") + return Storage.transaction(async () => { + const scopeID = ScopeContext.current.scope.id + const sid = Identifier.asScopeID(scopeID) + const now = Date.now() + const runID = Identifier.ascending("superplan_run") - const nodes = (input.nodes ?? []).map( - (node): SuperPlanTypes.Node => - SuperPlanTypes.Node.parse({ - id: node.id ?? Identifier.ascending("superplan_node"), - runID, - title: node.title, - description: node.description, - deps: node.deps ?? [], - blueprintNoteID: node.blueprintNoteID, - baseCommit: node.baseCommit ?? input.baseCommit, - status: "pending", - time: { created: now, updated: now }, - }), - ) + const nodes = (input.nodes ?? []).map( + (node): SuperPlanTypes.Node => + SuperPlanTypes.Node.parse({ + id: node.id ?? Identifier.ascending("superplan_node"), + runID, + title: node.title, + description: node.description, + deps: node.deps ?? [], + blueprintNoteID: node.blueprintNoteID, + baseCommit: node.baseCommit ?? input.baseCommit, + status: "pending", + time: { created: now, updated: now }, + }), + ) - const merges = (input.merges ?? []).map( - (merge): SuperPlanTypes.Merge => - SuperPlanTypes.Merge.parse({ - id: merge.id ?? Identifier.ascending("superplan_merge"), - runID, - wave: merge.wave, - inputNodeIDs: merge.inputNodeIDs, - inputCommits: merge.inputCommits ?? [], - baseCommit: merge.baseCommit ?? input.baseCommit, - status: "pending", - time: { created: now, updated: now }, - }), - ) + const merges = (input.merges ?? []).map( + (merge): SuperPlanTypes.Merge => + SuperPlanTypes.Merge.parse({ + id: merge.id ?? Identifier.ascending("superplan_merge"), + runID, + wave: merge.wave, + inputNodeIDs: merge.inputNodeIDs, + inputCommits: merge.inputCommits ?? [], + baseCommit: merge.baseCommit ?? input.baseCommit, + status: "pending", + time: { created: now, updated: now }, + }), + ) - const run = SuperPlanTypes.Run.parse({ - id: runID, - scopeID, - title: input.title, - description: input.description, - status: "planning", - plannerSessionID: input.plannerSessionID, - summarySessionID: input.summarySessionID, - baseCommit: input.baseCommit, - nodes, - merges, - time: { created: now, updated: now }, - }) + const run = SuperPlanTypes.Run.parse({ + id: runID, + scopeID, + title: input.title, + description: input.description, + status: "planning", + plannerSessionID: input.plannerSessionID, + summarySessionID: input.summarySessionID, + baseCommit: input.baseCommit, + nodes, + merges, + time: { created: now, updated: now }, + }) - await Storage.write(StoragePath.superPlanRun(sid, runID), run) - await Bus.publish(SuperPlanEvent.Created, { run }) - await appendEvent(scopeID, runID, { - kind: "run_created", - message: `SuperPlan run created: ${run.title}`, + await Storage.write(StoragePath.superPlanRun(sid, runID), run) + await Bus.publish(SuperPlanEvent.Created, { run }) + await appendEvent(scopeID, runID, { + kind: "run_created", + message: `SuperPlan run created: ${run.title}`, + }) + return run }) - return run } export async function get(scopeID: string, runID: string): Promise { @@ -108,15 +110,17 @@ export namespace SuperPlanStore { runID: string, editor: (run: SuperPlanTypes.Run) => void, ): Promise { - const sid = Identifier.asScopeID(scopeID) - const run = await Storage.update(StoragePath.superPlanRun(sid, runID), (draft) => { - editor(draft) - draft.time.updated = Date.now() + return Storage.transaction(async () => { + const sid = Identifier.asScopeID(scopeID) + const run = await Storage.update(StoragePath.superPlanRun(sid, runID), (draft) => { + editor(draft) + draft.time.updated = Date.now() + }) + const parsed = SuperPlanTypes.Run.parse(run) + await Storage.write(StoragePath.superPlanRun(sid, runID), parsed) + await Bus.publish(SuperPlanEvent.Updated, { run: parsed }) + return parsed }) - const parsed = SuperPlanTypes.Run.parse(run) - await Storage.write(StoragePath.superPlanRun(sid, runID), parsed) - await Bus.publish(SuperPlanEvent.Updated, { run: parsed }) - return parsed } export async function appendEvent( @@ -130,21 +134,23 @@ export namespace SuperPlanStore { data?: Record }, ): Promise { - const sid = Identifier.asScopeID(scopeID) - const event = SuperPlanTypes.EventInfo.parse({ - id: Identifier.ascending("superplan_event"), - runID, - scopeID, - kind: input.kind, - nodeID: input.nodeID, - mergeID: input.mergeID, - message: input.message, - data: input.data, - time: { created: Date.now() }, + return Storage.transaction(async () => { + const sid = Identifier.asScopeID(scopeID) + const event = SuperPlanTypes.EventInfo.parse({ + id: Identifier.ascending("superplan_event"), + runID, + scopeID, + kind: input.kind, + nodeID: input.nodeID, + mergeID: input.mergeID, + message: input.message, + data: input.data, + time: { created: Date.now() }, + }) + await Storage.write(StoragePath.superPlanEvent(sid, runID, event.id), event) + await Bus.publish(SuperPlanEvent.EventAppended, { event }) + return event }) - await Storage.write(StoragePath.superPlanEvent(sid, runID, event.id), event) - await Bus.publish(SuperPlanEvent.EventAppended, { event }) - return event } export async function listEvents(scopeID: string, runID: string): Promise { diff --git a/packages/workflows/test/blueprint/storage-atomicity.test.ts b/packages/workflows/test/blueprint/storage-atomicity.test.ts new file mode 100644 index 000000000..8879542f9 --- /dev/null +++ b/packages/workflows/test/blueprint/storage-atomicity.test.ts @@ -0,0 +1,27 @@ +import { expect, spyOn, test } from "bun:test" +import { tmpdir } from "@ericsanchezok/synergy-harness/test/support/fixture" +import { ScopeContext } from "@ericsanchezok/synergy-harness/scope/context" +import { Storage } from "@ericsanchezok/synergy-harness/storage/storage" +import { NoteStore } from "@ericsanchezok/synergy-note" +import { BlueprintLoopStore } from "../../src/blueprint/loop-store" + +test("Blueprint creation rolls back when the Note projection cannot commit", async () => { + await using tmp = await tmpdir({ git: true }) + const scope = await tmp.scope() + await ScopeContext.provide({ + scope, + fn: async () => { + const note = await NoteStore.create({ title: "atomic blueprint", kind: "blueprint" }) + const write = Storage.write + using failure = spyOn(Storage, "write").mockImplementation(async (key, value) => { + if (key[0] === "notes" && key.at(-1) === "_index") throw new Error("projection failed") + return write(key, value) + }) + await expect(BlueprintLoopStore.create({ noteID: note.id, title: "run", sessionID: "ses_test" })).rejects.toThrow( + "projection failed", + ) + expect(await BlueprintLoopStore.list(scope.id)).toEqual([]) + expect((await NoteStore.get(scope.id, note.id)).blueprint?.activeLoopID).toBeUndefined() + }, + }) +}) diff --git a/packages/workflows/test/lattice/run-service.test.ts b/packages/workflows/test/lattice/run-service.test.ts index 6c98895f5..df5b9e8b2 100644 --- a/packages/workflows/test/lattice/run-service.test.ts +++ b/packages/workflows/test/lattice/run-service.test.ts @@ -285,9 +285,9 @@ describe("LatticeRunService v2", () => { const scopeID = ScopeContext.current.scope.id const pointerPath = StoragePath.latticeCurrent(Identifier.asScopeID(scopeID), crashSession.id) const write = Storage.write - const pointerWrite = spyOn(Storage, "write").mockImplementation(async (key, content, options) => { + const pointerWrite = spyOn(Storage, "write").mockImplementation(async (key, content) => { if (key.join("/") === pointerPath.join("/")) throw new Error("pointer write failed") - return write(key, content, options) + return write(key, content) }) try { await expect( @@ -296,13 +296,7 @@ describe("LatticeRunService v2", () => { } finally { pointerWrite.mockRestore() } - expect(await LatticeStore.listBySession(scopeID, crashSession.id)).toMatchObject([ - { - revision: 0, - goalSeed: "Crash-safe seed", - effect: { kind: "deliver_prompt", promptType: "state_entry" }, - }, - ]) + expect(await LatticeStore.listBySession(scopeID, crashSession.id)).toEqual([]) const session = await Session.create({}) const projected = await WorkflowSessionService.enableLattice(session.id, { diff --git a/packages/workflows/test/lattice/storage-atomicity.test.ts b/packages/workflows/test/lattice/storage-atomicity.test.ts new file mode 100644 index 000000000..471133eb9 --- /dev/null +++ b/packages/workflows/test/lattice/storage-atomicity.test.ts @@ -0,0 +1,31 @@ +import { expect, spyOn, test } from "bun:test" +import { Identifier } from "@ericsanchezok/synergy-harness/id/id" +import { Scope } from "@ericsanchezok/synergy-harness/scope" +import { ScopeContext } from "@ericsanchezok/synergy-harness/scope/context" +import { Storage } from "@ericsanchezok/synergy-harness/storage/storage" +import { StoragePath } from "@ericsanchezok/synergy-harness/storage/path" +import { tmpdir } from "@ericsanchezok/synergy-harness/test/support/fixture" +import { LatticeStore } from "../../src/lattice/store" + +test("a failed current pointer cannot publish an orphan active Run", async () => { + await using tmp = await tmpdir({ git: true }) + const scope = (await Scope.fromDirectory(tmp.path)).scope + await ScopeContext.provide({ + scope, + fn: async () => { + const sessionID = Identifier.ascending("session") + const pointer = StoragePath.latticeCurrent(Identifier.asScopeID(scope.id), sessionID) + const write = Storage.write + const failure = spyOn(Storage, "write").mockImplementation(async (key, value) => { + if (JSON.stringify(key) === JSON.stringify(pointer)) throw new Error("pointer unavailable") + return write(key, value) + }) + try { + await expect(LatticeStore.create({ sessionID, mode: "auto" })).rejects.toThrow("pointer unavailable") + } finally { + failure.mockRestore() + } + expect(await LatticeStore.listBySession(scope.id, sessionID)).toEqual([]) + }, + }) +}) diff --git a/packages/workflows/test/lattice/store.test.ts b/packages/workflows/test/lattice/store.test.ts index f48d63783..7ea13daa9 100644 --- a/packages/workflows/test/lattice/store.test.ts +++ b/packages/workflows/test/lattice/store.test.ts @@ -107,7 +107,7 @@ describe("LatticeStore v2", () => { }) }) - test("quarantines the selected duplicate before an interrupted cleanup can expose its effect", async () => { + test("rolls back interrupted duplicate quarantine and retries the complete reconciliation", async () => { await withScope(async () => { const scopeID = ScopeContext.current.scope.id const sid = Identifier.asScopeID(scopeID) @@ -146,12 +146,8 @@ describe("LatticeStore v2", () => { ;(LatticeMachine.quarantineDuplicate as typeof quarantine) = quarantine } - expect(await LatticeStore.getByRunID(scopeID, second.id)).toMatchObject({ - status: "paused", - statusReason: "duplicate_active_run", - }) - expect((await LatticeStore.getByRunID(scopeID, second.id))?.effect).toBeUndefined() - expect((await LatticeStore.getByRunID(scopeID, first.id))?.status).toBe("active") + expect(await LatticeStore.getByRunID(scopeID, second.id)).toEqual(second) + expect(await LatticeStore.getByRunID(scopeID, first.id)).toEqual(firstWithEffect) await LatticeStore.listCurrent(scopeID) expect(await LatticeStore.getByRunID(scopeID, first.id)).toMatchObject({ diff --git a/packages/workflows/test/migration/lattice-v2-reset.test.ts b/packages/workflows/test/migration/lattice-v2-reset.test.ts index bf12877d9..55370f374 100644 --- a/packages/workflows/test/migration/lattice-v2-reset.test.ts +++ b/packages/workflows/test/migration/lattice-v2-reset.test.ts @@ -589,12 +589,12 @@ describe("Lattice v2 reset migration", () => { const originalWrite = Storage.write let failAuditWrite = true - using _write = spyOn(Storage, "write").mockImplementation(async (key, content, options) => { + using _write = spyOn(Storage, "write").mockImplementation(async (key, content) => { if (failAuditWrite && key.join("/") === auditSessionPath.join("/")) { failAuditWrite = false throw new Error("injected audit Session write failure") } - return originalWrite(key, content, options) + return originalWrite(key, content) }) const migration = resetMigration() diff --git a/script/dev.ts b/script/dev.ts index 693dbc2e0..775c11eb4 100644 --- a/script/dev.ts +++ b/script/dev.ts @@ -817,6 +817,7 @@ async function runPrepare(repoRoot: string, bunPath: string): Promise { if (initial !== 0) return initial const platform = process.platform + if (platform === "darwin") await (await import("../packages/harness/script/build-sqlite")).buildSqlite() if (platform === "linux") { const { buildWatcher } = await import("../packages/runtime-local/script/build-watcher") await buildWatcher() diff --git a/script/release/shared/runtime-assets.ts b/script/release/shared/runtime-assets.ts index 0a0bade71..a90c3bf68 100644 --- a/script/release/shared/runtime-assets.ts +++ b/script/release/shared/runtime-assets.ts @@ -1,3 +1,4 @@ +import { buildSqlite } from "../../../packages/harness/script/build-sqlite" import { buildWatcher } from "../../../packages/runtime-local/script/build-watcher" import { existsSync } from "node:fs" import fs from "node:fs/promises" @@ -78,6 +79,7 @@ export async function prepareRuntimeAssets(name: string, profile: RuntimeArtifac const dependencies = await runtimeDependencies(profile) const { targetOs, targetArch, musl } = runtimeTarget(name) + if (targetOs === "darwin") await fs.copyFile(await buildSqlite(), path.join(runtimeDir, "libsqlite3.dylib")) if (musl) { await removeUnsupportedMuslAssets(runtimeDir) console.warn(`Skipping ast-grep and sqlite-vec for ${name}; no musl-compatible release assets are available`) diff --git a/script/release/shared/runtime-contract.ts b/script/release/shared/runtime-contract.ts index 1de3a6210..f9f7e1707 100644 --- a/script/release/shared/runtime-contract.ts +++ b/script/release/shared/runtime-contract.ts @@ -23,6 +23,7 @@ export function requiredRuntimeArtifactPaths(name: string, profile: RuntimeArtif const sqliteVec = target.os === "windows" ? "vec0.dll" : target.os === "darwin" ? "vec0.dylib" : "vec0.so" return [ binary, + ...(target.os === "darwin" ? ["libsqlite3.dylib"] : []), ...(!target.musl && profile === "full" ? [astGrep] : []), ...(!target.musl && profile === "full" ? [sqliteVec] : []), // The watcher binding ships for every target: @parcel/watcher publishes From 199edc7b93118eb136c3b1a768788bace29d6276 Mon Sep 17 00:00:00 2001 From: EricSanchezok <115814526+EricSanchezok@users.noreply.github.com> Date: Mon, 14 Sep 2026 21:28:41 +0800 Subject: [PATCH 02/14] fix(storage): complete explicit migration activation Activate verified authority after a successful full offline migration and initialize the isolated Library migration fixture with its own storage handle. Co-authored-by: synergy-agent <299070056+synergy-agent@users.noreply.github.com> --- packages/cli/src/main.ts | 13 ++++++++++++- packages/harness/src/storage/maintenance.ts | 12 ++++++++---- packages/library/test/migration-tracking.test.ts | 2 ++ 3 files changed, 22 insertions(+), 5 deletions(-) diff --git a/packages/cli/src/main.ts b/packages/cli/src/main.ts index dbbde8f9d..ca55bde09 100644 --- a/packages/cli/src/main.ts +++ b/packages/cli/src/main.ts @@ -228,7 +228,18 @@ async function runCliImplementation(options: CliOptions): Promise { process.on("uncaughtException", onException) try { if (argv.length === 0 && !options.defaultCommand) cli.showHelp() - else await cli.parse() + else { + const parsed = await cli.parse() + if ( + parsed._.length === 2 && + parsed._[0] === "migration" && + parsed._[1] === "run" && + !parsed["dry-run"] && + storage && + "activate" in storage + ) + await storage.activate() + } } catch (e) { let data: Record = {} if (e instanceof NamedError) { diff --git a/packages/harness/src/storage/maintenance.ts b/packages/harness/src/storage/maintenance.ts index 1eb7b786c..620e50a25 100644 --- a/packages/harness/src/storage/maintenance.ts +++ b/packages/harness/src/storage/maintenance.ts @@ -44,13 +44,17 @@ export namespace StorageMaintenance { const handle = { store: prepared.store, artifactDirectory: Global.Path.data } uninstall = Storage.install(handle) await SessionStaging.recover() + const pending = prepared + const activate = async () => { + if (pending.manifest.phase !== "active") await StorageRecovery.validate() + await pending.activate() + await StorageRecovery.recoverOwners() + } if (options.migrate !== false) { await ensureMigrations({ output: "silent" }) - if (prepared.manifest.phase !== "active") await StorageRecovery.validate() - await prepared.activate() - await StorageRecovery.recoverOwners() + await activate() } - return { ...handle, manifest: prepared.manifest, close, [Symbol.asyncDispose]: close } + return { ...handle, manifest: prepared.manifest, activate, close, [Symbol.asyncDispose]: close } } catch (error) { await close() throw error diff --git a/packages/library/test/migration-tracking.test.ts b/packages/library/test/migration-tracking.test.ts index 8f1bdcc10..9f6dcb4a7 100644 --- a/packages/library/test/migration-tracking.test.ts +++ b/packages/library/test/migration-tracking.test.ts @@ -13,6 +13,8 @@ test("legacy Library tracking stays untouched until its owner registers", async const { runMigrations } = await import("@ericsanchezok/synergy-harness/migration") const { Storage } = await import("@ericsanchezok/synergy-harness/storage/storage") const { StoragePath } = await import("@ericsanchezok/synergy-harness/storage/path") + const { StorageMaintenance } = await import("@ericsanchezok/synergy-harness/storage/maintenance") + await using maintenance = await StorageMaintenance.open({ migrate: false }) const oldKey = StoragePath.metaMigrationLogDomain("engram") const newKey = StoragePath.metaMigrationLogDomain("library") const old = { "20260324-engram-experience-source-model": 42, "unknown-engram-step": 24 } From 346c4787caaa17c9188f5b7223dfabbbdadb8412 Mon Sep 17 00:00:00 2001 From: EricSanchezok <115814526+EricSanchezok@users.noreply.github.com> Date: Mon, 14 Sep 2026 21:31:04 +0800 Subject: [PATCH 03/14] fix(storage): fence search rebuilds with record revisions Use durable revisions instead of millisecond timestamps when clearing search invalidation, and exercise offline migration activation through the real CLI. Co-authored-by: synergy-agent <299070056+synergy-agent@users.noreply.github.com> --- .../cli/test/cli/storage-maintenance.test.ts | 29 ++++++++++++++ packages/harness/src/session/search-index.ts | 40 +++++++++++++------ .../runtime-local/src/tools/session-search.ts | 8 ++-- .../test/session/search-index.test.ts | 17 +++++++- 4 files changed, 75 insertions(+), 19 deletions(-) create mode 100644 packages/cli/test/cli/storage-maintenance.test.ts diff --git a/packages/cli/test/cli/storage-maintenance.test.ts b/packages/cli/test/cli/storage-maintenance.test.ts new file mode 100644 index 000000000..3ce26d058 --- /dev/null +++ b/packages/cli/test/cli/storage-maintenance.test.ts @@ -0,0 +1,29 @@ +import { expect, test } from "bun:test" +import { createIsolatedTestEnv } from "@ericsanchezok/synergy-testing/env" + +test("a full offline migration activates authority and subsequent read-only status succeeds", async () => { + const isolated = await createIsolatedTestEnv() + try { + const entry = new URL("../../src/index.ts", import.meta.url).pathname + for (const args of [ + ["migration", "run"], + ["data", "storage", "status"], + ]) { + const child = Bun.spawn({ + cmd: [process.execPath, "run", entry, ...args], + env: isolated.env, + stdout: "pipe", + stderr: "pipe", + }) + const [code, stdout, stderr] = await Promise.all([ + child.exited, + new Response(child.stdout).text(), + new Response(child.stderr).text(), + ]) + expect(code, stderr).toBe(0) + if (args[0] === "data") expect(JSON.parse(stdout).phase).toBe("active") + } + } finally { + await isolated.dispose() + } +}, 30000) diff --git a/packages/harness/src/session/search-index.ts b/packages/harness/src/session/search-index.ts index 4a5848f2f..6453ccba3 100644 --- a/packages/harness/src/session/search-index.ts +++ b/packages/harness/src/session/search-index.ts @@ -7,18 +7,17 @@ import { Log } from "../util/log" /** * Versioned per-session search index (session-search P2). The index is an * optimization: a query reads one compact record per clean session instead of - * streaming every message file. Correctness never depends on it — dirty, + * streaming every message record. Correctness never depends on it — dirty, * missing, or stale records fall back to the existing message scan path and * are rebuilt write-through. * * Write path: message/session mutations mark the owning session dirty via a - * tiny marker file; the next search rebuilds lazily. Part streaming deltas + * small marker record; the next search rebuilds lazily. Part streaming deltas * (updatePartDelta) deliberately do NOT mark dirty — text/tool parts settle * into a message before Session.updateMessage fires, which is the single hook. * * Concurrency: markDirty (message writers) and commit/rebuild (search queries) - * serialize on a per-session lock so a marker can never be cleared after a - * newer write — the last writer wins and any interleaved rebuild observes it. + * use SQL revisions so a rebuild cannot clear a marker changed during its scan. */ export namespace SessionSearchIndex { const log = Log.create({ service: "session.search-index" }) @@ -213,7 +212,10 @@ export namespace SessionSearchIndex { ): Promise { const record = await Storage.read(recordKey(scopeID, sessionID), { silentNotFound: true, - }).catch(() => undefined) + }).catch((error) => { + if (error instanceof Storage.NotFoundError) return + throw error + }) // A record from an older format lacks current semantics (e.g. nested tool // attachments); treat it as absent so the next query rescans and rebuilds // instead of trusting content that no longer matches the scan path. @@ -223,7 +225,10 @@ export namespace SessionSearchIndex { export async function isDirty(scopeID: Identifier.ScopeID, sessionID: Identifier.SessionID): Promise { const marker = await Storage.read(dirtyKey(scopeID, sessionID), { silentNotFound: true }).catch( - () => undefined, + (error) => { + if (error instanceof Storage.NotFoundError) return + throw error + }, ) return marker !== undefined } @@ -232,11 +237,22 @@ export namespace SessionSearchIndex { await Storage.write(dirtyKey(scopeID, sessionID), { dirtyAt: Date.now() } satisfies DirtyMarker) } + export async function dirtyRevision(scopeID: Identifier.ScopeID, sessionID: Identifier.SessionID): Promise { + return Storage.snapshot(async (tx) => { + try { + return (await tx.versioned(dirtyKey(scopeID, sessionID))).revision + } catch (error) { + if (error instanceof Storage.NotFoundError) return 0n + throw error + } + }) + } + export async function commitRebuild( scopeID: Identifier.ScopeID, sessionID: Identifier.SessionID, messages: IndexedMessage[], - opts?: { sinceMs?: number }, + opts: { revision: bigint }, ): Promise { await Storage.transaction(async (tx) => { const record: SearchIndexRecord = { @@ -248,9 +264,7 @@ export namespace SessionSearchIndex { messages, } await tx.write(recordKey(scopeID, sessionID), record) - const [marker] = await tx.readMany([dirtyKey(scopeID, sessionID)]) - if (!marker || (opts?.sinceMs !== undefined && marker.dirtyAt < opts.sinceMs)) - await tx.remove(dirtyKey(scopeID, sessionID)) + if ((await dirtyRevision(scopeID, sessionID)) === opts.revision) await tx.remove(dirtyKey(scopeID, sessionID)) }) } @@ -264,20 +278,20 @@ export namespace SessionSearchIndex { /** * Rebuild a session's index record from its persisted messages and parts via * the canonical MessageV2 read path, then clear its dirty marker (guarded by - * the rebuild start time). Returns the fresh record. + * the dirty record revision). Returns the fresh record. */ export async function rebuildSession( scopeID: Identifier.ScopeID, sessionID: Identifier.SessionID, ): Promise { - const startedAt = Date.now() + const revision = await dirtyRevision(scopeID, sessionID) const infos = await MessageV2.readInfoList({ scopeID, sessionID }) const messages: IndexedMessage[] = [] for (const info of infos) { const parts = await MessageV2.parts({ scopeID, sessionID, messageID: info.id }) messages.push(messageEntryFromParts(info, parts)) } - await commitRebuild(scopeID, sessionID, messages, { sinceMs: startedAt }) + await commitRebuild(scopeID, sessionID, messages, { revision }) log.debug("rebuilt session search index", { scopeID, sessionID, messages: messages.length }) return { version: VERSION, diff --git a/packages/runtime-local/src/tools/session-search.ts b/packages/runtime-local/src/tools/session-search.ts index 1c995661a..ff96c893c 100644 --- a/packages/runtime-local/src/tools/session-search.ts +++ b/packages/runtime-local/src/tools/session-search.ts @@ -458,7 +458,7 @@ async function searchSessions(params: z.infer, ctx: Tool.Cont // been seen — then persist the rebuilt record. Because the scan always // covers the full session, the committed record is never partial. scannedSessions++ - const startedAt = Date.now() + const revision = await SessionSearchIndex.dirtyRevision(scopeID, sessionID) const indexEntries: SessionSearchIndex.IndexedMessage[] = [] for await (const msg of MessageV2.stream({ scopeID, sessionID })) { ctx.abort.throwIfAborted() @@ -476,10 +476,8 @@ async function searchSessions(params: z.infer, ctx: Tool.Cont phase: "tool.session_search.progress", }) } - // Race guard: only clear a dirty marker written before this scan began; - // a marker landed mid-scan means content changed under us and must - // survive to force another pass. - await SessionSearchIndex.commitRebuild(scopeID, sessionID, indexEntries, { sinceMs: startedAt }) + // Revision equality preserves writes that arrive during the scan, including within the same millisecond. + await SessionSearchIndex.commitRebuild(scopeID, sessionID, indexEntries, { revision }) } for (const match of matches) ranked.push({ match, session }) diff --git a/packages/runtime-local/test/session/search-index.test.ts b/packages/runtime-local/test/session/search-index.test.ts index ace9f0490..4f68ff398 100644 --- a/packages/runtime-local/test/session/search-index.test.ts +++ b/packages/runtime-local/test/session/search-index.test.ts @@ -1,4 +1,4 @@ -import { describe, expect, test } from "bun:test" +import { describe, expect, spyOn, test } from "bun:test" import { tmpdir } from "@ericsanchezok/synergy-harness/test/support/fixture" import { Identifier } from "@ericsanchezok/synergy-harness/id/id" import { ScopeContext } from "@ericsanchezok/synergy-harness/scope/context" @@ -435,3 +435,18 @@ describe("session.search-index", () => { }) }) }) + +test("a rebuild cannot clear a new dirty write within the same millisecond", async () => { + const scopeID = Identifier.asScopeID("search_revision_scope") + const sessionID = Identifier.asSessionID("ses_search_revision") + using clock = spyOn(Date, "now").mockReturnValue(1000) + await SessionSearchIndex.markDirty(scopeID, sessionID) + const revision = await SessionSearchIndex.dirtyRevision(scopeID, sessionID) + await SessionSearchIndex.markDirty(scopeID, sessionID) + await SessionSearchIndex.commitRebuild(scopeID, sessionID, [], { revision }) + expect(await SessionSearchIndex.isDirty(scopeID, sessionID)).toBe(true) + await SessionSearchIndex.commitRebuild(scopeID, sessionID, [], { + revision: await SessionSearchIndex.dirtyRevision(scopeID, sessionID), + }) + expect(await SessionSearchIndex.isDirty(scopeID, sessionID)).toBe(false) +}) From a6848156d2dbab4f7dfa03fbf4abe9bc62336343 Mon Sep 17 00:00:00 2001 From: EricSanchezok <115814526+EricSanchezok@users.noreply.github.com> Date: Mon, 14 Sep 2026 21:37:06 +0800 Subject: [PATCH 04/14] fix(storage): initialize snapshot engines at use boundaries Keep bundled workflow loading independent of SQLite initialization and verify its real worker and storage lifecycle in a compiled isolated fixture. Align inbox assertions with atomic transcript publication and queue consumption. Co-authored-by: synergy-agent <299070056+synergy-agent@users.noreply.github.com> --- packages/harness/src/session/snapshot-transfer.ts | 3 +-- .../workflows/test/session/bundled-workflow.test.ts | 12 ++++++++++-- .../test/session/fixture/bundled-workflow.ts | 6 ++++++ packages/workflows/test/session/inbox.test.ts | 2 +- 4 files changed, 18 insertions(+), 5 deletions(-) diff --git a/packages/harness/src/session/snapshot-transfer.ts b/packages/harness/src/session/snapshot-transfer.ts index cfba484bb..c2e8603db 100644 --- a/packages/harness/src/session/snapshot-transfer.ts +++ b/packages/harness/src/session/snapshot-transfer.ts @@ -6,8 +6,6 @@ import { Global } from "../global" import { SnapshotGit } from "./snapshot-git" import { SnapshotStore } from "./snapshot-store" -initializeSqliteEngine() - export namespace SnapshotTransfer { export async function recoverImports(target: string, signal?: AbortSignal) { const packs = path.join(target, "objects", "pack") @@ -57,6 +55,7 @@ export namespace SnapshotTransfer { readonly target: string, readonly directory: string, ) { + initializeSqliteEngine() this.db = new Database(path.join(directory, "inventory.sqlite")) this.db.exec( "PRAGMA journal_mode=MEMORY; PRAGMA synchronous=OFF; CREATE TABLE known (oid TEXT PRIMARY KEY); CREATE TABLE incoming (oid TEXT PRIMARY KEY, type TEXT NOT NULL); CREATE TABLE covered (oid TEXT PRIMARY KEY)", diff --git a/packages/workflows/test/session/bundled-workflow.test.ts b/packages/workflows/test/session/bundled-workflow.test.ts index 3427dcb27..a3151c396 100644 --- a/packages/workflows/test/session/bundled-workflow.test.ts +++ b/packages/workflows/test/session/bundled-workflow.test.ts @@ -1,3 +1,4 @@ +import fs from "node:fs/promises" import { expect, test } from "bun:test" import path from "node:path" import { createIsolatedTestEnv } from "@ericsanchezok/synergy-testing/env" @@ -6,13 +7,20 @@ test("bundled workflow delegates idle detection and clearing to the shared sessi const isolation = await createIsolatedTestEnv() try { const directory = isolation.env.SYNERGY_TEST_ROOT! - const output = path.join(directory, "workflow.js") + const output = path.join(directory, "bin", process.platform === "win32" ? "workflow.exe" : "workflow") + await fs.mkdir(path.dirname(output), { recursive: true }) + if (process.platform === "darwin") + await fs.copyFile( + path.resolve(import.meta.dir, "../../../harness/.artifacts/sqlite/libsqlite3.dylib"), + path.join(directory, "libsqlite3.dylib"), + ) const build = Bun.spawn( [ process.execPath, "build", path.join(import.meta.dir, "fixture/bundled-workflow.ts"), "--target=bun", + "--compile", "--outfile", output, ], @@ -24,7 +32,7 @@ test("bundled workflow delegates idle detection and clearing to the shared sessi ) const [code, error] = await Promise.all([build.exited, new Response(build.stderr).text()]) expect(code, error).toBe(0) - const run = Bun.spawn([process.execPath, output], { + const run = Bun.spawn([output], { cwd: directory, env: { ...isolation.env, SYNERGY_DISABLE_MODELS_FETCH: "true" }, stdout: "pipe", diff --git a/packages/workflows/test/session/fixture/bundled-workflow.ts b/packages/workflows/test/session/fixture/bundled-workflow.ts index 1d8417312..8e4998e64 100644 --- a/packages/workflows/test/session/fixture/bundled-workflow.ts +++ b/packages/workflows/test/session/fixture/bundled-workflow.ts @@ -4,7 +4,13 @@ import { Session } from "@ericsanchezok/synergy-harness/session" import { WorkflowSessionService } from "../../../src/session/workflow" import { registerWorkflowSessions } from "../../../src/session/register" +if (process.argv.includes("__storage-worker-runner")) { + await import("@ericsanchezok/synergy-harness/storage/sqlite-worker") + await new Promise(() => {}) +} registerWorkflowSessions() +const { StorageMaintenance } = await import("@ericsanchezok/synergy-harness/storage/maintenance") +await using maintenance = await StorageMaintenance.open() const { scope } = await Scope.fromDirectory(process.cwd()) await ScopeContext.provide({ scope, diff --git a/packages/workflows/test/session/inbox.test.ts b/packages/workflows/test/session/inbox.test.ts index 012b0b6d3..d9349d814 100644 --- a/packages/workflows/test/session/inbox.test.ts +++ b/packages/workflows/test/session/inbox.test.ts @@ -58,7 +58,7 @@ describe("SessionInbox", () => { expect(first?.info.id).toBe(queued.messageID) expect(retry?.info.id).toBe(queued.messageID) - expect((await SessionInbox.list(session.id)).map((item) => item.id)).toEqual([queued.id]) + expect(await SessionInbox.list(session.id)).toEqual([]) expect((await Session.messages({ sessionID: session.id })).map((message) => message.info.id)).toEqual([ queued.messageID, ]) From 516f1aa979f02eeb835603365d0af78100af6565 Mon Sep 17 00:00:00 2001 From: EricSanchezok <115814526+EricSanchezok@users.noreply.github.com> Date: Mon, 14 Sep 2026 22:04:07 +0800 Subject: [PATCH 05/14] fix(storage): close lifecycle and migration edge cases Co-authored-by: synergy-agent <299070056+synergy-agent@users.noreply.github.com> --- benchmark/runtime/inspect.ts | 4 +- benchmark/test/runtime.test.ts | 39 +++++++++-- docs/architecture/agent-storage.md | 6 +- packages/cli/src/cli/cmd/data/shared.ts | 58 +++++++++++---- packages/cli/test/cli/data-files.test.ts | 19 +++++ .../test/channel/diagnostics.test.ts | 28 +++----- packages/harness/src/session/index.ts | 59 +++++++++------- .../src/storage/transactional-store.ts | 16 +++-- packages/harness/test/storage/context.test.ts | 18 +++++ .../test/config/config.test.ts | 16 ++++- packages/workbench/src/push/store.ts | 70 ++++++++++--------- packages/workbench/test/push/bridge.test.ts | 12 +++- packages/workbench/test/push/service.test.ts | 12 +++- packages/workbench/test/push/store.test.ts | 31 ++++++-- test/package/fixture/runtime-composition.ts | 26 +++---- 15 files changed, 290 insertions(+), 124 deletions(-) diff --git a/benchmark/runtime/inspect.ts b/benchmark/runtime/inspect.ts index 1d2f71215..113ad25a8 100644 --- a/benchmark/runtime/inspect.ts +++ b/benchmark/runtime/inspect.ts @@ -17,6 +17,7 @@ async function inspect() { Config.schema().strict().parse(config) const experiment = process.argv[4] ? Experiment.File.parse(await Bun.file(process.argv[4]).json()) : undefined let measured: unknown + const runtime = process.argv[5] ? await composition.open({ mode: "oneshot" }) : undefined try { if (process.argv[5]) measured = await ScopeContext.provide({ @@ -60,7 +61,8 @@ async function inspect() { }), ) } finally { - await ScopeRuntime.disposeAll() + if (runtime) await runtime.close() + else await ScopeRuntime.disposeAll() } } diff --git a/benchmark/test/runtime.test.ts b/benchmark/test/runtime.test.ts index b3df712b3..857dfc674 100644 --- a/benchmark/test/runtime.test.ts +++ b/benchmark/test/runtime.test.ts @@ -3,14 +3,22 @@ import { mkdtemp, rm } from "node:fs/promises" import os from "node:os" import path from "node:path" -async function inspect(runtime: string, config: Record = {}) { +async function inspect(runtime: string, config: Record = {}, model?: string) { const home = await mkdtemp(path.join(os.tmpdir(), "synergy-bench-contract-")) try { const file = path.join(home, "config.json") await Bun.write(file, JSON.stringify(config)) - const child = Bun.spawn([process.execPath, "runtime/inspect.ts", runtime, file], { + const child = Bun.spawn([process.execPath, "runtime/inspect.ts", runtime, file, "", model ?? "", "synergy"], { cwd: path.resolve(import.meta.dir, ".."), - env: { PATH: process.env.PATH, SYNERGY_HOME: home, SYNERGY_CONFIG_CONTENT: "{}" }, + env: { + PATH: process.env.PATH, + SYNERGY_HOME: home, + SYNERGY_CONFIG: file, + SYNERGY_CONFIG_CONTENT: "{}", + SYNERGY_DISABLE_MODELS_FETCH: "1", + SYNERGY_DISABLE_DEFAULT_PLUGINS: "1", + MODELS_DEV_API_JSON: path.resolve(import.meta.dir, "../../packages/testing/fixtures/models-api.json"), + }, stdout: "pipe", stderr: "pipe", }) @@ -82,4 +90,27 @@ test("offline preflight rejects an unavailable measured model before inference", } finally { await rm(home, { recursive: true, force: true }) } -}) +}, 30000) + +test("full preflight measures a configured model with an owned transactional runtime", async () => { + const result = await inspect( + "full", + { + execution: { agentWorkerMinIdle: 0 }, + pluginMarketplace: { enabled: false }, + provider: { + fixture: { + name: "Fixture", + npm: "@ai-sdk/openai-compatible", + env: [], + options: { baseURL: "http://127.0.0.1:1/v1", apiKey: "fixture-only" }, + models: { fixture: { name: "Fixture", tool_call: true, limit: { context: 128000, output: 4096 } } }, + }, + }, + }, + "fixture/fixture", + ) + expect(result.code, result.stderr).toBe(0) + const report = JSON.parse(result.stdout) + expect(report.measured.model.id).toBe("fixture") +}, 30000) diff --git a/docs/architecture/agent-storage.md b/docs/architecture/agent-storage.md index f99025699..bcf2b3a00 100644 --- a/docs/architecture/agent-storage.md +++ b/docs/architecture/agent-storage.md @@ -26,7 +26,7 @@ Plugin installation has a durable recovery intent and a private snapshot of the ## Streaming and notifications -Streaming part writes coalesce for 500 ms. Terminal writes wait for prior in-flight writes; drain boundaries include timer-triggered writes and propagate persistence failures. Buffered values and caches belong to the Storage Handle. Part persistence verifies the owning Session and Message and rejects a deleted part, preventing late writes from resurrecting removed data. +Streaming part writes coalesce for 500 ms. Terminal writes wait for prior in-flight writes; drain boundaries include timer-triggered writes and propagate persistence failures. Buffered values and caches belong to the Storage Handle. Part persistence verifies the owning Session and Message and rejects a deleted part. Every ordinary Session-owned record write also checks the Session tombstone in the same transaction, so detached Rollout settlement cannot revive deleted data. Explicit portable recovery is the only path that may restore a deleted record. State notifications enter a durable SQL outbox in the business transaction. Publication and cache changes happen after commit. An observer failure cannot roll back an already committed mutation. A new Runtime changes the frontend event epoch and reconciles outstanding notifications by requiring a fresh snapshot; it does not replay arbitrary subscribers that might perform external actions. Stream deltas remain provisional until their persistence boundary completes. @@ -51,3 +51,7 @@ See [storage and paths](../reference/storage-and-paths.md) for locations and [tr `bun packages/harness/script/benchmark-storage.ts` measures 1,000 transactions with two records each, a 1 KiB payload, 32 concurrent callers and a 100-record page read. A local macOS run with Bun 1.3.14 and PostgreSQL 16 in Docker measured SQLite at 1,186 transactions/s (queued p95 31.76 ms, page read 1.50 ms) and PostgreSQL at 134 transactions/s (queued p95 236.98 ms, page read 1.91 ms). The run shared the host with build/test processes. These are reproducible development measurements, not production capacity guarantees or a comparison against the old JSON writer. PostgreSQL throughput currently includes namespace serialization and ownership checks. Historical upgrade fixtures reconstruct the published v1.2.33, v2.4.4 and v3.0.22 writer formats with their exact source commits. See the [fixture provenance](../../packages/harness/test/storage/fixtures/README.md). Fault tests cover worker crashes, owner loss, rollback, stale revisions, ambiguous commit receipts, malformed legacy records, interrupted target activation, unpublished Session recovery and plugin installation recovery. + +Recursive key traversal drives each step from the current frontier into the `(namespace, parent_id)` index. SQLite requires this join order to avoid a namespace scan for every descendant; the retention and long Rollout contracts cover broad trees and permanent deletion. Push subscriptions use authority records; the VAPID signing key remains in the private credential file and is preserved across migration. + +Data transfer copies file contents to a temporary sibling, synchronizes them, then publishes without replacing an existing destination. Destination directory links are rejected and interrupted temporary copies are excluded from subsequent copies. A successful transfer therefore cannot reference a partially copied file. diff --git a/packages/cli/src/cli/cmd/data/shared.ts b/packages/cli/src/cli/cmd/data/shared.ts index 8daa2d445..1f3130e59 100644 --- a/packages/cli/src/cli/cmd/data/shared.ts +++ b/packages/cli/src/cli/cmd/data/shared.ts @@ -1,3 +1,4 @@ +import { randomUUID } from "node:crypto" import fs from "fs/promises" import fsSync from "fs" import path from "path" @@ -195,11 +196,26 @@ export async function copyDirSkipExisting( // Shared mutable counters so recursive calls accumulate correctly const acc = { copied: 0, skipped: 0 } + async function exists(filename: string) { + return fs.lstat(filename).then( + () => true, + (error: NodeJS.ErrnoException) => { + if (error.code === "ENOENT") return false + throw error + }, + ) + } + async function walk(currentSrc: string, currentDst: string) { + if (await exists(currentDst)) { + if ((await fs.lstat(currentDst)).isSymbolicLink()) + throw new Error("Data copy cannot traverse a destination symbolic link") + } await fs.mkdir(currentDst, { recursive: true }) const entries = await fs.readdir(currentSrc, { withFileTypes: true }) for (const entry of entries) { + if (entry.name.startsWith(".synergy-copy-")) continue const srcPath = path.join(currentSrc, entry.name) const dstPath = path.join(currentDst, entry.name) const relative = path.relative(src, srcPath) @@ -208,15 +224,29 @@ export async function copyDirSkipExisting( if (entry.isDirectory()) { await walk(srcPath, dstPath) } else if (entry.isFile()) { - const exists = await fs - .access(dstPath) - .then(() => true) - .catch(() => false) - if (exists) { + if (await exists(dstPath)) { acc.skipped++ } else { - await fs.copyFile(srcPath, dstPath) - acc.copied++ + const temporary = path.join(currentDst, `.synergy-copy-${randomUUID()}`) + try { + await fs.copyFile(srcPath, temporary, fsSync.constants.COPYFILE_EXCL) + const handle = await fs.open(temporary, "r") + try { + await handle.sync() + } finally { + await handle.close() + } + // Publish a complete file without replacing a concurrent destination. + try { + await fs.link(temporary, dstPath) + acc.copied++ + } catch (error) { + if (error && typeof error === "object" && "code" in error && error.code === "EEXIST") acc.skipped++ + else throw error + } + } finally { + await fs.rm(temporary, { force: true }) + } } if (onProgress && totalFiles) { onProgress({ @@ -227,11 +257,7 @@ export async function copyDirSkipExisting( }) } } else if (entry.isSymbolicLink()) { - const exists = await fs - .access(dstPath) - .then(() => true) - .catch(() => false) - if (exists) { + if (await exists(dstPath)) { acc.skipped++ } else { const linkTarget = await fs.readlink(srcPath) @@ -248,6 +274,14 @@ export async function copyDirSkipExisting( } } } + if (process.platform !== "win32") { + const directory = await fs.open(currentDst, "r") + try { + await directory.sync() + } finally { + await directory.close() + } + } } await walk(src, dst) diff --git a/packages/cli/test/cli/data-files.test.ts b/packages/cli/test/cli/data-files.test.ts index 3d32df759..c5f320e3c 100644 --- a/packages/cli/test/cli/data-files.test.ts +++ b/packages/cli/test/cli/data-files.test.ts @@ -72,3 +72,22 @@ test("data size summaries combine categories and disk-space checks reserve space expect(archiveExclusions("state")).toEqual([path.join("daemon", "runtime-lock.json")]) expect(archiveExclusions("media")).toEqual([]) }) + +test("data merge never follows destination links or replaces a dangling link", async () => { + await using tmp = await tmpdir() + const source = path.join(tmp.path, "source") + const target = path.join(tmp.path, "target") + const outside = path.join(tmp.path, "outside") + await fs.mkdir(path.join(source, "nested"), { recursive: true }) + await Bun.write(path.join(source, "nested/new.txt"), "new") + await fs.mkdir(target) + await fs.mkdir(outside) + await fs.symlink(outside, path.join(target, "nested")) + await expect(copyDirSkipExisting(source, target)).rejects.toThrow("symbolic link") + expect(await Bun.file(path.join(outside, "new.txt")).exists()).toBe(false) + await fs.unlink(path.join(target, "nested")) + await fs.mkdir(path.join(target, "nested")) + await fs.symlink(path.join(outside, "absent"), path.join(target, "nested/new.txt")) + expect(await copyDirSkipExisting(source, target)).toEqual({ copied: 0, skipped: 1 }) + expect(await Bun.file(path.join(outside, "absent")).exists()).toBe(false) +}) diff --git a/packages/connections/test/channel/diagnostics.test.ts b/packages/connections/test/channel/diagnostics.test.ts index 5763d721d..3f37cc547 100644 --- a/packages/connections/test/channel/diagnostics.test.ts +++ b/packages/connections/test/channel/diagnostics.test.ts @@ -11,7 +11,6 @@ import { ScopeContext } from "@ericsanchezok/synergy-harness/scope/context" import { Storage } from "@ericsanchezok/synergy-harness/storage/storage" import { StoragePath } from "@ericsanchezok/synergy-harness/storage/path" import { externalIdentityHash } from "@ericsanchezok/synergy-harness/util/identity" -import { Global } from "@ericsanchezok/synergy-harness/global" // --------------------------------------------------------------------------- // Helpers @@ -89,23 +88,18 @@ async function seedDiagnosticRecords(input: { count: number firstTimestamp: number }): Promise { - const root = path.join(Global.Path.data, ...StoragePath.channelDiagnosticsRecordsRoot(input.accountHash)) - await fs.mkdir(root, { recursive: true }) const firstID = `${input.firstTimestamp.toString().padStart(13, "0")}-seed-00000` - const concurrency = 128 - for (let offset = 0; offset < input.count; offset += concurrency) { - await Promise.all( - Array.from({ length: Math.min(concurrency, input.count - offset) }, (_, index) => { - const position = offset + index - const timestamp = input.firstTimestamp + position - const id = `${timestamp.toString().padStart(13, "0")}-seed-${position.toString().padStart(5, "0")}` - return Bun.write( - path.join(root, `${id}.json`), - JSON.stringify({ timestamp, level: "info", message: `seed ${position}` }), - ) - }), - ) - } + await Storage.transaction(async () => { + for (let position = 0; position < input.count; position++) { + const timestamp = input.firstTimestamp + position + const id = `${timestamp.toString().padStart(13, "0")}-seed-${position.toString().padStart(5, "0")}` + await Storage.write(StoragePath.channelDiagnosticsRecord(input.accountHash, id), { + timestamp, + level: "info", + message: `seed ${position}`, + }) + } + }) return firstID } diff --git a/packages/harness/src/session/index.ts b/packages/harness/src/session/index.ts index 0ef3a360d..a31340b2d 100644 --- a/packages/harness/src/session/index.ts +++ b/packages/harness/src/session/index.ts @@ -839,19 +839,22 @@ export namespace Session { return withClientInfo(info) }) - // This queue orders completion mutations with their completion events. Canonical writes still use SessionMutation. - const completionNoticeMutations = new Map>() + // Acquire this publication queue before SQL; a transaction must never wait + // on a queued mutation that needs the same SQL writer. + const completionNoticeMutations = Storage.state(() => new Map>()) function serializeCompletionNoticeMutation(id: string, mutation: () => Promise): Promise { - const previous = completionNoticeMutations.get(id) ?? Promise.resolve() + if (Storage.inTransaction()) return mutation() + const queue = completionNoticeMutations() + const previous = queue.get(id) ?? Promise.resolve() const current = previous.then(mutation, mutation) const settled = current.then( () => undefined, () => undefined, ) - completionNoticeMutations.set(id, settled) + queue.set(id, settled) void settled.finally(() => { - if (completionNoticeMutations.get(id) === settled) completionNoticeMutations.delete(id) + if (queue.get(id) === settled) queue.delete(id) }) return current } @@ -897,12 +900,12 @@ export namespace Session { acknowledgedCount: number, options?: { repairNavOnNoop?: boolean }, ) { - return Storage.transaction(async () => { - if (!Number.isSafeInteger(acknowledgedCount) || acknowledgedCount < 0) { - throw new TypeError("acknowledgedCount must be a non-negative safe integer") - } + return serializeCompletionNoticeMutation(id, () => + Storage.transaction(async () => { + if (!Number.isSafeInteger(acknowledgedCount) || acknowledgedCount < 0) { + throw new TypeError("acknowledgedCount must be a non-negative safe integer") + } - return serializeCompletionNoticeMutation(id, async () => { const session = await SessionManager.requireSession(id) const scope = session.scope as Scope const scopeID = asScopeID(scope.id) @@ -923,8 +926,8 @@ export namespace Session { const navEntry = await SessionNav.upsertNavEntry(toNavEntry(result)) await publishInfo(SessionEvent.Updated, result, navEntry) return { info: await withRuntimeInfo(result), acknowledgedCount: actualAcknowledgedCount } - }) - }) + }), + ) } export async function acknowledgeCompletionNotice(id: string, acknowledgedCount: number) { @@ -956,21 +959,23 @@ export namespace Session { } export async function recordCompletionNotice(id: string, options?: { publishEvent?: boolean }) { - return serializeCompletionNoticeMutation(id, async () => { - let unreadCount: number | undefined - const result = await update(id, (draft) => { - if (draft.time.archived || draft.completionNotice.silent) return - const current = draft.completionNotice.unreadCount ?? (draft.completionNotice.unread ? 1 : 0) - const next = Math.min(Number.MAX_SAFE_INTEGER, current + 1) - draft.completionNotice.unread = true - draft.completionNotice.unreadCount = next - if (next !== current) unreadCount = next - }) - if (unreadCount !== undefined && options?.publishEvent !== false) { - await Bus.publish(SessionEvent.Completion, { sessionID: id, unreadCount }) - } - return result - }) + return serializeCompletionNoticeMutation(id, () => + Storage.transaction(async () => { + let unreadCount: number | undefined + const result = await update(id, (draft) => { + if (draft.time.archived || draft.completionNotice.silent) return + const current = draft.completionNotice.unreadCount ?? (draft.completionNotice.unread ? 1 : 0) + const next = Math.min(Number.MAX_SAFE_INTEGER, current + 1) + draft.completionNotice.unread = true + draft.completionNotice.unreadCount = next + if (next !== current) unreadCount = next + }) + if (unreadCount !== undefined && options?.publishEvent !== false) { + await Bus.publish(SessionEvent.Completion, { sessionID: id, unreadCount }) + } + return result + }), + ) } async function updateInternal( diff --git a/packages/harness/src/storage/transactional-store.ts b/packages/harness/src/storage/transactional-store.ts index 694d23808..4e2bf4545 100644 --- a/packages/harness/src/storage/transactional-store.ts +++ b/packages/harness/src/storage/transactional-store.ts @@ -171,6 +171,12 @@ export class StoreTransaction { } async write(key: string[], value: T, options: { expectedRevision?: bigint } = {}): Promise { + this.check(true) + if (key[0] === "sessions" && key.length >= 4) await this.assertNotDeleted([...key.slice(0, 3), "info"]) + await this.put(key, value, options) + } + + private async put(key: string[], value: T, options: { expectedRevision?: bigint } = {}): Promise { this.check(true) if (!key.length) throw new StorageIntegrityError("Cannot write the storage root") const previous = await this.row(key) @@ -221,10 +227,12 @@ export class StoreTransaction { ) } + // SQLite must drive recursion from the frontier to use both columns of the parent index. + // CROSS JOIN prevents a namespace-wide node scan for every visited node. async scan(prefix: string[]): Promise { this.check() const rows = await this.connection.query( - "WITH RECURSIVE tree(key_id, child) AS (SELECT key_id, segment FROM storage_nodes WHERE namespace = ? AND parent_id = ? UNION ALL SELECT node.key_id, tree.child FROM storage_nodes node JOIN tree ON node.parent_id = tree.key_id WHERE node.namespace = ?) SELECT DISTINCT tree.child FROM tree JOIN storage_records record ON record.key_id = tree.key_id AND record.namespace = ? WHERE record.body IS NOT NULL", + "WITH RECURSIVE tree(key_id, child) AS (SELECT key_id, segment FROM storage_nodes WHERE namespace = ? AND parent_id = ? UNION ALL SELECT node.key_id, tree.child FROM tree CROSS JOIN storage_nodes node WHERE node.namespace = ? AND node.parent_id = tree.key_id) SELECT DISTINCT tree.child FROM tree JOIN storage_records record ON record.key_id = tree.key_id AND record.namespace = ? WHERE record.body IS NOT NULL", [this.namespace, keyID(prefix), this.namespace, this.namespace], ) return rows.map((row) => row.child).sort() @@ -233,7 +241,7 @@ export class StoreTransaction { async list(prefix: string[]): Promise { this.check() const rows = await this.connection.query( - "WITH RECURSIVE tree(key_id) AS (SELECT key_id FROM storage_nodes WHERE namespace = ? AND parent_id = ? UNION ALL SELECT node.key_id FROM storage_nodes node JOIN tree ON node.parent_id = tree.key_id WHERE node.namespace = ?) SELECT record.key_text FROM tree JOIN storage_records record ON record.key_id = tree.key_id AND record.namespace = ? WHERE record.body IS NOT NULL", + "WITH RECURSIVE tree(key_id) AS (SELECT key_id FROM storage_nodes WHERE namespace = ? AND parent_id = ? UNION ALL SELECT node.key_id FROM tree CROSS JOIN storage_nodes node WHERE node.namespace = ? AND node.parent_id = tree.key_id) SELECT record.key_text FROM tree JOIN storage_records record ON record.key_id = tree.key_id AND record.namespace = ? WHERE record.body IS NOT NULL", [this.namespace, keyID(prefix), this.namespace, this.namespace], ) return rows.map((row) => JSON.parse(row.key_text) as string[]).sort() @@ -249,7 +257,7 @@ export class StoreTransaction { return } await this.connection.query( - "WITH RECURSIVE tree(key_id) AS (SELECT key_id FROM storage_nodes WHERE namespace = ? AND key_id = ? UNION ALL SELECT node.key_id FROM storage_nodes node JOIN tree ON node.parent_id = tree.key_id WHERE node.namespace = ?) UPDATE storage_records SET body = NULL, revision = revision + 1, updated = ? WHERE namespace = ? AND key_id IN (SELECT key_id FROM tree) AND body IS NOT NULL", + "WITH RECURSIVE tree(key_id) AS (SELECT key_id FROM storage_nodes WHERE namespace = ? AND key_id = ? UNION ALL SELECT node.key_id FROM tree CROSS JOIN storage_nodes node WHERE node.namespace = ? AND node.parent_id = tree.key_id) UPDATE storage_records SET body = NULL, revision = revision + 1, updated = ? WHERE namespace = ? AND key_id IN (SELECT key_id FROM tree) AND body IS NOT NULL", [this.namespace, keyID(prefix), this.namespace, Date.now(), this.namespace], ) } @@ -351,7 +359,7 @@ export class StoreTransaction { if (BigInt(entry.revision) > 9223372036854775807n) throw new StorageIntegrityError("Unsupported record revision") if ((await this.readMany([entry.key]))[0] !== undefined) throw new StorageConflictError("Portable record conflicts with existing target data") - await this.write(entry.key, entry.value) + await this.put(entry.key, entry.value) await this.connection.query( "UPDATE storage_records SET revision = CASE WHEN revision > ? THEN revision ELSE ? END WHERE namespace = ? AND key_id = ?", [BigInt(entry.revision), BigInt(entry.revision), this.namespace, keyID(entry.key)], diff --git a/packages/harness/test/storage/context.test.ts b/packages/harness/test/storage/context.test.ts index 07fd7dfaa..f0615f4c5 100644 --- a/packages/harness/test/storage/context.test.ts +++ b/packages/harness/test/storage/context.test.ts @@ -72,3 +72,21 @@ test("notification failure does not turn a confirmed commit into a retryable wri expect(lastObserver).toBe(true) }) }) + +test("deleted sessions reject delayed rollout and inbox writes in the same transaction", async () => { + await Storage.provide({ store, artifactDirectory: root }, async () => { + const session = ["sessions", "scope_deleted", "ses_deleted"] + await Storage.write([...session, "info"], { id: "ses_deleted" }) + await Storage.removeTree(session) + for (const suffix of [["rollout", "journal", "head"], ["inbox", "late"], ["info"]]) { + await expect( + Storage.transaction(async () => { + await Storage.write(["unrelated", "late-owner"], true) + await Storage.write([...session, ...suffix], { delayed: true }) + }), + ).rejects.toThrow("deleted") + } + expect(await Storage.list(session)).toEqual([]) + expect(await Storage.readMany([["unrelated", "late-owner"]])).toEqual([undefined]) + }) +}) diff --git a/packages/product-runtime/test/config/config.test.ts b/packages/product-runtime/test/config/config.test.ts index c3850e1a4..da241c6ce 100644 --- a/packages/product-runtime/test/config/config.test.ts +++ b/packages/product-runtime/test/config/config.test.ts @@ -429,6 +429,7 @@ test("legacy monolithic config with a retired root key stays intact as a migrati await expect(Config.globalRaw()).rejects.toThrow() expect(JSON.parse(await Bun.file(legacy).text())).toEqual({ auto_classifier: true, theme: "test_theme" }) + await Storage.remove(StoragePath.metaMigrationLogDomain("config")) resetMigrations() await runMigrations({ targetDomain: "config" }) // The lazy global cache is still holding the rejected load from the @@ -1380,6 +1381,7 @@ test("migrates legacy channel holos config to top-level holos", async () => { }`, ) + await Storage.remove(StoragePath.metaMigrationLogDomain("config")) resetMigrations() await runMigrations({ targetDomain: "config" }) @@ -1431,6 +1433,7 @@ test("removes legacy channel holos config when top-level holos already exists", }`, ) + await Storage.remove(StoragePath.metaMigrationLogDomain("config")) resetMigrations() await runMigrations({ targetDomain: "config" }) @@ -1464,6 +1467,7 @@ test("migrates legacy auto_classifier config to smartAllow", async () => { }`, ) + await Storage.remove(StoragePath.metaMigrationLogDomain("config")) resetMigrations() await runMigrations({ targetDomain: "config" }) @@ -1495,6 +1499,7 @@ test("migrates project permissions domain auto_classifier config to smartAllow", ) process.chdir(project) + await Storage.remove(StoragePath.metaMigrationLogDomain("config")) resetMigrations() await runMigrations({ targetDomain: "config" }) @@ -1530,6 +1535,7 @@ test("removes deprecated autoupdate from monolithic and domain configs", async ( await Bun.write(projectGeneral, `{"autoupdate": true, "snapshot": false}`) process.chdir(project) + await Storage.remove(StoragePath.metaMigrationLogDomain("config")) resetMigrations() await runMigrations({ targetDomain: "config" }) @@ -1568,6 +1574,7 @@ test("removes deprecated providerCatalog from monolithic and domain configs", as await Bun.write(projectProviders, `{"providerCatalog": {"enabled": false}, "enabled_providers": ["legacy"]}`) process.chdir(project) + await Storage.remove(StoragePath.metaMigrationLogDomain("config")) resetMigrations() await runMigrations({ targetDomain: "config" }) @@ -1602,11 +1609,10 @@ test("removes deprecated providerCatalog from persisted project scopes", async ( await Bun.write(providersFile, `{"providerCatalog": {"enabled": true}, "enabled_providers": ["legacy"]}`) await fs.mkdir(path.join(projectB, ".synergy", "synergy.d"), { recursive: true }) - const dataDir = path.join(home, ".synergy", "data", "projects") - await fs.mkdir(dataDir, { recursive: true }) - await Bun.write(path.join(dataDir, "scope-record.json"), JSON.stringify({ worktree: projectA })) + await Storage.write(["projects", "scope-record"], { worktree: projectA }) process.chdir(projectB) + await Storage.remove(StoragePath.metaMigrationLogDomain("config")) resetMigrations() await runMigrations({ targetDomain: "config" }) @@ -1671,6 +1677,7 @@ test("migrates legacy identity config to valid library config", async () => { }`, ) + await Storage.remove(StoragePath.metaMigrationLogDomain("config")) resetMigrations() await runMigrations({ targetDomain: "config" }) @@ -1747,6 +1754,7 @@ test("migrates legacy engram domain config to library and general domains", asyn }), ) + await Storage.remove(StoragePath.metaMigrationLogDomain("config")) resetMigrations() await runMigrations({ targetDomain: "config" }) @@ -1789,6 +1797,7 @@ test("repairs invalid library shapes written by legacy identity migration", asyn }), ) + await Storage.remove(StoragePath.metaMigrationLogDomain("config")) resetMigrations() await runMigrations({ targetDomain: "config" }) @@ -1835,6 +1844,7 @@ test("provider profile normalize migration rewrites known provider aliases", asy }), ) + await Storage.remove(StoragePath.metaMigrationLogDomain("config")) resetMigrations() await runMigrations({ targetDomain: "config" }) diff --git a/packages/workbench/src/push/store.ts b/packages/workbench/src/push/store.ts index 830ac0d79..df788e7f5 100644 --- a/packages/workbench/src/push/store.ts +++ b/packages/workbench/src/push/store.ts @@ -1,4 +1,4 @@ -import fs from "node:fs/promises" +import { z } from "zod" import path from "node:path" import webpush from "web-push" import { Storage } from "@ericsanchezok/synergy-harness/storage/storage" @@ -18,11 +18,6 @@ export namespace PushStore { return path.join(Global.Path.data, ...key) + ".json" } - async function writeCredential(key: string[], content: T): Promise { - await Storage.write(key, content) - await fs.chmod(credentialFile(key), 0o600).catch(() => undefined) - } - export async function list(): Promise { const ids = await Storage.scan(StoragePath.pushSubscriptionsRoot()) const records = await Storage.readMany(ids.map((id) => StoragePath.pushSubscription(id))) @@ -34,30 +29,35 @@ export namespace PushStore { * refreshes its keys/categories instead of duplicating fan-out targets. */ export async function upsert(input: PushTypes.SubscribeInput): Promise { - const existing = await findByEndpoint(input.endpoint) - const record: PushTypes.Subscription = { - id: existing?.id ?? `push_${crypto.randomUUID().replace(/-/g, "").slice(0, 16)}`, - endpoint: input.endpoint, - keys: input.keys, - ...(input.deviceLabel !== undefined ? { deviceLabel: input.deviceLabel } : {}), - created: existing?.created ?? Date.now(), - categories: input.categories ?? existing?.categories ?? PushTypes.DEFAULT_CATEGORIES, - } - await writeCredential(StoragePath.pushSubscription(record.id), record) - return record + return Storage.transaction(async () => { + const existing = await findByEndpoint(input.endpoint) + const record: PushTypes.Subscription = { + id: existing?.id ?? `push_${crypto.randomUUID().replace(/-/g, "").slice(0, 16)}`, + endpoint: input.endpoint, + keys: input.keys, + ...(input.deviceLabel !== undefined ? { deviceLabel: input.deviceLabel } : {}), + created: existing?.created ?? Date.now(), + categories: input.categories ?? existing?.categories ?? PushTypes.DEFAULT_CATEGORIES, + } + await Storage.write(StoragePath.pushSubscription(record.id), record) + return record + }) } export async function removeByEndpoint(endpoint: string): Promise { - const existing = await findByEndpoint(endpoint) - if (!existing) return - await Storage.remove(StoragePath.pushSubscription(existing.id)) + await Storage.transaction(async () => { + const existing = await findByEndpoint(endpoint) + if (existing) await Storage.remove(StoragePath.pushSubscription(existing.id)) + }) } export async function removeById(id: string): Promise { - const existing = await Storage.read(StoragePath.pushSubscription(id)).catch(() => undefined) - if (!existing) return false - await Storage.remove(StoragePath.pushSubscription(id)) - return true + return Storage.transaction(async () => { + const [existing] = await Storage.readMany([StoragePath.pushSubscription(id)]) + if (!existing) return false + await Storage.remove(StoragePath.pushSubscription(id)) + return true + }) } export async function findByEndpoint(endpoint: string): Promise { @@ -66,9 +66,10 @@ export namespace PushStore { } export async function updateCategories(id: string, categories: PushTypes.Categories): Promise { - const existing = await Storage.read(StoragePath.pushSubscription(id)).catch(() => undefined) - if (!existing) return - await writeCredential(StoragePath.pushSubscription(id), { ...existing, categories }) + await Storage.transaction(async () => { + const [existing] = await Storage.readMany([StoragePath.pushSubscription(id)]) + if (existing) await Storage.write(StoragePath.pushSubscription(id), { ...existing, categories }) + }) } // Memoized first-use generation keyed by the data home: two concurrent @@ -88,12 +89,17 @@ export namespace PushStore { if (vapidInit && vapidInitHome === home) return vapidInit vapidInitHome = home vapidInit = (async () => { - const existing = await Storage.read<{ publicKey: string; privateKey: string }>(StoragePath.pushVapid()).catch( - () => undefined, - ) - if (existing?.publicKey && existing?.privateKey) return existing + const filename = credentialFile(StoragePath.pushVapid()) + const existing = await Bun.file(filename) + .json() + .catch((error) => { + if (error?.code === "ENOENT") return + throw error + }) + if (existing !== undefined) + return z.object({ publicKey: z.string().min(1), privateKey: z.string().min(1) }).parse(existing) const generated = webpush.generateVAPIDKeys() - await writeCredential(StoragePath.pushVapid(), generated) + await Storage.writeJsonAtomic(filename, JSON.stringify(generated), { private: true, durable: true }) log.info("generated VAPID key pair") return generated })().catch((error) => { diff --git a/packages/workbench/test/push/bridge.test.ts b/packages/workbench/test/push/bridge.test.ts index 7253ae69f..29c7439c5 100644 --- a/packages/workbench/test/push/bridge.test.ts +++ b/packages/workbench/test/push/bridge.test.ts @@ -1,3 +1,5 @@ +import { TransactionalStore } from "@ericsanchezok/synergy-harness/storage/transactional-store" +import { Storage } from "@ericsanchezok/synergy-harness/storage/storage" import path from "node:path" import fs from "node:fs/promises" import { afterEach, describe, expect, test } from "bun:test" @@ -43,9 +45,17 @@ async function withIsolatedHome(fn: () => Promise): Promise { const home = path.join(tmp.path, "home") process.env.SYNERGY_TEST_HOME = home await fs.mkdir(home, { recursive: true }) + const store = await TransactionalStore.open({ + backend: "sqlite", + namespace: "push-test", + filename: path.join(home, "authority.sqlite"), + }) try { - return await ScopeContext.provide({ scope: Scope.home(), fn }) + return await Storage.provide({ store, artifactDirectory: path.join(home, ".synergy", "data") }, () => + ScopeContext.provide({ scope: Scope.home(), fn }), + ) } finally { + await store.close() if (previous === undefined) delete process.env.SYNERGY_TEST_HOME else process.env.SYNERGY_TEST_HOME = previous } diff --git a/packages/workbench/test/push/service.test.ts b/packages/workbench/test/push/service.test.ts index a577644c1..e15bb1609 100644 --- a/packages/workbench/test/push/service.test.ts +++ b/packages/workbench/test/push/service.test.ts @@ -1,3 +1,5 @@ +import { TransactionalStore } from "@ericsanchezok/synergy-harness/storage/transactional-store" +import { Storage } from "@ericsanchezok/synergy-harness/storage/storage" import path from "node:path" import fs from "node:fs/promises" import { afterEach, describe, expect, test } from "bun:test" @@ -22,9 +24,17 @@ async function withIsolatedHome(fn: () => Promise): Promise { const home = path.join(tmp.path, "home") process.env.SYNERGY_TEST_HOME = home await fs.mkdir(home, { recursive: true }) + const store = await TransactionalStore.open({ + backend: "sqlite", + namespace: "push-test", + filename: path.join(home, "authority.sqlite"), + }) try { - return await ScopeContext.provide({ scope: Scope.home(), fn }) + return await Storage.provide({ store, artifactDirectory: path.join(home, ".synergy", "data") }, () => + ScopeContext.provide({ scope: Scope.home(), fn }), + ) } finally { + await store.close() if (previous === undefined) delete process.env.SYNERGY_TEST_HOME else process.env.SYNERGY_TEST_HOME = previous } diff --git a/packages/workbench/test/push/store.test.ts b/packages/workbench/test/push/store.test.ts index ce89fbab6..0094e349b 100644 --- a/packages/workbench/test/push/store.test.ts +++ b/packages/workbench/test/push/store.test.ts @@ -1,3 +1,4 @@ +import { TransactionalStore } from "@ericsanchezok/synergy-harness/storage/transactional-store" import path from "node:path" import fs from "node:fs/promises" import { describe, expect, test } from "bun:test" @@ -18,15 +19,34 @@ async function withIsolatedHome(fn: () => Promise): Promise { const home = path.join(tmp.path, "home") process.env.SYNERGY_TEST_HOME = home await fs.mkdir(home, { recursive: true }) + const store = await TransactionalStore.open({ + backend: "sqlite", + namespace: "push-test", + filename: path.join(home, "authority.sqlite"), + }) try { - return await ScopeContext.provide({ scope: Scope.home(), fn }) + return await Storage.provide({ store, artifactDirectory: path.join(home, ".synergy", "data") }, () => + ScopeContext.provide({ scope: Scope.home(), fn }), + ) } finally { + await store.close() if (previous === undefined) delete process.env.SYNERGY_TEST_HOME else process.env.SYNERGY_TEST_HOME = previous } } describe("PushStore", () => { + test("preserves the existing VAPID credential when authority moves to SQL", async () => { + await withIsolatedHome(async () => { + const pair = { publicKey: "existing-public-key", privateKey: "existing-private-key" } + await Storage.writeJsonAtomic(path.join(Global.Path.data, "push/vapid.json"), JSON.stringify(pair), { + private: true, + }) + expect(await PushStore.vapidKeys()).toEqual(pair) + expect(await Storage.readMany([StoragePath.pushVapid()])).toEqual([undefined]) + }) + }) + test("round-trips subscriptions", async () => { await withIsolatedHome(async () => { const created = await PushStore.upsert({ @@ -82,7 +102,7 @@ describe("PushStore", () => { const first = await PushStore.vapidKeys() expect(first.publicKey).toBeTruthy() expect(first.privateKey).toBeTruthy() - const persisted = await Storage.read<{ publicKey: string; privateKey: string }>(StoragePath.pushVapid()) + const persisted = await Bun.file(path.join(Global.Path.data, "push/vapid.json")).json() expect(persisted).toEqual(first) const second = await PushStore.vapidKeys() expect(second).toEqual(first) @@ -113,8 +133,11 @@ describe("PushStore", () => { if (process.platform !== "win32") { const vapidMode = (await fs.stat(path.join(dataRoot, "push/vapid.json"))).mode expect(vapidMode & 0o777).toBe(0o600) - const subMode = (await fs.stat(path.join(dataRoot, `push/subscriptions/${sub.id}.json`))).mode - expect(subMode & 0o777).toBe(0o600) + const store = Storage.current().store + if (store.options.backend !== "sqlite") throw new Error("Expected isolated SQLite fixture") + expect((await fs.stat(store.options.filename)).mode & 0o777).toBe(0o600) + expect(await Bun.file(path.join(dataRoot, `push/subscriptions/${sub.id}.json`)).exists()).toBe(false) + expect(await Storage.read(StoragePath.pushSubscription(sub.id))).toMatchObject({ id: sub.id }) } }) }) diff --git a/test/package/fixture/runtime-composition.ts b/test/package/fixture/runtime-composition.ts index 882631bfd..ae4b423a4 100644 --- a/test/package/fixture/runtime-composition.ts +++ b/test/package/fixture/runtime-composition.ts @@ -132,30 +132,22 @@ try { if (enabled("browser")) { const { BrowserRuntime } = await import("@ericsanchezok/synergy-browser-runtime/runtime") const { browserOwnerKey } = await import("@ericsanchezok/synergy-browser") - const { Global } = await import("@ericsanchezok/synergy-harness/global") + const { Storage } = await import("@ericsanchezok/synergy-harness/storage/storage") const owner = { mode: "session" as const, scopeID: scope.id, sessionID: session.id, directory } - const stateFile = path.join( - Global.Path.data, - "browser", - "sessions-v4", - `${createHash("sha256").update(browserOwnerKey(owner)).digest("hex")}.json`, - ) - await Bun.write( - stateFile, - JSON.stringify({ - version: 4, - status: "suspended", - page: { id: "page-installed", url: "https://example.com/research", title: "Installed Browser" }, - timestamp: Date.now(), - }), - ) + const stateKey = ["browser", "sessions-v4", createHash("sha256").update(browserOwnerKey(owner)).digest("hex")] + await Storage.write(stateKey, { + version: 4, + status: "suspended", + page: { id: "page-installed", url: "https://example.com/research", title: "Installed Browser" }, + timestamp: Date.now(), + }) const browser = await BrowserRuntime.getOrCreateSession(owner) assert.equal(browser.status, "suspended") assert.equal(browser.page, null) assert.equal(BrowserRuntime.resourceStats().ownerCount, 1) assert.equal(BrowserRuntime.resourceStats().processCount, 0) await browser.save() - assert.equal((await Bun.file(stateFile).json()).page?.title, "Installed Browser") + assert.equal((await Storage.read<{ page?: { title: string } }>(stateKey)).page?.title, "Installed Browser") } if (enabled("library")) { const { LibraryDB } = await import("@ericsanchezok/synergy-library") From 1c7748d5c6f77a65c48ca28cf9483e0285d8c82d Mon Sep 17 00:00:00 2001 From: EricSanchezok <115814526+EricSanchezok@users.noreply.github.com> Date: Mon, 14 Sep 2026 22:27:29 +0800 Subject: [PATCH 06/14] test(storage): align integration fixtures with SQL authority Regenerate the core configuration schema and update lifecycle, Push, observability and benchmark probes to use explicit storage ownership and committed records. Co-authored-by: synergy-agent <299070056+synergy-agent@users.noreply.github.com> --- benchmark/test/test_docker.py | 35 ++++++++++++------- packages/cli/schema/config.schema.json | 26 ++++++++++++++ .../harness/test/observability/fixture.ts | 2 ++ .../product-runtime/test/cli/services.test.ts | 6 +++- .../test/server/push-route.test.ts | 12 ++++++- .../test/server/resident-runtime.test.ts | 7 +++- .../test/server/runtime-handle.test.ts | 2 ++ 7 files changed, 74 insertions(+), 16 deletions(-) diff --git a/benchmark/test/test_docker.py b/benchmark/test/test_docker.py index 9c3c28770..9d5c64a43 100644 --- a/benchmark/test/test_docker.py +++ b/benchmark/test/test_docker.py @@ -1,6 +1,8 @@ import asyncio +import json import os import shutil +import zipfile from pathlib import Path import pytest @@ -157,8 +159,6 @@ def test_real_synergy_paired_rollout(prepared_fixture) -> None: def assert_retained_credentials_absent(root: Path) -> None: - import zipfile - sentinel = b"deterministic-local-fixture" for evidence_file in root.glob("trials/*/attempt-*/evidence.json"): evidence = read_json(evidence_file) @@ -173,10 +173,13 @@ def assert_retained_credentials_absent(root: Path) -> None: def primary_attempts(root: Path): - for call in root.glob("trials/*/attempt-*/*/agent/home/.synergy/data/sessions/*/*/rollout/runs/*/calls/*.json"): - if read_json(call)["purpose"] == "synergy": - for attempt in (call.parent.parent / "attempts" / call.stem).glob("*.json"): - yield read_json(attempt) + for file in root.glob("trials/*/attempt-*/*/agent/rollout.zip"): + with zipfile.ZipFile(file) as archive: + manifest = json.loads(archive.read("manifest.json")) + assert manifest["format"] == "synergy-rollout" and manifest["version"] == 1 + for snapshot in manifest["snapshots"]: + calls = {call["id"] for call in snapshot["calls"] if call["purpose"] == "synergy"} + yield from (attempt for attempt in snapshot["attempts"] if attempt["callID"] in calls) @pytest.mark.parametrize("mode", ["long", "disconnect", "timeout", "cancel", "docker-stop"]) @@ -219,14 +222,20 @@ async def run() -> None: container = containers.strip() probe = """ import json +import sqlite3 +from contextlib import closing from pathlib import Path -observed = False -for call in Path('/logs/agent/home').glob('.synergy/data/sessions/*/*/rollout/runs/*/calls/*.json'): - if json.loads(call.read_text())['purpose'] != 'synergy': - continue - for file in (call.parent.parent / 'attempts' / call.stem).glob('*.json'): - observed |= (json.loads(file.read_text()).get('response') or {}).get('bytes', 0) > 0 -print(observed) +root = Path('/logs/agent/home/.synergy/data/storage') +namespace = json.loads((root / 'manifest.json').read_text())['namespace'] +with closing(sqlite3.connect((root / 'agent.sqlite').as_uri() + '?mode=ro', uri=True)) as db: + rows = [(json.loads(key), json.loads(body)) for key, body in db.execute( + "SELECT key_text, body FROM storage_records WHERE namespace=? AND kind='rollout' AND body IS NOT NULL", + (namespace,), + )] +calls = {(tuple(key[:6]), key[7]) for key, value in rows + if len(key) == 8 and key[4] == 'runs' and key[6] == 'calls' and value['purpose'] == 'synergy'} +print(any(len(key) == 9 and key[6] == 'attempts' and (tuple(key[:6]), key[7]) in calls + and (value.get('response') or {}).get('bytes', 0) > 0 for key, value in rows)) """ while ( await asyncio.to_thread( diff --git a/packages/cli/schema/config.schema.json b/packages/cli/schema/config.schema.json index e17840656..2ecef592a 100644 --- a/packages/cli/schema/config.schema.json +++ b/packages/cli/schema/config.schema.json @@ -10,6 +10,32 @@ "type": "string", "enum": ["DEBUG", "INFO", "WARN", "ERROR"] }, + "storage": { + "description": "Global authoritative storage; backend changes require an explicit storage migration", + "anyOf": [ + { + "type": "object", + "properties": { + "backend": { "type": "string", "const": "sqlite" }, + "namespace": { "type": "string", "pattern": "^[a-zA-Z0-9_-]{1,128}$" }, + "filename": { "type": "string", "minLength": 1 } + }, + "required": ["backend"], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "backend": { "type": "string", "const": "postgres" }, + "namespace": { "type": "string", "pattern": "^[a-zA-Z0-9_-]{1,128}$" }, + "connectionEnv": { "type": "string", "pattern": "^[a-zA-Z_][a-zA-Z0-9_]*$" }, + "maxConnections": { "type": "integer", "minimum": 2, "maximum": 64 } + }, + "required": ["backend", "namespace", "connectionEnv"], + "additionalProperties": false + } + ] + }, "server": { "description": "Server configuration for synergy serve and web commands", "ref": "ServerConfig", diff --git a/packages/harness/test/observability/fixture.ts b/packages/harness/test/observability/fixture.ts index 100cbeace..08b3c5767 100644 --- a/packages/harness/test/observability/fixture.ts +++ b/packages/harness/test/observability/fixture.ts @@ -1,3 +1,4 @@ +import { initializeSqliteEngine } from "../../src/storage/sqlite-engine" import { mkdirSync, mkdtempSync, rmSync } from "fs" import { tmpdir } from "os" import path from "path" @@ -10,6 +11,7 @@ const originalHome = process.env.SYNERGY_TEST_HOME const originalInline = process.env.SYNERGY_OBSERVABILITY_INLINE export function resetObservabilityHome(prefix = "synergy-observability-") { + initializeSqliteEngine() const home = mkdtempSync(path.join(tmpdir(), prefix)) homes.push(home) process.env.SYNERGY_TEST_HOME = home diff --git a/packages/product-runtime/test/cli/services.test.ts b/packages/product-runtime/test/cli/services.test.ts index 40eb20ef3..a9dd00cbf 100644 --- a/packages/product-runtime/test/cli/services.test.ts +++ b/packages/product-runtime/test/cli/services.test.ts @@ -58,7 +58,11 @@ test("server command passes interactive and managed lifecycle options to the ful interactive: true, printBanner: true, printChannelStatus: true, - network: { port: 45217, hostname: "127.0.0.1" }, + }) + const network = started[0]?.network + expect(await (typeof network === "function" ? network() : network)).toMatchObject({ + port: 45217, + hostname: "127.0.0.1", }) expect(started[1]).toMatchObject({ interactive: false, printBanner: false, printChannelStatus: false }) }) diff --git a/packages/product-runtime/test/server/push-route.test.ts b/packages/product-runtime/test/server/push-route.test.ts index f02c588f3..34a49e438 100644 --- a/packages/product-runtime/test/server/push-route.test.ts +++ b/packages/product-runtime/test/server/push-route.test.ts @@ -1,3 +1,5 @@ +import { Storage } from "@ericsanchezok/synergy-harness/storage/storage" +import { TransactionalStore } from "@ericsanchezok/synergy-harness/storage/transactional-store" import path from "node:path" import fs from "node:fs/promises" import { afterEach, describe, expect, test } from "bun:test" @@ -22,9 +24,17 @@ async function withIsolatedHome(fn: () => Promise): Promise { const home = path.join(tmp.path, "home") process.env.SYNERGY_TEST_HOME = home await fs.mkdir(home, { recursive: true }) + const store = await TransactionalStore.open({ + backend: "sqlite", + namespace: "push-test", + filename: path.join(home, "authority.sqlite"), + }) try { - return await ScopeContext.provide({ scope: Scope.home(), fn }) + return await Storage.provide({ store, artifactDirectory: path.join(home, ".synergy", "data") }, () => + ScopeContext.provide({ scope: Scope.home(), fn }), + ) } finally { + await store.close() if (previous === undefined) delete process.env.SYNERGY_TEST_HOME else process.env.SYNERGY_TEST_HOME = previous } diff --git a/packages/product-runtime/test/server/resident-runtime.test.ts b/packages/product-runtime/test/server/resident-runtime.test.ts index 0c7ae7042..d1509db08 100644 --- a/packages/product-runtime/test/server/resident-runtime.test.ts +++ b/packages/product-runtime/test/server/resident-runtime.test.ts @@ -2,11 +2,16 @@ import { expect, test } from "bun:test" import { ProductRuntimeHandle } from "../../src/server/runtime-handle" import { ServerProcessLock } from "@ericsanchezok/synergy-harness/util/server-process-lock" import { ScopeStartup } from "@ericsanchezok/synergy-harness/scope/startup" +import { Storage } from "@ericsanchezok/synergy-harness/storage/storage" import { GlobalBus } from "@ericsanchezok/synergy-harness/bus/global" test("resident full runtime starts its product services and drains them before releasing the Home", async () => { const listeners = GlobalBus.listenerCount("event") - const runtime = await ProductRuntimeHandle.open({ mode: "server", network: { hostname: "127.0.0.1", port: 0 } }) + const runtime = await ProductRuntimeHandle.open({ + mode: "server", + storage: Storage.current(), + network: { hostname: "127.0.0.1", port: 0 }, + }) try { expect((await ServerProcessLock.read())?.mode).toBe("server") expect(ScopeStartup.resident()).toBe(true) diff --git a/packages/product-runtime/test/server/runtime-handle.test.ts b/packages/product-runtime/test/server/runtime-handle.test.ts index 7146d8f35..ad6259fbb 100644 --- a/packages/product-runtime/test/server/runtime-handle.test.ts +++ b/packages/product-runtime/test/server/runtime-handle.test.ts @@ -17,6 +17,7 @@ test("one-shot owns its Home, omits autonomous recovery, and awaits idempotent s const recovery: Array = [] const runtime = await ProductRuntimeHandle.open({ mode: "oneshot", + storage: Storage.current(), network: { hostname: "127.0.0.1", port: 0 }, recoveryReporter: { progress: (current) => recovery.push(current), @@ -62,6 +63,7 @@ test("failed recovery does not announce completion or retain home ownership", as await expect( ProductRuntimeHandle.open({ mode: "oneshot", + storage: Storage.current(), recoveryReporter: { progress: (current) => recovery.push(current), completed: () => recovery.push("completed"), From 50679f44408145ed7b6d0552b05cc06e897975a3 Mon Sep 17 00:00:00 2001 From: EricSanchezok <115814526+EricSanchezok@users.noreply.github.com> Date: Mon, 14 Sep 2026 22:41:15 +0800 Subject: [PATCH 07/14] fix(storage): preserve terminal writes during group cancellation Give the POSIX SQLite worker its own process group so SIGINT and SIGTERM cannot close authority before the Runtime drains cancellation evidence. Verify real group signals, terminal accounting and owner-loss recovery. Co-authored-by: synergy-agent <299070056+synergy-agent@users.noreply.github.com> --- .synergy/skill/change-persistence/SKILL.md | 3 +- docs/architecture/agent-storage.md | 2 +- ...026-09-14-transactional-agent-authority.md | 2 +- packages/cli/test/cli/send-signal.test.ts | 215 +++++++++--------- packages/harness/src/storage/sqlite-driver.ts | 2 + .../test/storage/crash-recovery.test.ts | 65 ++++++ 6 files changed, 180 insertions(+), 109 deletions(-) diff --git a/.synergy/skill/change-persistence/SKILL.md b/.synergy/skill/change-persistence/SKILL.md index e9926a516..274c24b90 100644 --- a/.synergy/skill/change-persistence/SKILL.md +++ b/.synergy/skill/change-persistence/SKILL.md @@ -21,7 +21,8 @@ description: Add or modify Synergy durable state, JSON storage keys, SQLite tabl 4. Treat commit uncertainty as an unresolved result; reconcile the operation receipt before retrying. Preserve storage, ownership and integrity errors instead of treating them as missing records. 5. Flush artifact bytes before publishing references. Stage unpublished large imports and register resumable post-deletion cleanup. Rollout evidence retains its separate allocation/evidence and projection/head transactions. 6. Physical writes retain the atomic-file transient-retry contract for Windows sharing violations. Extend the real-file retry tests when changing that helper. -7. Run the shared SQLite/PostgreSQL contract tests for engine changes; CI requires real PostgreSQL 16–18. macOS source development needs the verified SQLite engine from `bun packages/harness/script/build-sqlite.ts`. +7. Keep the SQLite subprocess alive through owner process-group cancellation so terminal evidence can drain. Verify real `SIGINT`/`SIGTERM` delivery to an isolated owner group and forced owner loss; explicit shutdown, request deadlines and parent-disconnection cleanup must still terminate the worker. +8. Run the shared SQLite/PostgreSQL contract tests for engine changes; CI requires real PostgreSQL 16–18. macOS source development needs the verified SQLite engine from `bun packages/harness/script/build-sqlite.ts`. ### SQLite and other domain stores diff --git a/docs/architecture/agent-storage.md b/docs/architecture/agent-storage.md index bcf2b3a00..22331f78a 100644 --- a/docs/architecture/agent-storage.md +++ b/docs/architecture/agent-storage.md @@ -32,7 +32,7 @@ State notifications enter a durable SQL outbox in the business transaction. Publ ## Engines and limits -SQLite runs in a dedicated Bun subprocess, keeping synchronous SQL off the Control Plane event loop. It uses WAL, `synchronous=FULL`, separate read and write connections, bounded admission, IPC byte limits, operation deadlines and explicit child drainage. macOS packages include a checksum-verified SQLite engine; initialization rejects versions without the WAL reset fix. See the [SQLite WAL documentation](https://www.sqlite.org/wal.html) and [synchronous pragma](https://www.sqlite.org/pragma.html#pragma_synchronous). +SQLite runs in a dedicated Bun subprocess, keeping synchronous SQL off the Control Plane event loop. On POSIX the worker owns a separate process group so an Agent group cancellation cannot interrupt terminal persistence. The Runtime still closes it explicitly, and IPC disconnection or owner death terminates it; forced worker deadlines remain effective. It uses WAL, `synchronous=FULL`, separate read and write connections, bounded admission, IPC byte limits, operation deadlines and explicit child drainage. macOS packages include a checksum-verified SQLite engine; initialization rejects versions without the WAL reset fix. See the [SQLite WAL documentation](https://www.sqlite.org/wal.html) and [synchronous pragma](https://www.sqlite.org/pragma.html#pragma_synchronous). PostgreSQL keeps its advisory ownership connection in a separate single-connection pool; loss of that connection permanently fences the Handle. SQLSTATE-based serialization retries are bounded to three attempts. PostgreSQL uses Bun's native SQL driver, `SERIALIZABLE` writes, `REPEATABLE READ READ ONLY` snapshots, synchronous commit, connection limits and statement/lock deadlines. PostgreSQL 16, 17 and 18 run the same contract suite in CI. Isolation does not make external side effects transactional; see [transaction isolation](https://www.postgresql.org/docs/current/transaction-iso.html) and [advisory locks](https://www.postgresql.org/docs/current/explicit-locking.html). diff --git a/docs/decisions/implemented/architecture/2026-09-14-transactional-agent-authority.md b/docs/decisions/implemented/architecture/2026-09-14-transactional-agent-authority.md index 671936a7a..9fb917af9 100644 --- a/docs/decisions/implemented/architecture/2026-09-14-transactional-agent-authority.md +++ b/docs/decisions/implemented/architecture/2026-09-14-transactional-agent-authority.md @@ -14,7 +14,7 @@ Business transactions include their projections and notification intents. Cache Bootstrap seals an immutable original-byte backup and independently readable inventory before importing JSON. Import checkpoints, domain migration ledgers, unknown owner fields and quarantine records are retained. Activation leaves one SQL authority. Pack/merge/move export logical records and retain artifacts separately. Conflicting Session IDs preserve the target aggregate and retain the skipped source evidence. Target switching uses a durable intent and verified archive rather than two uncoordinated configuration writes. -SQLite executes in a subprocess with bounded IPC and explicit shutdown. Bun 1.3.14 worker-thread tests exposed a persistence-await deadlock, so database isolation uses the same supported process mechanism as other runtime workers. macOS packages carry a checksum-pinned SQLite 3.51.3 engine; Linux and Windows validate their embedded SQLite version at initialization. The engine requirement follows the [SQLite WAL reset fix](https://www.sqlite.org/wal.html); durability follows the [synchronous pragma](https://www.sqlite.org/pragma.html#pragma_synchronous). PostgreSQL isolation and ownership follow its [transaction isolation](https://www.postgresql.org/docs/current/transaction-iso.html) and [explicit locking](https://www.postgresql.org/docs/current/explicit-locking.html) contracts. +SQLite executes in a subprocess with bounded IPC and explicit shutdown. Its POSIX process group is separate from the Agent owner: real Docker cancellation showed that inheriting the group kills storage before terminal evidence and accounting can commit. Explicit close and parent-disconnection cleanup retain ownership without blocking cancellation. Bun 1.3.14 worker-thread tests exposed a persistence-await deadlock, so database isolation uses the same supported process mechanism as other runtime workers. macOS packages carry a checksum-pinned SQLite 3.51.3 engine; Linux and Windows validate their embedded SQLite version at initialization. The engine requirement follows the [SQLite WAL reset fix](https://www.sqlite.org/wal.html); durability follows the [synchronous pragma](https://www.sqlite.org/pragma.html#pragma_synchronous). PostgreSQL isolation and ownership follow its [transaction isolation](https://www.postgresql.org/docs/current/transaction-iso.html) and [explicit locking](https://www.postgresql.org/docs/current/explicit-locking.html) contracts. ## Alternatives considered diff --git a/packages/cli/test/cli/send-signal.test.ts b/packages/cli/test/cli/send-signal.test.ts index 3756cd655..eae92ed25 100644 --- a/packages/cli/test/cli/send-signal.test.ts +++ b/packages/cli/test/cli/send-signal.test.ts @@ -2,111 +2,114 @@ import { expect, test } from "bun:test" import path from "node:path" import { createIsolatedTestEnv } from "@ericsanchezok/synergy-testing/env" -for (const signal of ["SIGTERM", "SIGINT"] as const) - test(`local send drains ${signal} in its owning Scope and retains accounting`, async () => { - const isolation = await createIsolatedTestEnv() - const started = Promise.withResolvers() - const server = Bun.serve({ - hostname: "127.0.0.1", - port: 0, - async fetch(request) { - const input = await request.json() - if (!input.stream) - return Response.json({ - id: "fixture", - object: "chat.completion", - created: 0, - model: "fixture", - choices: [{ index: 0, message: { role: "assistant", content: "Fixture" }, finish_reason: "stop" }], - usage: { prompt_tokens: 10, completion_tokens: 1, total_tokens: 11 }, - }) - return new Response( - new ReadableStream({ - start(controller) { - controller.enqueue( - new TextEncoder().encode( - `data: ${JSON.stringify({ - id: "fixture", - object: "chat.completion.chunk", - created: 0, - model: "fixture", - choices: [ - { index: 0, delta: { role: "assistant", content: "Retained prefix" }, finish_reason: null }, - ], - })}\n\n`, - ), - ) - started.resolve() - }, - }), - { headers: { "content-type": "text/event-stream" } }, - ) - }, - }) - const config = { - model: "fixture/fixture", - ...Object.fromEntries( - ["nano", "mini", "mid", "thinking", "long_context", "creative", "vision"].map((role) => [ - `${role}_model`, - "fixture/fixture", - ]), - ), - execution: { agentWorkers: 1, agentWorkerMinIdle: 0 }, - provider: { - fixture: { - name: "Fixture", - npm: "@ai-sdk/openai-compatible", - env: [], - models: { fixture: { name: "Fixture", tool_call: true, limit: { context: 128000, output: 4096 } } }, - options: { apiKey: "fixture", baseURL: server.url.toString() }, +for (const target of ["process", ...(process.platform === "win32" ? [] : ["group"])]) + for (const signal of ["SIGTERM", "SIGINT"] as const) + test(`local send drains ${target} ${signal} in its owning Scope and retains accounting`, async () => { + const isolation = await createIsolatedTestEnv() + const started = Promise.withResolvers() + const server = Bun.serve({ + hostname: "127.0.0.1", + port: 0, + async fetch(request) { + const input = await request.json() + if (!input.stream) + return Response.json({ + id: "fixture", + object: "chat.completion", + created: 0, + model: "fixture", + choices: [{ index: 0, message: { role: "assistant", content: "Fixture" }, finish_reason: "stop" }], + usage: { prompt_tokens: 10, completion_tokens: 1, total_tokens: 11 }, + }) + return new Response( + new ReadableStream({ + start(controller) { + controller.enqueue( + new TextEncoder().encode( + `data: ${JSON.stringify({ + id: "fixture", + object: "chat.completion.chunk", + created: 0, + model: "fixture", + choices: [ + { index: 0, delta: { role: "assistant", content: "Retained prefix" }, finish_reason: null }, + ], + })}\n\n`, + ), + ) + started.resolve() + }, + }), + { headers: { "content-type": "text/event-stream" } }, + ) }, - }, - } - const child = Bun.spawn( - [ - process.execPath, - path.resolve(import.meta.dir, "../../src/index.ts"), - "send", - "Reply briefly", - "--format", - "json", - "--non-interactive", - "--timeout", - "120", - ], - { - cwd: isolation.env.SYNERGY_TEST_ROOT, - env: { ...isolation.env, SYNERGY_CONFIG_CONTENT: JSON.stringify(config) }, - stdin: "ignore", - stdout: "pipe", - stderr: "pipe", - }, - ) - const output = new Response(child.stdout).text() - const errors = new Response(child.stderr).text() - try { - await Promise.race([ - started.promise, - child.exited.then(async (code) => { - throw new Error(`Send exited before provider stream: ${code}\n${await errors}\n${await output}`) - }), - ]) - child.kill(signal) - const [code, stdout, stderr] = await Promise.all([child.exited, output, errors]) - expect(code, stderr + stdout).toBe(130) - const terminal = JSON.parse(stdout.trim().split("\n").at(-1)!) - expect(terminal).toMatchObject({ - type: "result", - outcome: "cancelled", - exitCode: 130, - result: { run: { status: "cancelled", recording: "partial" } }, }) - expect(terminal.result.accounting.tokens.input.unknown).toBeGreaterThan(0) - expect(stderr).not.toContain("No context found for scope") - } finally { - if (child.exitCode === null) child.kill("SIGKILL") - await child.exited - server.stop(true) - await isolation.dispose() - } - }, 45_000) + const config = { + model: "fixture/fixture", + ...Object.fromEntries( + ["nano", "mini", "mid", "thinking", "long_context", "creative", "vision"].map((role) => [ + `${role}_model`, + "fixture/fixture", + ]), + ), + execution: { agentWorkers: 1, agentWorkerMinIdle: 0 }, + provider: { + fixture: { + name: "Fixture", + npm: "@ai-sdk/openai-compatible", + env: [], + models: { fixture: { name: "Fixture", tool_call: true, limit: { context: 128000, output: 4096 } } }, + options: { apiKey: "fixture", baseURL: server.url.toString() }, + }, + }, + } + const child = Bun.spawn( + [ + process.execPath, + path.resolve(import.meta.dir, "../../src/index.ts"), + "send", + "Reply briefly", + "--format", + "json", + "--non-interactive", + "--timeout", + "120", + ], + { + cwd: isolation.env.SYNERGY_TEST_ROOT, + env: { ...isolation.env, SYNERGY_CONFIG_CONTENT: JSON.stringify(config) }, + detached: target === "group", + stdin: "ignore", + stdout: "pipe", + stderr: "pipe", + }, + ) + const output = new Response(child.stdout).text() + const errors = new Response(child.stderr).text() + try { + await Promise.race([ + started.promise, + child.exited.then(async (code) => { + throw new Error(`Send exited before provider stream: ${code}\n${await errors}\n${await output}`) + }), + ]) + if (target === "group") process.kill(-child.pid, signal) + else child.kill(signal) + const [code, stdout, stderr] = await Promise.all([child.exited, output, errors]) + expect(code, stderr + stdout).toBe(130) + const terminal = JSON.parse(stdout.trim().split("\n").at(-1)!) + expect(terminal).toMatchObject({ + type: "result", + outcome: "cancelled", + exitCode: 130, + result: { run: { status: "cancelled", recording: "partial" } }, + }) + expect(terminal.result.accounting.tokens.input.unknown).toBeGreaterThan(0) + expect(stderr).not.toContain("No context found for scope") + } finally { + if (child.exitCode === null) child.kill("SIGKILL") + await child.exited + server.stop(true) + await isolation.dispose() + } + }, 45_000) diff --git a/packages/harness/src/storage/sqlite-driver.ts b/packages/harness/src/storage/sqlite-driver.ts index 80d6d787a..f80171bd4 100644 --- a/packages/harness/src/storage/sqlite-driver.ts +++ b/packages/harness/src/storage/sqlite-driver.ts @@ -37,6 +37,8 @@ export class SqliteDriver implements SqlDriver { this.worker = Bun.spawn({ cmd: existsSync(entry) ? [process.execPath, "run", entry] : [process.execPath, "__storage-worker-runner"], env: { ...process.env, SYNERGY_STORAGE_PARENT_PID: String(process.pid) }, + // Group cancellation must leave storage alive until its owner drains terminal writes. + detached: process.platform !== "win32", serialization: "advanced", stdout: "ignore", stderr: "inherit", diff --git a/packages/harness/test/storage/crash-recovery.test.ts b/packages/harness/test/storage/crash-recovery.test.ts index 081c9e255..b167ce9ff 100644 --- a/packages/harness/test/storage/crash-recovery.test.ts +++ b/packages/harness/test/storage/crash-recovery.test.ts @@ -93,3 +93,68 @@ for (const stage of ["inside", "committed"] as const) { } }, 15_000) } + +for (const signal of ["SIGINT", "SIGTERM"] as const) { + test.skipIf(process.platform === "win32")( + `process-group ${signal} allows the owner to persist terminal evidence before closing storage`, + async () => { + const root = await fs.mkdtemp(path.join(process.env.SYNERGY_TEST_ROOT!, "storage-signal-")) + const filename = path.join(root, "agent.sqlite") + const ready = Promise.withResolvers() + const child = Bun.spawn({ + cmd: [ + process.execPath, + "--eval", + ` + const { TransactionalStore } = await import(process.env.TEST_STORAGE_MODULE) + const stopping = Promise.withResolvers() + process.on("SIGINT", () => stopping.resolve()) + process.on("SIGTERM", () => stopping.resolve()) + const store = await TransactionalStore.open({ backend: "sqlite", filename: process.env.TEST_STORAGE_FILE, namespace: "signal" }) + try { + process.send({ ready: true }) + await stopping.promise + await store.transaction(async (tx) => { + await tx.write(["terminal"], { status: "cancelled" }) + await tx.write(["accounting"], { complete: false }) + }) + } finally { + await store.close() + process.disconnect() + } + `, + ], + detached: true, + env: { ...process.env, TEST_STORAGE_MODULE: entry, TEST_STORAGE_FILE: filename }, + stdout: "ignore", + stderr: "pipe", + ipc(message: unknown) { + if (message && typeof message === "object" && "ready" in message) ready.resolve() + }, + onExit(_child, code) { + ready.reject(new Error(`Child exited before the signal boundary: ${code}`)) + }, + }) + const stderr = new Response(child.stderr).text() + try { + await ready.promise + process.kill(-child.pid, signal) + expect({ code: await child.exited, stderr: await stderr }).toEqual({ code: 0, stderr: "" }) + const reopened = await TransactionalStore.open({ backend: "sqlite", filename, namespace: "signal" }) + try { + expect(await reopened.readMany([["terminal"], ["accounting"]])).toEqual([ + { status: "cancelled" }, + { complete: false }, + ]) + } finally { + await reopened.close() + } + } finally { + child.kill("SIGKILL") + await child.exited + await fs.rm(root, { recursive: true, force: true }) + } + }, + 15_000, + ) +} From aaa5fd039c14593de5df6409b6839b4167828f96 Mon Sep 17 00:00:00 2001 From: EricSanchezok <115814526+EricSanchezok@users.noreply.github.com> Date: Mon, 14 Sep 2026 23:14:36 +0800 Subject: [PATCH 08/14] test(storage): cover maintenance commands with runtime ownership Exercise real bootstrap, recovery, inspection and target switching through CLI commands. Keep fresh Runtime composition and coverage attribution consistent across test runners. Co-authored-by: synergy-agent <299070056+synergy-agent@users.noreply.github.com> --- .synergy/skill/testing-guide/SKILL.md | 1 + ...026-09-14-transactional-agent-authority.md | 2 + packages/cli/AGENTS.md | 2 + .../cli/test/cli/data-storage-command.test.ts | 120 ++++++++++++++++++ packages/testing/AGENTS.md | 1 + packages/testing/script/coverage-run.ts | 78 ++++++++---- packages/testing/script/test-ci.ts | 7 +- .../testing/test/script/coverage-run.test.ts | 21 +++ script/coverage-exempt.json | 8 ++ 9 files changed, 213 insertions(+), 27 deletions(-) create mode 100644 packages/cli/test/cli/data-storage-command.test.ts diff --git a/.synergy/skill/testing-guide/SKILL.md b/.synergy/skill/testing-guide/SKILL.md index ba7a40370..422a18593 100644 --- a/.synergy/skill/testing-guide/SKILL.md +++ b/.synergy/skill/testing-guide/SKILL.md @@ -128,6 +128,7 @@ Coverage has a floor. `bun run coverage:check` enforces per-package line/functio - Every exemption entry carries a `reason`; entries that match nothing, overlap, or cover more than 25% of a package fail validation. - Bun 1.3.14 supports no ignore comments (`istanbul ignore`, `v8 ignore`, and `c8 ignore` are all inert), so whole-file exemption is the only exclusion mechanism. Do not add ignore comments expecting them to work. - A source file never loaded by any test counts as 0% and fails the package — add a real test that loads it rather than exempting blindly. +- Runtime-owning CLI tests must start with only the shared isolation preload, because the harness preload installs a Handle that maintenance must reject. Register these suites in the shared batch planner; preserve the original package's coverage report directory when selecting the fresh composition. - For Solid wrappers exercised through a Vite-compiled DOM fixture, verify whether Bun attributes coverage to the emitted bundle instead of the TSX source. An exact-file exemption must identify the behavioral suite and this instrumentation boundary; keep directly testable logic measured separately. Use [Development reference](../../../docs/reference/development.md) and [Open-source quality](../../../docs/operations/open-source-quality.md) for current command ownership. Do not invent a root `bun test`; the root script intentionally rejects that ambiguous command. diff --git a/docs/decisions/implemented/architecture/2026-09-14-transactional-agent-authority.md b/docs/decisions/implemented/architecture/2026-09-14-transactional-agent-authority.md index 9fb917af9..b16a388d0 100644 --- a/docs/decisions/implemented/architecture/2026-09-14-transactional-agent-authority.md +++ b/docs/decisions/implemented/architecture/2026-09-14-transactional-agent-authority.md @@ -31,3 +31,5 @@ SQLite executes in a subprocess with bounded IPC and explicit shutdown. Its POSI The architecture can embed Agent storage without a project directory controlling record resolution. Transactions make canonical updates and their projections reviewable as one operation; SQLite and PostgreSQL share behavior and CI coverage. The cost is explicit Handle ownership, staged file operations, migration inventory, engine packaging and additional recovery paths. A PostgreSQL connection is not automatic high availability. Namespace ownership must be recovered deliberately after an unclean owner exit, and ambiguous external actions remain ambiguous. Portable transfer and immutable backups consume extra disk space. Downgrade requires a separate restored Home and does not include changes made after the historical snapshot. These boundaries are documented in [Agent storage](../../../architecture/agent-storage.md) and the [upgrade procedure](../../../migrations/transactional-agent-storage.md). + +Storage maintenance command tests execute real bootstrap, migration and read-only inspection with the shared isolation preload. They cannot borrow the harness preload's installed Handle, because maintenance correctly refuses an active Runtime. Both test orchestrators select the same fresh composition and attribute coverage to the original source package. Only the erased SQL type contract and the separately tested IPC worker require exact-file coverage exemptions; command behavior remains measured. diff --git a/packages/cli/AGENTS.md b/packages/cli/AGENTS.md index 3f598dd0c..62e00c992 100644 --- a/packages/cli/AGENTS.md +++ b/packages/cli/AGENTS.md @@ -8,4 +8,6 @@ Own the cli implementation and its public exports. Read the root AGENTS.md and t Run bun run typecheck and the affected tests, then the root package and dependency checks. +Storage maintenance commands own their Runtime lifecycle. Run their isolated suite with `bun test --cwd ../testing ../cli/test/cli/data-storage-command.test.ts` from this package; the package test and coverage orchestrators select the same shared preload without installing a harness Handle. + Keep `runCli()` as the sole parser. Product runtime injects command metadata, a runtime factory and nested `dataCommands`; core CLI must not import product implementation packages. Test command-load failures and preserve send cancellation and recording-error exit codes. diff --git a/packages/cli/test/cli/data-storage-command.test.ts b/packages/cli/test/cli/data-storage-command.test.ts new file mode 100644 index 000000000..7689e7b1f --- /dev/null +++ b/packages/cli/test/cli/data-storage-command.test.ts @@ -0,0 +1,120 @@ +import { afterEach, beforeEach, expect, test } from "bun:test" +import fs from "node:fs/promises" +import path from "node:path" +import yargs from "yargs" +import { Global } from "@ericsanchezok/synergy-harness/global" +import { Storage } from "@ericsanchezok/synergy-harness/storage/storage" +import { StorageBootstrap } from "@ericsanchezok/synergy-harness/storage/bootstrap" +import { StorageMaintenance } from "@ericsanchezok/synergy-harness/storage/maintenance" +import { ServerProcessLock } from "@ericsanchezok/synergy-harness/util/server-process-lock" +import { DataStorageCommand } from "../../src/cli/cmd/data/storage" + +const originalHome = process.env.SYNERGY_HOME +const originalLog = console.log +const originalExitCode = process.exitCode +let home: string +let output: string[] = [] +const invoke = (args: string[]) => + yargs(args) + .exitProcess(false) + .showHelpOnFail(false) + .fail((message, error) => { + throw error ?? new Error(message) + }) + .command(DataStorageCommand) + .parseAsync() +const report = () => JSON.parse(output.at(-1)!) + +beforeEach(async () => { + expect(Storage.available()).toBe(false) + home = await fs.mkdtemp(path.join(process.env.SYNERGY_TEST_ROOT!, "storage-command-")) + process.env.SYNERGY_HOME = home + output = [] + console.log = (...args) => output.push(args.map(String).join(" ")) +}) + +afterEach(async () => { + console.log = originalLog + process.exitCode = originalExitCode ?? 0 + if (originalHome === undefined) delete process.env.SYNERGY_HOME + else process.env.SYNERGY_HOME = originalHome + if (home) await fs.rm(home, { recursive: true, force: true }) +}) + +test("status is read-only before initialization and resume creates a verifiable store", async () => { + await invoke(["storage", "status"]) + expect(report()).toEqual({ phase: "uninitialized" }) + expect(await fs.readdir(home)).toEqual([]) + await expect(invoke(["storage", "verify"])).rejects.toThrow("has not been initialized") + await invoke(["storage", "resume"]) + expect(report()).toEqual({ backend: "sqlite", phase: "active", status: "ready" }) + expect(Storage.available()).toBe(false) + expect(await ServerProcessLock.read()).toBeUndefined() + await invoke(["storage", "verify"]) + expect(report()).toMatchObject({ backend: "sqlite", issues: [] }) + await invoke(["storage", "status"]) + expect(report()).toMatchObject({ backend: "sqlite", phase: "active", recoveryRecords: 0, pendingEvents: 0 }) +}) + +test("resume preserves an interrupted JSON import and migrate activates a verified target", async () => { + const legacy = path.join(Global.Path.data, "notes", "home", "retained.json") + await Bun.write(legacy, JSON.stringify({ text: "retained", extension: { version: 7 } })) + const prepared = await StorageBootstrap.prepare({ root: Global.Path.root }) + await prepared.store.close() + await invoke(["storage", "status"]) + expect(report()).toMatchObject({ phase: "validating", backend: "sqlite", backupID: prepared.manifest.backupID }) + await invoke(["storage", "resume"]) + expect(report().status).toBe("ready") + expect(await Bun.file(legacy).exists()).toBe(false) + { + await using handle = await StorageMaintenance.open() + await handle.store.write(["storage_recovery", "fixture"], { pending: true }) + await handle.store.transaction((tx) => + tx.enqueue({ id: "pending", scopeID: "scope", type: "changed", payload: { value: 7 } }), + ) + } + await invoke(["storage", "status"]) + expect(report()).toMatchObject({ recoveryRecords: 1, pendingEvents: 1 }) + const target = path.join(home, "target.jsonc") + await Bun.write( + target, + '{ // target configuration\n "storage": { "backend": "sqlite", "filename": "data/storage/moved.sqlite" } }', + ) + await invoke(["storage", "migrate", "--target", target]) + expect(output.at(-1)).toContain("migrated and verified") + await using moved = await StorageMaintenance.open({ readonly: true }) + expect(moved.store.options).toMatchObject({ filename: path.join(Global.Path.data, "storage", "moved.sqlite") }) + expect( + await moved.store.read<{ text: string; extension: { version: number } }>(["notes", "home", "retained"]), + ).toEqual({ text: "retained", extension: { version: 7 } }) + expect(await moved.store.pendingEventCount()).toBe(1) +}) + +test("verify reports invalid relationships without repairing or discarding evidence", async () => { + await invoke(["storage", "resume"]) + const key = ["sessions", "scope", "session", "messages", "message", "info"] + { + await using handle = await StorageMaintenance.open() + await handle.store.write(key, { id: "message", retained: true }) + } + await invoke(["storage", "verify"]) + expect(process.exitCode).toBe(1) + expect(report().issues).toContainEqual({ key, reason: "missing_session" }) + await using handle = await StorageMaintenance.open({ readonly: true }) + expect(await handle.store.read<{ id: string; retained: boolean }>(key)).toEqual({ id: "message", retained: true }) +}) + +test("maintenance rejects an installed Runtime and malformed targets without changing authority", async () => { + await invoke(["storage", "resume"]) + const manifest = await StorageBootstrap.status(Global.Path.root) + { + await using handle = await StorageMaintenance.open() + await expect(invoke(["storage", "resume"])).rejects.toThrow("cannot replace an installed Runtime Handle") + } + const target = path.join(home, "invalid.jsonc") + await Bun.write(target, "{ invalid") + await expect(invoke(["storage", "migrate", "--target", target])).rejects.toThrow("not valid JSONC") + expect(await StorageBootstrap.status(Global.Path.root)).toEqual(manifest) + expect(Storage.available()).toBe(false) + expect(await ServerProcessLock.read()).toBeUndefined() +}) diff --git a/packages/testing/AGENTS.md b/packages/testing/AGENTS.md index a6c234618..d5185225e 100644 --- a/packages/testing/AGENTS.md +++ b/packages/testing/AGENTS.md @@ -6,6 +6,7 @@ This private package owns deterministic test environments, the pinned model cata - `preload` establishes a positive `SYNERGY_TEST_HOME` and fixture root before any core import. It does not initialize core or install a provider hook. - Harness initialization and in-process AgentTurn hooks belong to harness test support. Shared fixtures receive their Scope and configuration bindings explicitly and never import the harness. - Spawn-based orchestrators use `createIsolatedTestEnv()` and pass its environment to every child. Never bypass TestHomeGuard or use the running product home. +- Suites that own Runtime startup use the shared preload without harness initialization. Register them in the batch planner so test and coverage commands keep the same isolated composition and write reports under the source package. - Fixtures stay under `SYNERGY_TEST_ROOT` until owned asynchronous work settles. Dispose child processes, timers, and runtimes before deleting the process fixture root. - Keep the pinned model catalog deterministic; source and release builds may resolve its exported JSON path without initializing test code. diff --git a/packages/testing/script/coverage-run.ts b/packages/testing/script/coverage-run.ts index c46fd3be4..d4eda1748 100644 --- a/packages/testing/script/coverage-run.ts +++ b/packages/testing/script/coverage-run.ts @@ -17,6 +17,36 @@ import path from "node:path" import { createIsolatedTestEnv } from "../src/env" const repositoryRoot = path.resolve(import.meta.dir, "../../..") +const ownsRuntimeFiles = new Set(["packages/cli/test/cli/data-storage-command.test.ts"]) + +function repositoryFile(file: string, packageRoot: string) { + return file.startsWith("packages/") + ? file + : path.relative(repositoryRoot, path.resolve(packageRoot, file)).split(path.sep).join("/") +} + +export function batchInvocation(args: string[], packageRoot = process.cwd()) { + const files = args.filter((arg) => /\.(test|spec)\.tsx?$/.test(arg)) + if (!files.some((file) => ownsRuntimeFiles.has(repositoryFile(file, packageRoot)))) return { args, cwd: packageRoot } + if (files.length !== 1) throw new Error("A Runtime-owning test must run in its own batch") + // The shared preload isolates the Home without installing a harness Runtime. + return { + args: args.map((arg) => { + if (arg === files[0]) return path.resolve(repositoryRoot, repositoryFile(arg, packageRoot)) + if (arg.startsWith("--reporter-outfile=")) + return `--reporter-outfile=${path.resolve(packageRoot, arg.slice("--reporter-outfile=".length))}` + return arg + }), + cwd: path.join(repositoryRoot, "packages/testing"), + } +} + +export function relocateCoverage(source: string, from: string, to: string) { + return source.replace( + /^SF:(.+)$/gm, + (_, file: string) => `SF:${path.relative(to, path.resolve(from, file)).split(path.sep).join("/")}`, + ) +} /** * Test files that fail even inside a small coverage shard. Each passes in @@ -68,6 +98,7 @@ const repositoryRoot = path.resolve(import.meta.dir, "../../..") * load on CI. Passes in its own process with coverage (verified 2026-09-04). */ export const ISOLATED_BATCH_FILES: ReadonlySet = new Set([ + ...ownsRuntimeFiles, // This suite owns stderr/TTY capture and resets process-wide migration progress. "packages/harness/test/migration/terminal-progress.test.ts", // Config projection registration is permanent for the process composition. @@ -146,10 +177,7 @@ export interface CoverageBatches { export function splitBatchFiles(files: string[], packageRoot = process.cwd()): CoverageBatches { const isIsolated = (file: string) => { - const key = file.startsWith("packages/") - ? file - : path.relative(repositoryRoot, path.resolve(packageRoot, file)).split(path.sep).join("/") - return ISOLATED_BATCH_FILES.has(key) + return ISOLATED_BATCH_FILES.has(repositoryFile(file, packageRoot)) } return { main: files.filter((file) => !isIsolated(file)), isolated: files.filter(isIsolated) } } @@ -201,26 +229,28 @@ export async function runBatch( env: Record, ): Promise { if (files.length === 0) return 0 - const child = Bun.spawn( - [ - process.execPath, - "test", - "--timeout", - "30000", - "--coverage", - "--coverage-reporter=lcov", - `--coverage-dir=${path.join("coverage", "shards", String(shard))}`, - ...files, - ], - { - cwd: process.cwd(), - env, - stdin: "inherit", - stdout: "inherit", - stderr: "inherit", - }, - ) - return child.exited + const invocation = batchInvocation([ + "test", + "--timeout", + "30000", + "--coverage", + "--coverage-reporter=lcov", + `--coverage-dir=${path.resolve("coverage", "shards", String(shard))}`, + ...files, + ]) + const child = Bun.spawn([process.execPath, ...invocation.args], { + cwd: invocation.cwd, + env, + stdin: "inherit", + stdout: "inherit", + stderr: "inherit", + }) + const code = await child.exited + if (invocation.cwd !== process.cwd()) { + const report = Bun.file(path.resolve("coverage", "shards", String(shard), "lcov.info")) + if (await report.exists()) await report.write(relocateCoverage(await report.text(), invocation.cwd, process.cwd())) + } + return code } export async function runBatches( diff --git a/packages/testing/script/test-ci.ts b/packages/testing/script/test-ci.ts index e63cd66d8..1b345b545 100644 --- a/packages/testing/script/test-ci.ts +++ b/packages/testing/script/test-ci.ts @@ -1,7 +1,7 @@ import fs from "node:fs/promises" import path from "node:path" import { createIsolatedTestEnv } from "../src/env" -import { batchShardCount, collectTests, shardMainFiles, splitBatchFiles } from "./coverage-run" +import { batchInvocation, batchShardCount, collectTests, shardMainFiles, splitBatchFiles } from "./coverage-run" export interface ShardPlan { batches: Array<{ files: string[]; shard: number }> @@ -55,8 +55,9 @@ export async function runSequentialShards( } export async function runBunTest(args: string[], env: Record): Promise { - const child = Bun.spawn([process.execPath, ...args], { - cwd: process.cwd(), + const invocation = batchInvocation(args) + const child = Bun.spawn([process.execPath, ...invocation.args], { + cwd: invocation.cwd, env, stdin: "inherit", stdout: "inherit", diff --git a/packages/testing/test/script/coverage-run.test.ts b/packages/testing/test/script/coverage-run.test.ts index ed50f0a5e..669f83344 100644 --- a/packages/testing/test/script/coverage-run.test.ts +++ b/packages/testing/test/script/coverage-run.test.ts @@ -4,6 +4,8 @@ import path from "node:path" import { createIsolatedTestEnv } from "../../src/env" import { batchShardCount, + batchInvocation, + relocateCoverage, collectTests, ISOLATED_BATCH_FILES, runBatches, @@ -45,6 +47,7 @@ describe("coverage batch splitting", () => { "packages/agent-integrations/test/lsp/owner-runtime.test.ts", "packages/agent-integrations/test/mcp/owner-lifecycle.test.ts", "packages/cli/test/cli/data-home-command.test.ts", + "packages/cli/test/cli/data-storage-command.test.ts", "packages/cli/test/cli/migration-command.test.ts", "packages/cli/test/cli/read-commands.test.ts", "packages/cli/test/cli/transcript-commands.test.ts", @@ -103,6 +106,24 @@ describe("coverage batch splitting", () => { }) describe("main batch sharding", () => { + test("Runtime-owning suites receive only the isolation preload and cannot share a batch", () => { + const root = path.resolve(import.meta.dir, "../../..") + const owner = path.join(root, "cli") + const file = "test/cli/data-storage-command.test.ts" + expect(batchInvocation(["test", file, "--reporter-outfile=reports/test.xml"], owner)).toEqual({ + args: ["test", path.join(owner, file), `--reporter-outfile=${path.join(owner, "reports/test.xml")}`], + cwd: path.join(root, "testing"), + }) + expect(() => batchInvocation(["test", file, "test/other.test.ts"], owner)).toThrow("own batch") + }) + + test("coverage relocation preserves counts and attributes paths to the original package", () => { + const root = path.resolve(import.meta.dir, "../../..") + const source = "SF:../cli/src/command.ts\nDA:1,3\nend_of_record\nSF:src/env.ts\nDA:2,5\nend_of_record\n" + expect(relocateCoverage(source, path.join(root, "testing"), path.join(root, "cli"))).toBe( + "SF:src/command.ts\nDA:1,3\nend_of_record\nSF:../testing/src/env.ts\nDA:2,5\nend_of_record\n", + ) + }) test("batchShardCount reads SYNERGY_BATCH_SHARDS and defaults to 4", () => { expect(batchShardCount({})).toBe(4) expect(batchShardCount({ SYNERGY_BATCH_SHARDS: "6" })).toBe(6) diff --git a/script/coverage-exempt.json b/script/coverage-exempt.json index 56c7c7150..34052250e 100644 --- a/script/coverage-exempt.json +++ b/script/coverage-exempt.json @@ -1402,6 +1402,14 @@ { "glob": "src/session/tool-resolver.ts", "reason": "session/LLM-loop runtime requires the full model pipeline" + }, + { + "glob": "src/storage/sql-contract.ts", + "reason": "types-only SQL and IPC contracts, no runtime logic" + }, + { + "glob": "src/storage/sqlite-worker.ts", + "reason": "dedicated IPC child process exercised by test/storage/transactional-store.test.ts and test/storage/crash-recovery.test.ts, including committed snapshots, rollback, owner loss and process-group cancellation; Bun does not merge child-process coverage" } ] }, From bf7813ce8c8cf6188d9366b93335c9da0a2e9c14 Mon Sep 17 00:00:00 2001 From: yzxoi Date: Tue, 15 Sep 2026 15:10:52 +0800 Subject: [PATCH 09/14] fix(session): recover streaming part writes and keep order reads off the writer A retained part failure now clears when a later write for the same key succeeds and retries at every drain boundary, so one transient storage error cannot wedge a session until Runtime restart. Message order reads use a read-only snapshot and escalate to a transaction only when the stored index needs rebuilding. Co-authored-by: synergy-agent <299070056+synergy-agent@users.noreply.github.com> --- packages/harness/src/session/message-v2.ts | 41 ++++++++----------- .../harness/src/session/part-write-buffer.ts | 14 ++++--- .../test/session/part-write-buffer.test.ts | 28 +++++++++++++ 3 files changed, 55 insertions(+), 28 deletions(-) diff --git a/packages/harness/src/session/message-v2.ts b/packages/harness/src/session/message-v2.ts index e2813d4c0..024bd4ec3 100644 --- a/packages/harness/src/session/message-v2.ts +++ b/packages/harness/src/session/message-v2.ts @@ -1436,29 +1436,24 @@ export namespace MessageV2 { return cached } - const [rawState, storedMarkers] = await Promise.all([ - Storage.read(StoragePath.sessionMessageOrderState(scopeID, sessionID)).catch(() => undefined), - Storage.scan(StoragePath.sessionMessageOrderMarkersRoot(scopeID, sessionID)), - ]) - const state = MessageOrderState.safeParse(rawState) - const markers = storedMarkers.toSorted(compareMessageOrderMarker) - const indexedIDs = new Set(markers.map(markerMessageID).filter((id): id is string => id !== undefined)) - if (state.success && state.data.count === markers.length && indexedIDs.size === markers.length) { - return cacheMessageOrder(scopeID, sessionID, markers) - } - return rebuildMessageOrder(scopeID, sessionID) - } - - async function messageOrderSnapshot(scopeID: Identifier.ScopeID, sessionID: Identifier.SessionID) { - return Storage.transaction(async () => { - const key = messageOrderKey(scopeID, sessionID) - if (messageOrderCache().has(key)) { - const cached = Storage.inTransaction() ? undefined : messageOrderCache().get(key) - if (cached) return cached.markers.slice() - } - - return (await loadMessageOrder(scopeID, sessionID)).markers.slice() + // A read-only snapshot keeps the hot read path off the single SQLite + // writer; a rebuild escalates to its own transaction only when the + // stored index is unusable. + const consistent = await Storage.snapshot(async () => { + const [rawState, storedMarkers] = await Promise.all([ + Storage.read(StoragePath.sessionMessageOrderState(scopeID, sessionID)).catch(() => undefined), + Storage.scan(StoragePath.sessionMessageOrderMarkersRoot(scopeID, sessionID)), + ]) + const state = MessageOrderState.safeParse(rawState) + const markers = storedMarkers.toSorted(compareMessageOrderMarker) + const indexedIDs = new Set(markers.map(markerMessageID).filter((id): id is string => id !== undefined)) + return state.success && state.data.count === markers.length && indexedIDs.size === markers.length + ? markers + : undefined }) + if (consistent) return cacheMessageOrder(scopeID, sessionID, consistent) + if (Storage.inTransaction()) return rebuildMessageOrder(scopeID, sessionID) + return Storage.transaction(() => rebuildMessageOrder(scopeID, sessionID)) } export async function writeInfo(input: { scopeID: Identifier.ScopeID; info: Info }) { @@ -1553,7 +1548,7 @@ export namespace MessageV2 { } export async function* readNewestInfos(input: { scopeID: Identifier.ScopeID; sessionID: Identifier.SessionID }) { - const markers = await messageOrderSnapshot(input.scopeID, input.sessionID) + const markers = (await loadMessageOrder(input.scopeID, input.sessionID)).markers.slice() let index = markers.length - 1 let yielded = 0 while (index >= 0) { diff --git a/packages/harness/src/session/part-write-buffer.ts b/packages/harness/src/session/part-write-buffer.ts index d11cb7776..05f62e3b3 100644 --- a/packages/harness/src/session/part-write-buffer.ts +++ b/packages/harness/src/session/part-write-buffer.ts @@ -54,6 +54,7 @@ export class PartWriteBuffer { const promise = writing.then( () => { this.bytes -= entry.bytes + this.failures.delete(key) if (this.running.get(key)?.promise === promise) this.running.delete(key) }, (error: unknown) => { @@ -71,8 +72,6 @@ export class PartWriteBuffer { async writeNow(key: string, path: P, value: T, write = this.write): Promise { this.cancel(key) - const failure = this.failures.get(key) - if (failure) throw failure.error await this.execute( key, { path, value: structuredClone(value), bytes: Buffer.byteLength(JSON.stringify(value)) }, @@ -88,10 +87,15 @@ export class PartWriteBuffer { const keys = new Set() for (const [key, entry] of this.latest) if (predicate(entry.value, entry.path)) keys.add(key) for (const [key, { entry }] of this.running) if (predicate(entry.value, entry.path)) keys.add(key) - const results = await Promise.allSettled([...keys].map((key) => this.flush(key))) + // Retained failures retry at every drain boundary instead of poisoning + // later turns: one transient storage error must not wedge a session + // until the Runtime restarts. Keys with a newer buffered or running + // entry flush that entry instead, and its success clears the failure. + const retries = [...this.failures.entries()] + .filter(([key, { entry }]) => predicate(entry.value, entry.path) && !keys.has(key)) + .map(([key, { entry }]) => this.execute(key, entry)) + const results = await Promise.allSettled([...keys].map((key) => this.flush(key)).concat(retries)) const errors = results.flatMap((result) => (result.status === "rejected" ? [result.reason] : [])) - for (const { entry, error } of this.failures.values()) - if (predicate(entry.value, entry.path) && !errors.includes(error)) errors.push(error) if (errors.length === 1) throw errors[0] if (errors.length) throw new AggregateError(errors, "Part persistence failed") } diff --git a/packages/harness/test/session/part-write-buffer.test.ts b/packages/harness/test/session/part-write-buffer.test.ts index c4791ebe8..917c3ff33 100644 --- a/packages/harness/test/session/part-write-buffer.test.ts +++ b/packages/harness/test/session/part-write-buffer.test.ts @@ -40,6 +40,34 @@ describe("PartWriteBuffer", () => { await Bun.sleep(10) await expect(buffer.flushAll()).rejects.toThrow("disk full") }) + + test("a later successful write clears the retained failure for the same key", async () => { + let fail = true + const writes: string[] = [] + const buffer = new PartWriteBuffer(async (_path, value) => { + if (fail) throw new Error("transient") + writes.push(value) + }, 10_000) + buffer.defer("part", "part", "stream") + await expect(buffer.flush("part")).rejects.toThrow("transient") + fail = false + await buffer.writeNow("part", "part", "complete") + expect(writes).toEqual(["complete"]) + await buffer.flushAll() + expect(writes).toEqual(["complete"]) + }) + + test("a transient part failure does not block later drains of the same session", async () => { + let fail = true + const buffer = new PartWriteBuffer<{ sessionID: string; text: string }>(async (_path, value) => { + if (fail) throw new Error("transient") + }, 10_000) + buffer.defer("p1", "path/p1", { sessionID: "ses_1", text: "one" }) + await expect(buffer.flushWhere((value) => value.sessionID === "ses_1")).rejects.toThrow("transient") + fail = false + buffer.defer("p2", "path/p2", { sessionID: "ses_1", text: "two" }) + await buffer.flushWhere((value) => value.sessionID === "ses_1") + }) test("coalesces deferred writes: many defers, one flush writes the latest", () => { const r = recorder() const buf = new PartWriteBuffer(r.write, 10_000) From 6900f8680be5e90889c926301621304b360e69ef Mon Sep 17 00:00:00 2001 From: yzxoi Date: Tue, 15 Sep 2026 15:10:59 +0800 Subject: [PATCH 10/14] fix(storage): keep SQLite files owner-only and exempt maintenance from the request deadline The worker enforces an owner-only umask so the database and its WAL sidecars never appear world-readable, and the driver re-tightens them to 0600 on open to repair sidecars from older engines. Integrity verification runs under an extended deadline instead of killing the worker mid-maintenance on large datasets. Co-authored-by: synergy-agent <299070056+synergy-agent@users.noreply.github.com> --- packages/harness/src/storage/sql-contract.ts | 9 +++- packages/harness/src/storage/sqlite-driver.ts | 54 +++++++++++++++---- packages/harness/src/storage/sqlite-worker.ts | 5 ++ .../src/storage/transactional-store.ts | 2 +- .../test/storage/transactional-store.test.ts | 16 ++++++ 5 files changed, 75 insertions(+), 11 deletions(-) diff --git a/packages/harness/src/storage/sql-contract.ts b/packages/harness/src/storage/sql-contract.ts index 5bcb548fc..32c39349b 100644 --- a/packages/harness/src/storage/sql-contract.ts +++ b/packages/harness/src/storage/sql-contract.ts @@ -1,8 +1,14 @@ export type SqlValue = string | number | bigint | Uint8Array | null export type SqlRow = Record +export interface SqlQueryOptions { + // Maintenance statements (integrity verification) legitimately run longer + // than ordinary operations; engines may extend their deadline. + maintenance?: boolean +} + export interface SqlConnection { - query(statement: string, values?: SqlValue[]): Promise + query(statement: string, values?: SqlValue[], options?: SqlQueryOptions): Promise } export interface SqlDriver extends SqlConnection { @@ -29,6 +35,7 @@ export type SqliteRequest = { reader?: boolean statement?: string values?: SqlValue[] + maintenance?: boolean } export type SqliteResponse = { diff --git a/packages/harness/src/storage/sqlite-driver.ts b/packages/harness/src/storage/sqlite-driver.ts index f80171bd4..c29d96525 100644 --- a/packages/harness/src/storage/sqlite-driver.ts +++ b/packages/harness/src/storage/sqlite-driver.ts @@ -11,7 +11,15 @@ import { } from "./errors" import { ServerProcessLock } from "../util/server-process-lock" import { StorageQueue } from "./queue" -import type { SqlConnection, SqlDriver, SqliteRequest, SqliteResponse, SqlRow, SqlValue } from "./sql-contract" +import type { + SqlConnection, + SqlDriver, + SqlQueryOptions, + SqliteRequest, + SqliteResponse, + SqlRow, + SqlValue, +} from "./sql-contract" export class SqliteDriver implements SqlDriver { readonly backend = "sqlite" as const @@ -87,7 +95,17 @@ export class SqliteDriver implements SqlDriver { } } await driver.request({ action: "open", filename, readonly }) - if (!readonly && process.platform !== "win32") await fs.chmod(filename, 0o600) + if (!readonly && process.platform !== "win32") { + // The worker's umask keeps new files owner-only; chmod also repairs + // sidecars left behind by an older engine before this invariant. + for (const suffix of ["", "-wal", "-shm"]) { + try { + await fs.chmod(`${filename}${suffix}`, 0o600) + } catch (error) { + if ((error as NodeJS.ErrnoException).code !== "ENOENT") throw error + } + } + } return driver } catch (error) { driver.worker.kill() @@ -108,12 +126,16 @@ export class SqliteDriver implements SqlDriver { if (this.queuedBytes + bytes > 32 * 1024 * 1024) return Promise.reject(new StorageBusyError("Authoritative storage byte queue is full")) const id = ++this.sequence + // Maintenance statements (integrity verification over the whole database) + // legitimately outlast ordinary operations; killing the worker at the + // shared deadline would close the driver mid-maintenance. + const deadline = request.maintenance ? 600_000 : 30_000 const promise = new Promise((resolve, reject) => { const timeout = setTimeout(() => { this.closed = true this.worker.kill() reject(new StorageBusyError("SQLite worker exceeded its request deadline")) - }, 30_000) + }, deadline) this.pending.set(id, { resolve, reject, bytes, timeout }) }) this.queuedBytes += bytes @@ -129,10 +151,14 @@ export class SqliteDriver implements SqlDriver { return promise } - query(statement: string, values: SqlValue[] = []): Promise { - return this.readerQueue.run(() => this.request({ action: "query", reader: true, statement, values })) as Promise< - Row[] - > + query( + statement: string, + values: SqlValue[] = [], + options?: SqlQueryOptions, + ): Promise { + return this.readerQueue.run(() => + this.request({ action: "query", reader: true, statement, values, maintenance: options?.maintenance }), + ) as Promise } transaction( @@ -141,8 +167,18 @@ export class SqliteDriver implements SqlDriver { ): Promise { const queue = options.readOnly ? this.readerQueue : this.writerQueue return queue.run(async () => { - const query = (statement: string, values: SqlValue[] = []) => - this.request({ action: "query", reader: options.readOnly, statement, values }) as Promise + const query = ( + statement: string, + values: SqlValue[] = [], + queryOptions?: SqlQueryOptions, + ) => + this.request({ + action: "query", + reader: options.readOnly, + statement, + values, + maintenance: queryOptions?.maintenance, + }) as Promise await query(options.readOnly ? "BEGIN" : "BEGIN IMMEDIATE") let committing = false try { diff --git a/packages/harness/src/storage/sqlite-worker.ts b/packages/harness/src/storage/sqlite-worker.ts index bd2b4a769..29c145c64 100644 --- a/packages/harness/src/storage/sqlite-worker.ts +++ b/packages/harness/src/storage/sqlite-worker.ts @@ -3,6 +3,11 @@ import { Database } from "bun:sqlite" import { watchManagedParent } from "../util/managed-parent" import type { SqliteRequest, SqliteResponse } from "./sql-contract" +// SQLite creates the database and its WAL sidecars (-wal/-shm) directly, +// outside AtomicFile's private mode; a restrictive umask keeps every storage +// file at owner-only permissions no matter when SQLite recreates them. +if (process.platform !== "win32") process.umask(0o077) + let writer: Database | undefined let reader: Database | undefined diff --git a/packages/harness/src/storage/transactional-store.ts b/packages/harness/src/storage/transactional-store.ts index 4e2bf4545..c00b7bd1c 100644 --- a/packages/harness/src/storage/transactional-store.ts +++ b/packages/harness/src/storage/transactional-store.ts @@ -617,7 +617,7 @@ export class TransactionalStore { return this.driver.transaction( async (connection) => { if (this.driver.backend === "sqlite") { - const rows = await connection.query("PRAGMA integrity_check") + const rows = await connection.query("PRAGMA integrity_check", [], { maintenance: true }) if (rows.length !== 1 || rows[0].integrity_check !== "ok") throw new StorageIntegrityError("SQLite integrity verification failed") } diff --git a/packages/harness/test/storage/transactional-store.test.ts b/packages/harness/test/storage/transactional-store.test.ts index f8dd46cf6..b938d141e 100644 --- a/packages/harness/test/storage/transactional-store.test.ts +++ b/packages/harness/test/storage/transactional-store.test.ts @@ -161,3 +161,19 @@ for (const backend of ["sqlite", ...(process.env.SYNERGY_TEST_POSTGRES_URL ? ["p }) }) } + +test("sqlite keeps the database and WAL sidecars owner-only", async () => { + const filename = path.join(root, "permissions.sqlite") + const store = await TransactionalStore.open({ backend: "sqlite", namespace: "permissions", filename }) + try { + await store.write(["record"], { value: 1 }) + const mode = async (suffix: string) => (await fs.stat(`${filename}${suffix}`)).mode & 0o777 + if (process.platform !== "win32") { + expect(await mode("")).toBe(0o600) + expect(await mode("-wal")).toBe(0o600) + expect(await mode("-shm")).toBe(0o600) + } + } finally { + await store.close() + } +}) From 082577b674c7f0f546acfa3a3748b9db486a611b Mon Sep 17 00:00:00 2001 From: yzxoi Date: Tue, 15 Sep 2026 15:11:15 +0800 Subject: [PATCH 11/14] fix(storage): refuse authority records from another home's portable archive Grants, consent and trust decisions (plugin approvals, permissions, registry and related plugin state) never cross homes through an untrusted data merge or the bootstrap convenience import: an imported approval would silently satisfy the consent prompt for a later plugin install. Same-home relocation (data move) opts in explicitly. Co-authored-by: synergy-agent <299070056+synergy-agent@users.noreply.github.com> --- packages/harness/src/storage/bootstrap.ts | 9 ++-- packages/harness/src/storage/portable.ts | 15 +++++++ packages/product-runtime/src/cli/data/move.ts | 2 +- .../product-runtime/src/cli/data/transfer.ts | 19 ++++++-- .../test/cli/data-transfer.test.ts | 43 +++++++++++++++++++ 5 files changed, 81 insertions(+), 7 deletions(-) diff --git a/packages/harness/src/storage/bootstrap.ts b/packages/harness/src/storage/bootstrap.ts index 6d72b3a42..aaa6e41c5 100644 --- a/packages/harness/src/storage/bootstrap.ts +++ b/packages/harness/src/storage/bootstrap.ts @@ -5,7 +5,7 @@ import path from "node:path" import { z } from "zod" import { withFileLock } from "@ericsanchezok/synergy-util/fs-lock" import { AtomicFile } from "./atomic-file" -import { StoragePortable } from "./portable" +import { authorityRecordRoots, StoragePortable } from "./portable" import { StorageConfiguration, readStorageConfiguration, resolveStoreOptions } from "./config" import { StorageIntegrityError } from "./errors" import { LegacyJsonImporter, legacySources, legacyRecordKey, type ImportProgress } from "./legacy-import" @@ -244,13 +244,16 @@ export namespace StorageBootstrap { operationID: `portable-${manifest.backupID}`, accept: (entry) => entry.type !== "record" || - ![ + (![ "storage_meta", "storage_import", "storage_import_files", "storage_staging", "storage_transfer", - ].includes(entry.key[0]), + ].includes(entry.key[0]) && + // A convenience archive may originate from another home; + // grants and trust decisions must not arrive with it. + !authorityRecordRoots.has(entry.key[0])), }) manifest.phase = "validating" await persist() diff --git a/packages/harness/src/storage/portable.ts b/packages/harness/src/storage/portable.ts index 56603d85b..533cb5e90 100644 --- a/packages/harness/src/storage/portable.ts +++ b/packages/harness/src/storage/portable.ts @@ -39,6 +39,21 @@ const Header = z.object({ format: z.literal("synergy-agent-data"), version: z.li const Footer = z .object({ end: z.literal(true), count: z.number().int().nonnegative(), sha256: z.string().regex(/^[a-f0-9]{64}$/) }) .strict() +// Record roots that carry grants, consent or trust decisions. Portable +// archives from another home must not pre-place them: an imported approval +// would suppress the consent prompt for a later plugin install. Same-home +// relocation (data move / target switch) is the only trusted transfer. +export const authorityRecordRoots = new Set([ + "plugin-approvals", + "plugin-audit", + "plugin-incompatible", + "plugin-install-intents", + "plugin-lock", + "plugin-runtime-state", + "permission-rules", + "permissions", + "registry", +]) const MAX_LINE_BYTES = 32 * 1024 * 1024 export namespace StoragePortable { diff --git a/packages/product-runtime/src/cli/data/move.ts b/packages/product-runtime/src/cli/data/move.ts index 083dd3054..10cd95a8b 100644 --- a/packages/product-runtime/src/cli/data/move.ts +++ b/packages/product-runtime/src/cli/data/move.ts @@ -249,7 +249,7 @@ export async function executeMove(opts: MoveOptions) { try { const result = subdir === "data" - ? await DataTransfer.merge(sourceRoot, targetPath) + ? await DataTransfer.merge(sourceRoot, targetPath, { trusted: true }) : await copyDirSkipExisting( src, dst, diff --git a/packages/product-runtime/src/cli/data/transfer.ts b/packages/product-runtime/src/cli/data/transfer.ts index 3a76fe2b4..975e27945 100644 --- a/packages/product-runtime/src/cli/data/transfer.ts +++ b/packages/product-runtime/src/cli/data/transfer.ts @@ -4,7 +4,11 @@ import { fileURLToPath } from "node:url" import { existsSync } from "node:fs" import { randomUUID } from "node:crypto" import { StorageBootstrap } from "@ericsanchezok/synergy-harness/storage/bootstrap" -import { StoragePortable, type StorageEntry } from "@ericsanchezok/synergy-harness/storage/portable" +import { + authorityRecordRoots, + StoragePortable, + type StorageEntry, +} from "@ericsanchezok/synergy-harness/storage/portable" import { legacyRecordKey } from "@ericsanchezok/synergy-harness/storage/legacy-import" import { Storage } from "@ericsanchezok/synergy-harness/storage/storage" import { Session } from "@ericsanchezok/synergy-harness/session" @@ -63,7 +67,11 @@ export namespace DataTransfer { } } - export async function merge(sourceRoot: string, targetRoot: string, progress?: (progress: CopyProgress) => void) { + export async function merge( + sourceRoot: string, + targetRoot: string, + options: { progress?: (progress: CopyProgress) => void; trusted?: boolean } = {}, + ) { const source = await StorageBootstrap.inspect(sourceRoot) if (!source) throw new Error("Source storage has not been initialized") let target: StorageBootstrap.Prepared | undefined @@ -81,7 +89,7 @@ export namespace DataTransfer { const copied = await copyDirSkipExisting( source.artifactDirectory, path.join(targetRoot, "data"), - progress, + options.progress, undefined, undefined, (relative) => exclude(relative, skipped), @@ -94,6 +102,11 @@ export namespace DataTransfer { if (entry.type === "event") return false if (entry.type === "receipt") return true if (local.has(entry.key[0]) || derived.has(entry.key[0])) return false + // Grants, consent and trust decisions never cross homes through an + // untrusted merge: an imported approval would silently satisfy the + // consent prompt for a later plugin install. Same-home relocation + // (data move) opts in explicitly. + if (!options.trusted && authorityRecordRoots.has(entry.key[0])) return false const owner = sessionOwner(entry) if (owner && skipped.has(owner)) { conflicts.add(owner) diff --git a/packages/product-runtime/test/cli/data-transfer.test.ts b/packages/product-runtime/test/cli/data-transfer.test.ts index 32dc28552..f61805a27 100644 --- a/packages/product-runtime/test/cli/data-transfer.test.ts +++ b/packages/product-runtime/test/cli/data-transfer.test.ts @@ -67,6 +67,49 @@ test("merge keeps the target session aggregate and retains skipped source eviden } }) +test("merge refuses authority records from another home; trusted relocation keeps them", async () => { + await using tmp = await tmpdir() + const sourceRoot = path.join(tmp.path, "source") + const targetRoot = path.join(tmp.path, "target") + const trustedRoot = path.join(tmp.path, "trusted") + const source = await StorageBootstrap.prepare({ root: sourceRoot }) + try { + await source.store.write(["plugin-approvals", "records", "plugin-x"], { grant: "broad" }) + await source.store.write(["notes", "scope", "note"], { text: "payload" }) + await source.activate() + } finally { + await source.store.close() + } + for (const root of [targetRoot, trustedRoot]) { + const prepared = await StorageBootstrap.prepare({ root }) + try { + await prepared.activate() + } finally { + await prepared.store.close() + } + } + await using locks = await SnapshotArchive.lockHomes([sourceRoot, targetRoot, trustedRoot]) + await DataTransfer.merge(sourceRoot, targetRoot) + const target = await StorageBootstrap.inspect(targetRoot) + if (!target) throw new Error("missing target") + try { + expect((await target.store.readMany([["plugin-approvals", "records", "plugin-x"]]))[0]).toBeUndefined() + expect(await target.store.read<{ text: string }>(["notes", "scope", "note"])).toEqual({ text: "payload" }) + } finally { + await target.store.close() + } + await DataTransfer.merge(sourceRoot, trustedRoot, { trusted: true }) + const trusted = await StorageBootstrap.inspect(trustedRoot) + if (!trusted) throw new Error("missing trusted target") + try { + expect(await trusted.store.read<{ grant: string }>(["plugin-approvals", "records", "plugin-x"])).toEqual({ + grant: "broad", + }) + } finally { + await trusted.store.close() + } +}) + test("portable pack restores authority without copying the source database identity", async () => { await using tmp = await tmpdir() const sourceRoot = path.join(tmp.path, "source") From 01a0cf7325967a63474fb118f9a3a9d146c3e85a Mon Sep 17 00:00:00 2001 From: yzxoi Date: Tue, 15 Sep 2026 15:11:20 +0800 Subject: [PATCH 12/14] docs(storage): record streaming recovery, private sidecars and authority transfer boundaries Co-authored-by: synergy-agent <299070056+synergy-agent@users.noreply.github.com> --- docs/architecture/agent-storage.md | 6 +++--- .../2026-09-14-transactional-agent-authority.md | 2 +- docs/migrations/transactional-agent-storage.md | 2 +- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/docs/architecture/agent-storage.md b/docs/architecture/agent-storage.md index 22331f78a..d4e40bde2 100644 --- a/docs/architecture/agent-storage.md +++ b/docs/architecture/agent-storage.md @@ -26,13 +26,13 @@ Plugin installation has a durable recovery intent and a private snapshot of the ## Streaming and notifications -Streaming part writes coalesce for 500 ms. Terminal writes wait for prior in-flight writes; drain boundaries include timer-triggered writes and propagate persistence failures. Buffered values and caches belong to the Storage Handle. Part persistence verifies the owning Session and Message and rejects a deleted part. Every ordinary Session-owned record write also checks the Session tombstone in the same transaction, so detached Rollout settlement cannot revive deleted data. Explicit portable recovery is the only path that may restore a deleted record. +Streaming part writes coalesce for 500 ms. Terminal writes wait for prior in-flight writes; drain boundaries include timer-triggered writes and propagate persistence failures. A failed part write is retained and retried at the next drain boundary, and a successful later write for the same key clears it, so one transient storage error cannot wedge a Session until Runtime restart. Buffered values and caches belong to the Storage Handle. Part persistence verifies the owning Session and Message and rejects a deleted part. Every ordinary Session-owned record write also checks the Session tombstone in the same transaction, so detached Rollout settlement cannot revive deleted data. Explicit portable recovery is the … State notifications enter a durable SQL outbox in the business transaction. Publication and cache changes happen after commit. An observer failure cannot roll back an already committed mutation. A new Runtime changes the frontend event epoch and reconciles outstanding notifications by requiring a fresh snapshot; it does not replay arbitrary subscribers that might perform external actions. Stream deltas remain provisional until their persistence boundary completes. ## Engines and limits -SQLite runs in a dedicated Bun subprocess, keeping synchronous SQL off the Control Plane event loop. On POSIX the worker owns a separate process group so an Agent group cancellation cannot interrupt terminal persistence. The Runtime still closes it explicitly, and IPC disconnection or owner death terminates it; forced worker deadlines remain effective. It uses WAL, `synchronous=FULL`, separate read and write connections, bounded admission, IPC byte limits, operation deadlines and explicit child drainage. macOS packages include a checksum-verified SQLite engine; initialization rejects versions without the WAL reset fix. See the [SQLite WAL documentation](https://www.sqlite.org/wal.html) and [synchronous pragma](https://www.sqlite.org/pragma.html#pragma_synchronous). +SQLite runs in a dedicated Bun subprocess, keeping synchronous SQL off the Control Plane event loop. On POSIX the worker owns a separate process group so an Agent group cancellation cannot interrupt terminal persistence. The Runtime still closes it explicitly, and IPC disconnection or owner death terminates it; forced worker deadlines remain effective. It uses WAL, `synchronous=FULL`, separate read and write connections, bounded admission, IPC byte limits, operation deadlines and explicit child drainage. The worker enforces an owner-only umask and the driver re-tightens the database and its `-wal`/`-shm` sidecars to 0600 on open, so committed data never leaves the private-file discipline. Maintenance statements such as `PRAGMA integrity_check` receive an extended deadline instead of killing the worker mid-verification. macOS packages include a checksum-verified SQLite engine; initialization rejects versions without the WAL reset fix. See the [SQLite WAL documentation](https://www.sqlite.org/wal.html) and [synchronous pragma](https://www.sqlite.org/pragma.html#pragma_synchronous). PostgreSQL keeps its advisory ownership connection in a separate single-connection pool; loss of that connection permanently fences the Handle. SQLSTATE-based serialization retries are bounded to three attempts. PostgreSQL uses Bun's native SQL driver, `SERIALIZABLE` writes, `REPEATABLE READ READ ONLY` snapshots, synchronous commit, connection limits and statement/lock deadlines. PostgreSQL 16, 17 and 18 run the same contract suite in CI. Isolation does not make external side effects transactional; see [transaction isolation](https://www.postgresql.org/docs/current/transaction-iso.html) and [advisory locks](https://www.postgresql.org/docs/current/explicit-locking.html). @@ -40,7 +40,7 @@ PostgreSQL keeps its advisory ownership connection in a separate single-connecti Home ownership excludes the running legacy writer. Bootstrap backs up original bytes, hashes an independently readable inventory, imports records with resumable checkpoints, runs registered owner migrations, validates relationships and activates the new authority. Malformed historical Session data is preserved and quarantined with a persistent execution block. Permission, I/O, identity or backup-integrity failures stop activation. Once active, normal code has one SQL record path; reappearing legacy authority files cause startup to fail rather than silently selecting a dataset. -Portable data contains records, revisions, command receipts and pending events with a checksum footer. Pack, merge and move use this logical representation and separately preserve artifact bytes and Git objects; they do not copy a live SQLite database or assume PostgreSQL data resides in the Home. A conflicting Session ID keeps the target aggregate intact, including its artifacts and snapshot references. Skipped source data and a transfer report remain available even when move removes the original Home. Session indexes are rebuilt in the import transaction. +Portable data contains records, revisions, command receipts and pending events with a checksum footer. Pack, merge and move use this logical representation and separately preserve artifact bytes and Git objects; they do not copy a live SQLite database or assume PostgreSQL data resides in the Home. Record roots carrying grants, consent and trust decisions (`plugin-approvals`, `permissions`, `registry` and related plugin state) never cross homes through an untrusted `data merge` or convenience import; only same-home relocation (`data move`, target switch) carries them. A conflicting Session ID keeps the target aggregate intact, including its artifacts and snapshot references. Skipped source data and a transfer report remain available even when move removes the original Home. Session indexes are rebuilt in the import transaction. A target switch first saves a verified portable archive and a durable switch intent. Import is idempotent, verification precedes activation, and the intent blocks normal startup until both configuration and dataset identity agree. `data storage resume` completes an interrupted switch. Storage configuration cannot be hot-reloaded. Downgrade uses the immutable pre-upgrade backup in a separate Home; there is no reverse writer or live JSON mirror. diff --git a/docs/decisions/implemented/architecture/2026-09-14-transactional-agent-authority.md b/docs/decisions/implemented/architecture/2026-09-14-transactional-agent-authority.md index b16a388d0..9cd4e6ba8 100644 --- a/docs/decisions/implemented/architecture/2026-09-14-transactional-agent-authority.md +++ b/docs/decisions/implemented/architecture/2026-09-14-transactional-agent-authority.md @@ -12,7 +12,7 @@ The Runtime owns an explicit Storage Handle containing a transactional logical-r Business transactions include their projections and notification intents. Cache updates and publication follow commit. Rollout retains its application evidence journal, including its separate evidence/allocation and projection/head commits. Files commit before their database references. Session import/fork stages unpublished identities, deletion records cleanup work, and plugin installation retains a recoverable intent for its database/configuration/directory boundary. -Bootstrap seals an immutable original-byte backup and independently readable inventory before importing JSON. Import checkpoints, domain migration ledgers, unknown owner fields and quarantine records are retained. Activation leaves one SQL authority. Pack/merge/move export logical records and retain artifacts separately. Conflicting Session IDs preserve the target aggregate and retain the skipped source evidence. Target switching uses a durable intent and verified archive rather than two uncoordinated configuration writes. +Bootstrap seals an immutable original-byte backup and independently readable inventory before importing JSON. Import checkpoints, domain migration ledgers, unknown owner fields and quarantine records are retained. Activation leaves one SQL authority. Pack/merge/move export logical records and retain artifacts separately. Record roots carrying grants, consent and trust decisions never cross homes through an untrusted merge or convenience import; only same-home relocation carries them. Conflicting Session IDs preserve the target aggregate and retain the skipped source evidence. Target switching uses a durable intent and verified archive rather than two uncoordinated confi… SQLite executes in a subprocess with bounded IPC and explicit shutdown. Its POSIX process group is separate from the Agent owner: real Docker cancellation showed that inheriting the group kills storage before terminal evidence and accounting can commit. Explicit close and parent-disconnection cleanup retain ownership without blocking cancellation. Bun 1.3.14 worker-thread tests exposed a persistence-await deadlock, so database isolation uses the same supported process mechanism as other runtime workers. macOS packages carry a checksum-pinned SQLite 3.51.3 engine; Linux and Windows validate their embedded SQLite version at initialization. The engine requirement follows the [SQLite WAL reset fix](https://www.sqlite.org/wal.html); durability follows the [synchronous pragma](https://www.sqlite.org/pragma.html#pragma_synchronous). PostgreSQL isolation and ownership follow its [transaction isolation](https://www.postgresql.org/docs/current/transaction-iso.html) and [explicit locking](https://www.postgresql.org/docs/current/explicit-locking.html) contracts. diff --git a/docs/migrations/transactional-agent-storage.md b/docs/migrations/transactional-agent-storage.md index ea43d8b27..e5b9c827c 100644 --- a/docs/migrations/transactional-agent-storage.md +++ b/docs/migrations/transactional-agent-storage.md @@ -24,4 +24,4 @@ There is no SQL-to-legacy live writer. Restore the pre-upgrade snapshot into a s ## Transfer -Use `data pack` for a portable logical backup, and `data merge` or `data move` to restore or combine data. Agent records, revisions and operation receipts move independently of the engine; Git objects and artifacts move with their references. A target Session ID wins as a whole aggregate. The retained source snapshot and transfer report let an operator recover skipped content even after `move --remove-original`. +Use `data pack` for a portable logical backup, and `data merge` or `data move` to restore or combine data. Agent records, revisions and operation receipts move independently of the engine; Git objects and artifacts move with their references. Grants, consent and trust records (plugin approvals, permissions, registry) are refused from another home's archive during `data merge` and only travel with `data move`. A target Session ID wins as a whole aggregate. The retained source snapshot and transfer report let an operator recover skipped content even after `move --remove-original`. From d32db644c2f4069baf289a129d878088ca4ee20c Mon Sep 17 00:00:00 2001 From: yzxoi Date: Tue, 15 Sep 2026 17:17:52 +0800 Subject: [PATCH 13/14] revert(session): keep message order reads on the write transaction The read-only snapshot rewrite of loadMessageOrder broke Cortex task output: subagent assistant messages were read back empty after their finalizing turn (summary extraction, final_response capture, usage reads and durable output all lost). Root cause was isolated by reverting only message-v2.ts at bf7813ce8 while keeping the part-write-buffer recovery; the original messageOrderSnapshot semantics are restored until the read path can move off the writer without changing read-after-write visibility. Co-authored-by: synergy-agent <299070056+synergy-agent@users.noreply.github.com> --- packages/harness/src/session/message-v2.ts | 41 ++++++++++++---------- 1 file changed, 23 insertions(+), 18 deletions(-) diff --git a/packages/harness/src/session/message-v2.ts b/packages/harness/src/session/message-v2.ts index 024bd4ec3..e2813d4c0 100644 --- a/packages/harness/src/session/message-v2.ts +++ b/packages/harness/src/session/message-v2.ts @@ -1436,24 +1436,29 @@ export namespace MessageV2 { return cached } - // A read-only snapshot keeps the hot read path off the single SQLite - // writer; a rebuild escalates to its own transaction only when the - // stored index is unusable. - const consistent = await Storage.snapshot(async () => { - const [rawState, storedMarkers] = await Promise.all([ - Storage.read(StoragePath.sessionMessageOrderState(scopeID, sessionID)).catch(() => undefined), - Storage.scan(StoragePath.sessionMessageOrderMarkersRoot(scopeID, sessionID)), - ]) - const state = MessageOrderState.safeParse(rawState) - const markers = storedMarkers.toSorted(compareMessageOrderMarker) - const indexedIDs = new Set(markers.map(markerMessageID).filter((id): id is string => id !== undefined)) - return state.success && state.data.count === markers.length && indexedIDs.size === markers.length - ? markers - : undefined + const [rawState, storedMarkers] = await Promise.all([ + Storage.read(StoragePath.sessionMessageOrderState(scopeID, sessionID)).catch(() => undefined), + Storage.scan(StoragePath.sessionMessageOrderMarkersRoot(scopeID, sessionID)), + ]) + const state = MessageOrderState.safeParse(rawState) + const markers = storedMarkers.toSorted(compareMessageOrderMarker) + const indexedIDs = new Set(markers.map(markerMessageID).filter((id): id is string => id !== undefined)) + if (state.success && state.data.count === markers.length && indexedIDs.size === markers.length) { + return cacheMessageOrder(scopeID, sessionID, markers) + } + return rebuildMessageOrder(scopeID, sessionID) + } + + async function messageOrderSnapshot(scopeID: Identifier.ScopeID, sessionID: Identifier.SessionID) { + return Storage.transaction(async () => { + const key = messageOrderKey(scopeID, sessionID) + if (messageOrderCache().has(key)) { + const cached = Storage.inTransaction() ? undefined : messageOrderCache().get(key) + if (cached) return cached.markers.slice() + } + + return (await loadMessageOrder(scopeID, sessionID)).markers.slice() }) - if (consistent) return cacheMessageOrder(scopeID, sessionID, consistent) - if (Storage.inTransaction()) return rebuildMessageOrder(scopeID, sessionID) - return Storage.transaction(() => rebuildMessageOrder(scopeID, sessionID)) } export async function writeInfo(input: { scopeID: Identifier.ScopeID; info: Info }) { @@ -1548,7 +1553,7 @@ export namespace MessageV2 { } export async function* readNewestInfos(input: { scopeID: Identifier.ScopeID; sessionID: Identifier.SessionID }) { - const markers = (await loadMessageOrder(input.scopeID, input.sessionID)).markers.slice() + const markers = await messageOrderSnapshot(input.scopeID, input.sessionID) let index = markers.length - 1 let yielded = 0 while (index >= 0) { From 7cbef7dfcc31f7fef5e4b2a0150b812608aea4c0 Mon Sep 17 00:00:00 2001 From: yzxoi Date: Tue, 15 Sep 2026 19:44:49 +0800 Subject: [PATCH 14/14] fix(storage): preserve transaction rollback and portable upgrades Keep transaction-owned parts out of streaming retries, quarantine malformed legacy session aggregates before owner migrations, and bundle the verified SQLite engine in macOS Harness archives. Cover rollback, released upgrades, and packed-artifact initialization with regression tests. Co-authored-by: synergy-agent <299070056+synergy-agent@users.noreply.github.com> --- .github/workflows/build-helpers.yml | 9 ++ .synergy/skill/change-persistence/SKILL.md | 4 +- docs/architecture/agent-storage.md | 6 +- ...026-09-14-transactional-agent-authority.md | 6 +- docs/reference/packages.md | 2 +- packages/harness/AGENTS.md | 2 +- packages/harness/src/session/index.ts | 16 ++-- .../harness/src/session/part-write-buffer.ts | 5 ++ packages/harness/src/storage/legacy-import.ts | 42 +++++++++- packages/harness/src/storage/sqlite-engine.ts | 5 +- .../session/materialization-rollback.test.ts | 81 ++++++++++++++++++ .../test/session/part-write-buffer.test.ts | 23 +++++ .../test/storage/legacy-import.test.ts | 13 ++- .../released-upgrade-quarantine.test.ts | 84 +++++++++++++++++++ script/pack-workspace.ts | 5 +- test/script/release/workspace-sqlite.test.ts | 46 ++++++++++ 16 files changed, 326 insertions(+), 23 deletions(-) create mode 100644 packages/harness/test/session/materialization-rollback.test.ts create mode 100644 packages/harness/test/storage/released-upgrade-quarantine.test.ts create mode 100644 test/script/release/workspace-sqlite.test.ts diff --git a/.github/workflows/build-helpers.yml b/.github/workflows/build-helpers.yml index fb15455cf..68e846b1a 100644 --- a/.github/workflows/build-helpers.yml +++ b/.github/workflows/build-helpers.yml @@ -16,6 +16,9 @@ on: - "test/script/watcher-native.test.ts" - "packages/harness/script/build-sqlite.ts" - "packages/harness/src/storage/sqlite-engine.ts" + - "script/pack-workspace.ts" + - "script/build-workspace.ts" + - "test/script/release/workspace-sqlite.test.ts" push: branches: [dev, main] paths: @@ -29,6 +32,9 @@ on: - "test/script/watcher-native.test.ts" - "packages/harness/script/build-sqlite.ts" - "packages/harness/src/storage/sqlite-engine.ts" + - "script/pack-workspace.ts" + - "script/build-workspace.ts" + - "test/script/release/workspace-sqlite.test.ts" permissions: contents: read @@ -149,6 +155,9 @@ jobs: with: bun-version: "1.3.14" - run: bun packages/harness/script/build-sqlite.ts + - run: bun install --frozen-lockfile + - name: Verify packaged SQLite without host libraries + run: bun test --config /dev/null test/script/release/workspace-sqlite.test.ts - uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4 with: name: sqlite-assets-darwin diff --git a/.synergy/skill/change-persistence/SKILL.md b/.synergy/skill/change-persistence/SKILL.md index 274c24b90..57123a598 100644 --- a/.synergy/skill/change-persistence/SKILL.md +++ b/.synergy/skill/change-persistence/SKILL.md @@ -17,12 +17,12 @@ description: Add or modify Synergy durable state, JSON storage keys, SQLite tabl 1. Build logical keys through `StoragePath`; use an explicit `Storage.Handle`. Normal Agent record code must never read or write legacy JSON files. 2. Keep independently updated or streamed records independently addressable. Wrap the complete business mutation, indexes, receipts and outbox notifications in `Storage.transaction()`. -3. Nested writes join the caller's transaction. Defer cache and event effects until commit. Never run tools, network calls, plugin reloads, filesystem writes or buffer drains inside a retryable SQL transaction. +3. Nested writes join the caller's transaction. Defer cache and event effects until commit. Never run tools, network calls, plugin reloads, filesystem writes or buffer drains inside a retryable SQL transaction. Keep transaction-owned part writes out of streaming retry buffers; verify rollback followed by a delivery retry cannot resurrect the abandoned parts. 4. Treat commit uncertainty as an unresolved result; reconcile the operation receipt before retrying. Preserve storage, ownership and integrity errors instead of treating them as missing records. 5. Flush artifact bytes before publishing references. Stage unpublished large imports and register resumable post-deletion cleanup. Rollout evidence retains its separate allocation/evidence and projection/head transactions. 6. Physical writes retain the atomic-file transient-retry contract for Windows sharing violations. Extend the real-file retry tests when changing that helper. 7. Keep the SQLite subprocess alive through owner process-group cancellation so terminal evidence can drain. Verify real `SIGINT`/`SIGTERM` delivery to an isolated owner group and forced owner loss; explicit shutdown, request deadlines and parent-disconnection cleanup must still terminate the worker. -8. Run the shared SQLite/PostgreSQL contract tests for engine changes; CI requires real PostgreSQL 16–18. macOS source development needs the verified SQLite engine from `bun packages/harness/script/build-sqlite.ts`. +8. Run the shared SQLite/PostgreSQL contract tests for engine changes; CI requires real PostgreSQL 16–18. macOS source development needs the verified SQLite engine from `bun packages/harness/script/build-sqlite.ts`. For engine packaging changes, run `bun test --config /dev/null test/script/release/workspace-sqlite.test.ts` on macOS to open the actual unpacked archive with host libraries masked. ### SQLite and other domain stores diff --git a/docs/architecture/agent-storage.md b/docs/architecture/agent-storage.md index d4e40bde2..a0a8e18af 100644 --- a/docs/architecture/agent-storage.md +++ b/docs/architecture/agent-storage.md @@ -26,19 +26,19 @@ Plugin installation has a durable recovery intent and a private snapshot of the ## Streaming and notifications -Streaming part writes coalesce for 500 ms. Terminal writes wait for prior in-flight writes; drain boundaries include timer-triggered writes and propagate persistence failures. A failed part write is retained and retried at the next drain boundary, and a successful later write for the same key clears it, so one transient storage error cannot wedge a Session until Runtime restart. Buffered values and caches belong to the Storage Handle. Part persistence verifies the owning Session and Message and rejects a deleted part. Every ordinary Session-owned record write also checks the Session tombstone in the same transaction, so detached Rollout settlement cannot revive deleted data. Explicit portable recovery is the … +Streaming part writes coalesce for 500 ms. Terminal writes wait for prior in-flight writes; drain boundaries include timer-triggered writes and propagate persistence failures. A failed streaming write is retained and retried at the next drain boundary, and a successful later write for the same key clears it, so one transient storage error cannot wedge a Session until Runtime restart. Part writes inside an existing business transaction persist directly and never enter the streaming retry queue. Existing streaming writes for the same part must drain before entering that transaction. Buffered values and caches belong to the Storage Handle. Part persistence verifies the owning Session and Message and rejects a deleted part. Every ordinary Session-owned record write also checks the Session tombstone in the same transaction, so detached Rollout settlement cannot revive deleted data. Explicit portable recovery is the … State notifications enter a durable SQL outbox in the business transaction. Publication and cache changes happen after commit. An observer failure cannot roll back an already committed mutation. A new Runtime changes the frontend event epoch and reconciles outstanding notifications by requiring a fresh snapshot; it does not replay arbitrary subscribers that might perform external actions. Stream deltas remain provisional until their persistence boundary completes. ## Engines and limits -SQLite runs in a dedicated Bun subprocess, keeping synchronous SQL off the Control Plane event loop. On POSIX the worker owns a separate process group so an Agent group cancellation cannot interrupt terminal persistence. The Runtime still closes it explicitly, and IPC disconnection or owner death terminates it; forced worker deadlines remain effective. It uses WAL, `synchronous=FULL`, separate read and write connections, bounded admission, IPC byte limits, operation deadlines and explicit child drainage. The worker enforces an owner-only umask and the driver re-tightens the database and its `-wal`/`-shm` sidecars to 0600 on open, so committed data never leaves the private-file discipline. Maintenance statements such as `PRAGMA integrity_check` receive an extended deadline instead of killing the worker mid-verification. macOS packages include a checksum-verified SQLite engine; initialization rejects versions without the WAL reset fix. See the [SQLite WAL documentation](https://www.sqlite.org/wal.html) and [synchronous pragma](https://www.sqlite.org/pragma.html#pragma_synchronous). +SQLite runs in a dedicated Bun subprocess, keeping synchronous SQL off the Control Plane event loop. On POSIX the worker owns a separate process group so an Agent group cancellation cannot interrupt terminal persistence. The Runtime still closes it explicitly, and IPC disconnection or owner death terminates it; forced worker deadlines remain effective. It uses WAL, `synchronous=FULL`, separate read and write connections, bounded admission, IPC byte limits, operation deadlines and explicit child drainage. The worker enforces an owner-only umask and the driver re-tightens the database and its `-wal`/`-shm` sidecars to 0600 on open, so committed data never leaves the private-file discipline. Maintenance statements such as `PRAGMA integrity_check` receive an extended deadline instead of killing the worker mid-verification. macOS executables and independent Harness module archives include a checksum-verified SQLite engine, resolved relative to their own installation; initialization rejects versions without the WAL reset fix. See the [SQLite WAL documentation](https://www.sqlite.org/wal.html) and [synchronous pragma](https://www.sqlite.org/pragma.html#pragma_synchronous). PostgreSQL keeps its advisory ownership connection in a separate single-connection pool; loss of that connection permanently fences the Handle. SQLSTATE-based serialization retries are bounded to three attempts. PostgreSQL uses Bun's native SQL driver, `SERIALIZABLE` writes, `REPEATABLE READ READ ONLY` snapshots, synchronous commit, connection limits and statement/lock deadlines. PostgreSQL 16, 17 and 18 run the same contract suite in CI. Isolation does not make external side effects transactional; see [transaction isolation](https://www.postgresql.org/docs/current/transaction-iso.html) and [advisory locks](https://www.postgresql.org/docs/current/explicit-locking.html). ## Upgrade and movement -Home ownership excludes the running legacy writer. Bootstrap backs up original bytes, hashes an independently readable inventory, imports records with resumable checkpoints, runs registered owner migrations, validates relationships and activates the new authority. Malformed historical Session data is preserved and quarantined with a persistent execution block. Permission, I/O, identity or backup-integrity failures stop activation. Once active, normal code has one SQL record path; reappearing legacy authority files cause startup to fail rather than silently selecting a dataset. +Home ownership excludes the running legacy writer. Bootstrap backs up original bytes, hashes an independently readable inventory, imports records with resumable checkpoints, runs registered owner migrations, validates relationships and activates the new authority. Before owner migrations, the importer validates the historical Session title, Scope identity and creation/update timestamps without projecting through the current public schema or dropping unknown fields. Malformed historical Session data is preserved and quarantined with a persistent execution block. A quarantined Session owner also quarantines its descendant records before owner migrations run. Permission, I/O, identity or backup-integrity failures stop activation. Once active, normal code has one SQL record path; reappearing legacy authority files cause startup to fail rather than silently selecting a dataset. Portable data contains records, revisions, command receipts and pending events with a checksum footer. Pack, merge and move use this logical representation and separately preserve artifact bytes and Git objects; they do not copy a live SQLite database or assume PostgreSQL data resides in the Home. Record roots carrying grants, consent and trust decisions (`plugin-approvals`, `permissions`, `registry` and related plugin state) never cross homes through an untrusted `data merge` or convenience import; only same-home relocation (`data move`, target switch) carries them. A conflicting Session ID keeps the target aggregate intact, including its artifacts and snapshot references. Skipped source data and a transfer report remain available even when move removes the original Home. Session indexes are rebuilt in the import transaction. diff --git a/docs/decisions/implemented/architecture/2026-09-14-transactional-agent-authority.md b/docs/decisions/implemented/architecture/2026-09-14-transactional-agent-authority.md index 9cd4e6ba8..4e681a172 100644 --- a/docs/decisions/implemented/architecture/2026-09-14-transactional-agent-authority.md +++ b/docs/decisions/implemented/architecture/2026-09-14-transactional-agent-authority.md @@ -10,11 +10,11 @@ Individually atomic JSON file replacement cannot commit a Session mutation with The Runtime owns an explicit Storage Handle containing a transactional logical-record store and an independent artifact location. SQLite is the default implementation; PostgreSQL uses the same contract. One Runtime owns each namespace, with revision checks, idempotent command receipts, consistent read snapshots and a durable notification outbox. Runtime concurrency spans Sessions; this change does not introduce active-active Runtime replicas. -Business transactions include their projections and notification intents. Cache updates and publication follow commit. Rollout retains its application evidence journal, including its separate evidence/allocation and projection/head commits. Files commit before their database references. Session import/fork stages unpublished identities, deletion records cleanup work, and plugin installation retains a recoverable intent for its database/configuration/directory boundary. +Business transactions include their projections and notification intents. Cache updates and publication follow commit. Synchronous writes inside a business transaction bypass the streaming retry buffer: replaying one failed part outside its owning transaction could reconstruct rolled-back content when a delivery retries. Streaming work for that part must drain before the transaction begins. Rollout retains its application evidence journal, including its separate evidence/allocation and projection/head commits. Files commit before their database references. Session import/fork stages unpublished identities, deletion records cleanup work, and plugin installation retains a recoverable intent for its database/configuration/directory boundary. -Bootstrap seals an immutable original-byte backup and independently readable inventory before importing JSON. Import checkpoints, domain migration ledgers, unknown owner fields and quarantine records are retained. Activation leaves one SQL authority. Pack/merge/move export logical records and retain artifacts separately. Record roots carrying grants, consent and trust decisions never cross homes through an untrusted merge or convenience import; only same-home relocation carries them. Conflicting Session IDs preserve the target aggregate and retain the skipped source evidence. Target switching uses a durable intent and verified archive rather than two uncoordinated confi… +Bootstrap seals an immutable original-byte backup and independently readable inventory before importing JSON. Import checkpoints, domain migration ledgers, unknown owner fields and quarantine records are retained. Import validates the stable historical Session identity, title and timestamps before migrations traverse them; the full current schema cannot serve as an import gate because released schemas and unloaded owner fields must remain valid. Activation leaves one SQL authority. Pack/merge/move export logical records and retain artifacts separately. Record roots carrying grants, consent and trust decisions never cross homes through an untrusted merge or convenience import; only same-home relocation carries them. Conflicting Session IDs preserve the target aggregate and retain the skipped source evidence. Target switching uses a durable intent and verified archive rather than two uncoordinated confi… -SQLite executes in a subprocess with bounded IPC and explicit shutdown. Its POSIX process group is separate from the Agent owner: real Docker cancellation showed that inheriting the group kills storage before terminal evidence and accounting can commit. Explicit close and parent-disconnection cleanup retain ownership without blocking cancellation. Bun 1.3.14 worker-thread tests exposed a persistence-await deadlock, so database isolation uses the same supported process mechanism as other runtime workers. macOS packages carry a checksum-pinned SQLite 3.51.3 engine; Linux and Windows validate their embedded SQLite version at initialization. The engine requirement follows the [SQLite WAL reset fix](https://www.sqlite.org/wal.html); durability follows the [synchronous pragma](https://www.sqlite.org/pragma.html#pragma_synchronous). PostgreSQL isolation and ownership follow its [transaction isolation](https://www.postgresql.org/docs/current/transaction-iso.html) and [explicit locking](https://www.postgresql.org/docs/current/explicit-locking.html) contracts. +SQLite executes in a subprocess with bounded IPC and explicit shutdown. Its POSIX process group is separate from the Agent owner: real Docker cancellation showed that inheriting the group kills storage before terminal evidence and accounting can commit. Explicit close and parent-disconnection cleanup retain ownership without blocking cancellation. Bun 1.3.14 worker-thread tests exposed a persistence-await deadlock, so database isolation uses the same supported process mechanism as other runtime workers. macOS executables and independent module archives carry a checksum-pinned SQLite 3.51.3 engine, with a packed-artifact smoke that masks machine-wide libraries; Linux and Windows validate their embedded SQLite version at initialization. The engine requirement follows the [SQLite WAL reset fix](https://www.sqlite.org/wal.html); durability follows the [synchronous pragma](https://www.sqlite.org/pragma.html#pragma_synchronous). PostgreSQL isolation and ownership follow its [transaction isolation](https://www.postgresql.org/docs/current/transaction-iso.html) and [explicit locking](https://www.postgresql.org/docs/current/explicit-locking.html) contracts. ## Alternatives considered diff --git a/docs/reference/packages.md b/docs/reference/packages.md index e16325b99..21f3e761e 100644 --- a/docs/reference/packages.md +++ b/docs/reference/packages.md @@ -14,7 +14,7 @@ The root `package.json` selects workspace packages. Each package manifest owns i Run `bun script/build-workspace.ts packages/harness` from the repository root to build its importable modules. `bun script/pack-workspace.ts packages/cli .artifacts/packages` builds and packs the CLI workspace dependency closure. Archives contain compiled module exports and normal dependency versions. Packing compiles the existing HTTP SDK without regenerating the complete product API; run the root generator explicitly after API changes. Core archives do not include optional Browser, Library, MCP or UI packages. -The Runtime Local archive declares its build target OS and CPU. Pass `--target=linux-x64`, `--target=linux-arm64-musl` or `--target=win32-x64` to select a target; Linux and Windows builds require matching helper assets through `SYNERGY_SANDBOX_ASSETS_DIR`. The build verifies the binary format and embeds its digest alongside the packaged helper. Missing required assets fail the build. Darwin uses the operating system sandbox. +The Harness archive declares its build target OS; Runtime Local also declares its CPU. macOS Harness archives carry the verified universal SQLite engine in `dist/libsqlite3.dylib`, so consumers do not need Homebrew. Pass `--target=linux-x64`, `--target=linux-arm64-musl` or `--target=win32-x64` to select a target; Linux and Windows builds require matching helper assets through `SYNERGY_SANDBOX_ASSETS_DIR`. The build verifies the binary format and embeds its digest alongside the packaged helper. Missing required assets fail the build. Darwin uses the operating system sandbox. `bun script/package-install-check.ts .artifacts/packages` installs the archives through a temporary local registry into a fresh directory outside the repository and runs the same CLI artifact contracts used for compiled binaries. It does not publish packages. After packing `packages/product-runtime` into the same archive directory, `bun script/runtime-composition-check.ts .artifacts/packages` verifies the CLI core, Browser, Library, Notes and full server compositions. Package installation cannot rely on workspace symlinks or test preload dependencies. diff --git a/packages/harness/AGENTS.md b/packages/harness/AGENTS.md index f043b459e..2b72eecb6 100644 --- a/packages/harness/AGENTS.md +++ b/packages/harness/AGENTS.md @@ -16,4 +16,4 @@ Run bun run typecheck and the affected tests, then the root package and dependen - Keep application-facing operations under the explicit session, scope, tools, context, lifecycle, config, persistence and rollout entries. Host adapters may use individually declared contract leaves; never export processor, resolver, scheduler or journal implementations to production. Published manifests omit `test/` exports, and production sources must not import them. Within this package use relative owner imports, not public entry points. - File browsing, indexing, Ripgrep, Hashline editing and conflict resolution belong to runtime-local; this package retains execution read evidence and locking. -Agent authority is owned by `Storage.Handle`: read [Agent storage](../../docs/architecture/agent-storage.md) before persistence changes. Use SQL business transactions for records, indexes and outbox entries; keep files and external effects outside retryable callbacks. The central bootstrap owns historical JSON import and activation. Runtime and maintenance entry points run registered owner recovery before admission. Storage engine changes run `bun test test/storage`; the PostgreSQL CI matrix requires a real database via `SYNERGY_TEST_POSTGRES_URL`. +Agent authority is owned by `Storage.Handle`: read [Agent storage](../../docs/architecture/agent-storage.md) before persistence changes. Use SQL business transactions for records, indexes and outbox entries; keep files and external effects outside retryable callbacks. The central bootstrap owns historical JSON import and activation. Runtime and maintenance entry points run registered owner recovery before admission. Storage engine changes run `bun test test/storage`; macOS packaging changes also run `bun test --config /dev/null test/script/release/workspace-sqlite.test.ts` from the repository root; the PostgreSQL CI matrix requires a real database via `SYNERGY_TEST_POSTGRES_URL`. diff --git a/packages/harness/src/session/index.ts b/packages/harness/src/session/index.ts index a31340b2d..06a430952 100644 --- a/packages/harness/src/session/index.ts +++ b/packages/harness/src/session/index.ts @@ -1555,19 +1555,23 @@ export namespace Session { asMessageID(part.messageID), asPartID(part.id), ) - if (delta !== undefined) { + const transactional = Storage.inTransaction() + if (delta !== undefined && !transactional) { partWriteBuffer().defer(part.id, path, part) } else { - await partWriteBuffer().writeNow(part.id, path, part, (key, value) => + const write = (key: string[], value: MessageV2.Part) => Storage.transaction(async (tx) => { await assertPartOwner(tx, key) await Storage.write(key, value) if (value.type === "text" || value.type === "tool" || value.type === "attachment") await SessionSearchIndex.markDirty(scopeID, asSessionID(value.sessionID)) SessionMessageCache.upsertPart(value.sessionID, value) - await Bus.publish(MessageV2.Event.PartUpdated, { part: value }) - }), - ) + await Bus.publish(MessageV2.Event.PartUpdated, { part: value, delta }) + }) + if (transactional) { + partWriteBuffer().assertDrained(part.id) + await write(path, part) + } else await partWriteBuffer().writeNow(part.id, path, part, write) } if (part.type === "tool") { // Tool parts are published as unsequenced streaming events. Keep a @@ -1584,7 +1588,7 @@ export namespace Session { durable: delta === undefined, }) } - if (delta !== undefined) await Bus.publish(MessageV2.Event.PartUpdated, { part, delta }) + if (delta !== undefined && !transactional) await Bus.publish(MessageV2.Event.PartUpdated, { part, delta }) return part } diff --git a/packages/harness/src/session/part-write-buffer.ts b/packages/harness/src/session/part-write-buffer.ts index 05f62e3b3..0f45829a4 100644 --- a/packages/harness/src/session/part-write-buffer.ts +++ b/packages/harness/src/session/part-write-buffer.ts @@ -83,6 +83,11 @@ export class PartWriteBuffer { return this.flushWhere(() => true) } + assertDrained(key: string): void { + if (this.latest.has(key) || this.running.has(key) || this.failures.has(key)) + throw new StorageBusyError("Drain streaming part writes before entering a business transaction") + } + async flushWhere(predicate: (value: T, path: P) => boolean): Promise { const keys = new Set() for (const [key, entry] of this.latest) if (predicate(entry.value, entry.path)) keys.add(key) diff --git a/packages/harness/src/storage/legacy-import.ts b/packages/harness/src/storage/legacy-import.ts index 89c7a8de5..88f3c5bc7 100644 --- a/packages/harness/src/storage/legacy-import.ts +++ b/packages/harness/src/storage/legacy-import.ts @@ -2,6 +2,7 @@ import { createHash, randomUUID } from "node:crypto" import fs from "node:fs/promises" import { createReadStream } from "node:fs" import path from "node:path" +import { z } from "zod" import { AtomicFile } from "./atomic-file" import { StorageIntegrityError } from "./errors" import { TransactionalStore } from "./transactional-store" @@ -91,6 +92,12 @@ export interface ImportResult { const stateKey = ["storage_import", "info"] +const legacySessionInfo = z.object({ + scope: z.object({ id: z.string() }), + title: z.string(), + time: z.object({ created: z.number(), updated: z.number() }), +}) + function entryKey(relative: string) { return ["storage_import_files", createHash("sha256").update(relative).digest("hex")] } @@ -314,6 +321,24 @@ export class LegacyJsonImporter { { private: true, durable: true }, ) + // Validate owners first so migrations never see children of a quarantined Session. + let sessionAfter: string[] | undefined + for (;;) { + const batch = await store.query({ kind: "storage_import_files", after: sessionAfter, limit: 128 }) + if (!batch.length) break + for (const record of batch) { + const entry = record.value + if ( + entry.disposition === "backed-up" && + entry.key?.length === 4 && + entry.key[0] === "sessions" && + entry.key[3] === "info" + ) + await this.import(record.key, entry) + } + sessionAfter = batch.at(-1)!.key + } + const result: ImportResult = { files: count, bytes, imported: 0, quarantined: 0, retained: 0 } let after: string[] | undefined progress?.({ stage: "import", current: 0, total: count, bytes: 0 }) @@ -385,9 +410,18 @@ export class LegacyJsonImporter { await store.write(recordKey, entry) return } + const [owner, recovery] = + entry.key[0] === "sessions" && entry.key.length > 4 + ? await store.readMany([ + [...entry.key.slice(0, 3), "info"], + ["storage_recovery", "sessions", entry.key[2], "info"], + ]) + : [] let value: unknown try { value = JSON.parse(await fs.readFile(path.join(backupRoot, "data", entry.relative), "utf8")) + if (owner === undefined && recovery !== undefined) + throw new StorageIntegrityError("Legacy Session owner metadata is quarantined") if ( entry.key[0] === "projects" && (!value || typeof value !== "object" || Array.isArray(value) || !("id" in value) || value.id !== entry.key[1]) @@ -406,11 +440,17 @@ export class LegacyJsonImporter { const expectedID = entry.key.at(-1) === "info" ? entry.key.at(-2) : entry.key.at(-1) if (!value || typeof value !== "object" || Array.isArray(value) || !("id" in value) || value.id !== expectedID) throw new StorageIntegrityError("Legacy record identity does not match its owner") + if (entry.key.length === 4 && entry.key[3] === "info") { + const parsed = legacySessionInfo.safeParse(value) + if (!parsed.success) throw new StorageIntegrityError("Legacy Session metadata is malformed") + if (parsed.data.scope.id !== entry.key[1]) + throw new StorageIntegrityError("Legacy Session Scope does not match its owner") + } } } catch (error) { if (!(error instanceof SyntaxError) && !(error instanceof StorageIntegrityError)) throw error entry.disposition = "quarantined" - entry.error = error instanceof SyntaxError ? "Invalid JSON" : "Invalid record identity" + entry.error = error instanceof SyntaxError ? "Invalid JSON" : error.message await store.transaction(async (tx) => { await tx.write(recordKey, entry) if (entry.key?.[0] === "sessions" && entry.key[2]) { diff --git a/packages/harness/src/storage/sqlite-engine.ts b/packages/harness/src/storage/sqlite-engine.ts index bae1525ca..e858bc76e 100644 --- a/packages/harness/src/storage/sqlite-engine.ts +++ b/packages/harness/src/storage/sqlite-engine.ts @@ -7,11 +7,12 @@ export function initializeSqliteEngine() { if (initialized) return if (process.platform === "darwin") { const packaged = path.resolve(path.dirname(process.execPath), "../libsqlite3.dylib") + const moduleEngine = path.resolve(import.meta.dirname, "../libsqlite3.dylib") const source = path.resolve(import.meta.dirname, "../../.artifacts/sqlite/libsqlite3.dylib") - const candidates = [packaged, source] + const candidates = [moduleEngine, packaged, source] // Source development can use a verified Homebrew engine; packaged builds // include their own engine and never depend on machine-wide libraries. - if (existsSync(path.resolve(import.meta.dirname, "../../package.json"))) + if (existsSync(path.resolve(import.meta.dirname, "../../script/build-sqlite.ts"))) candidates.push("/opt/homebrew/opt/sqlite/lib/libsqlite3.dylib", "/usr/local/opt/sqlite/lib/libsqlite3.dylib") const selected = candidates.find(existsSync) if (!selected) diff --git a/packages/harness/test/session/materialization-rollback.test.ts b/packages/harness/test/session/materialization-rollback.test.ts new file mode 100644 index 000000000..3349eb710 --- /dev/null +++ b/packages/harness/test/session/materialization-rollback.test.ts @@ -0,0 +1,81 @@ +import { expect, spyOn, test } from "bun:test" +import path from "node:path" +import { tmpdir } from "../support/fixture" +import { StorageBootstrap } from "../../src/storage/bootstrap" +import { Storage } from "../../src/storage/storage" +import { Session } from "../../src/session" +import { ScopeContext } from "../../src/scope/context" +import { Scope } from "../../src/scope" +import { Identifier } from "../../src/id/id" +import { SessionUserMessageMaterialization } from "../../src/session/user-message-materialization" +import { MessageV2 } from "../../src/session/message-v2" + +test.each([false, true])( + "a rolled-back materialization does not survive in the drain buffer (retry=%s)", + async (retry) => { + await using tmp = await tmpdir() + const root = path.join(tmp.path, ".synergy") + const prepared = await StorageBootstrap.prepare({ root }) + try { + await Storage.provide({ store: prepared.store, artifactDirectory: path.join(root, "data") }, () => + ScopeContext.provide({ + scope: Scope.home(), + fn: async () => { + const session = await Session.create({ title: "part failure" }) + const messageID = Identifier.ascending("message") + const info: MessageV2.User = { + id: messageID, + sessionID: session.id, + role: "user", + time: { created: Date.now() }, + agent: "synergy", + model: { providerID: "openai", modelID: "gpt-4.1" }, + isRoot: true, + visible: true, + includeInContext: true, + origin: { type: "user" }, + } + const part: MessageV2.TextPart = { + id: Identifier.ascending("part"), + messageID, + sessionID: session.id, + type: "text", + text: "test", + } + const original = Storage.write + { + using failure = spyOn(Storage, "write").mockImplementation(async (key, value) => { + if (key[5] === "parts") throw new Error("transient part write failure") + return original(key, value) + }) + await expect(SessionUserMessageMaterialization.write({ info, parts: [part] })).rejects.toThrow( + "transient", + ) + } + expect(await Storage.readMany([["sessions", "home", session.id, "messages", messageID, "info"]])).toEqual([ + undefined, + ]) + const drained = await Session.flushPartWrites(session.id).then( + () => "ok", + (error: unknown) => (error instanceof Error ? error.message : String(error)), + ) + if (retry) { + await SessionUserMessageMaterialization.write({ + info, + parts: [{ ...part, id: Identifier.ascending("part") }], + }) + await Session.flushPartWrites(session.id) + expect( + (await MessageV2.parts({ sessionID: session.id, messageID })).map((item) => + item.type === "text" ? item.text : item.type, + ), + ).toEqual(["test"]) + } else expect(drained).toBe("ok") + }, + }), + ) + } finally { + await prepared.store.close() + } + }, +) diff --git a/packages/harness/test/session/part-write-buffer.test.ts b/packages/harness/test/session/part-write-buffer.test.ts index 917c3ff33..06ee9d497 100644 --- a/packages/harness/test/session/part-write-buffer.test.ts +++ b/packages/harness/test/session/part-write-buffer.test.ts @@ -12,6 +12,29 @@ function recorder() { } describe("PartWriteBuffer", () => { + test("transaction admission requires buffered, running and failed writes to drain first", async () => { + let fail = true + let release!: () => void + const blocked = new Promise((resolve) => { + release = resolve + }) + const buffer = new PartWriteBuffer(async () => { + await blocked + if (fail) throw new Error("transient") + }, 10_000) + expect(() => buffer.assertDrained("part")).not.toThrow() + buffer.defer("part", "part", "stream") + expect(() => buffer.assertDrained("part")).toThrow("Drain") + const pending = buffer.flush("part") + expect(() => buffer.assertDrained("part")).toThrow("Drain") + release() + await expect(pending).rejects.toThrow("transient") + expect(() => buffer.assertDrained("part")).toThrow("Drain") + fail = false + await buffer.flushAll() + expect(() => buffer.assertDrained("part")).not.toThrow() + }) + test("terminal writes wait for an already executing streaming write", async () => { const writes: string[] = [] let release!: () => void diff --git a/packages/harness/test/storage/legacy-import.test.ts b/packages/harness/test/storage/legacy-import.test.ts index edb94aa5e..298437ec6 100644 --- a/packages/harness/test/storage/legacy-import.test.ts +++ b/packages/harness/test/storage/legacy-import.test.ts @@ -30,7 +30,13 @@ test("backs up and imports historical records without dropping unloaded fields o const fixtureData = await fixture() const { store, data, backup } = fixtureData try { - const session = { id: "session", projectID: "scope", title: "old", unknownOwner: { value: 2 } } + const session = { + id: "session", + scope: { id: "scope" }, + title: "old", + time: { created: 1, updated: 2 }, + unknownOwner: { value: 2 }, + } await json(data, ["sessions", "scope", "session", "info"], session) await json(data, ["meta", "migration", "log-workflows"], { historical: 123 }) await json(data, ["auth", "provider-auth"], { private: "must-not-become-a-record" }) @@ -58,14 +64,15 @@ test("quarantines malformed evidence with original bytes and blocks the affected try { const target = await json(data, ["sessions", "scope", "broken", "messages", "message", "info"], {}) await Bun.write(target, "{broken-json") - await json(data, ["sessions", "scope", "good", "info"], { id: "good" }) + const good = { id: "good", scope: { id: "scope" }, title: "retained", time: { created: 1, updated: 2 } } + await json(data, ["sessions", "scope", "good", "info"], good) const result = await new LegacyJsonImporter({ dataRoot: data, backupRoot: backup, store }).run() expect(result.quarantined).toBe(1) expect(await Bun.file(path.join(backup, "data", path.relative(data, target))).text()).toBe("{broken-json") expect(await store.read>(["storage_recovery", "sessions", "broken", "info"])).toMatchObject( { blocked: true }, ) - expect(await store.read>(["sessions", "scope", "good", "info"])).toEqual({ id: "good" }) + expect(await store.read>(["sessions", "scope", "good", "info"])).toEqual(good) } finally { await store.close() } diff --git a/packages/harness/test/storage/released-upgrade-quarantine.test.ts b/packages/harness/test/storage/released-upgrade-quarantine.test.ts new file mode 100644 index 000000000..4ff7490e5 --- /dev/null +++ b/packages/harness/test/storage/released-upgrade-quarantine.test.ts @@ -0,0 +1,84 @@ +import { expect, test } from "bun:test" +import fs from "node:fs/promises" +import path from "node:path" +import { tmpdir } from "../support/fixture" +import { StorageBootstrap } from "../../src/storage/bootstrap" +import { Storage } from "../../src/storage/storage" +import { StorageRecovery } from "../../src/storage/recovery" +import { migrations as sessionMigrations } from "../../src/session/migration" + +test.each([ + { invalid: { time: null }, migrated: true }, + { invalid: { time: { created: 1700000000000 } }, migrated: true }, + { invalid: { scope: null }, migrated: true }, + { invalid: { time: null }, migrated: false }, +])( + "invalid historical session fields quarantine only that session during startup: %j", + async ({ invalid, migrated }) => { + await using tmp = await tmpdir() + const root = path.join(tmp.path, ".synergy") + const fixture = (await Bun.file(new URL("./fixtures/v3.0.22.json", import.meta.url)).json()) as { + records: Array<{ key: string[]; value: Record }> + } + const broken = fixture.records[0] + const healthyID = "ses_00000000000000000000000002" + const healthyKey = ["sessions", "home", healthyID, "info"] + fixture.records.push({ key: healthyKey, value: { ...broken.value, id: healthyID } }) + Object.assign(broken.value, invalid) + if (migrated) + fixture.records.push({ + key: ["meta", "migration", "log-session"], + value: Object.fromEntries( + sessionMigrations + .filter((migration) => !migration.id.startsWith("20260914")) + .map((migration) => [migration.id, 1700000000000]), + ), + }) + for (const record of fixture.records) { + const file = path.join(root, "data", ...record.key) + ".json" + await fs.mkdir(path.dirname(file), { recursive: true }) + await Bun.write(file, JSON.stringify(record.value)) + } + const entry = new URL("../../src/storage/maintenance.ts", import.meta.url).pathname + const script = `import { StorageMaintenance } from ${JSON.stringify(entry)}; await using handle = await StorageMaintenance.open(); if (handle.manifest.phase !== "active") throw new Error("Upgrade did not activate");` + for (let attempt = 0; attempt < 2; attempt++) { + const child = Bun.spawn([process.execPath, "-e", script], { + env: { ...process.env, SYNERGY_HOME: tmp.path }, + stdout: "pipe", + stderr: "pipe", + }) + const [stderr, , code] = await Promise.all([ + new Response(child.stderr).text(), + new Response(child.stdout).text(), + child.exited, + ]) + expect(code, stderr).toBe(0) + } + const handle = await StorageBootstrap.inspect(root) + if (!handle) throw new Error("Upgraded dataset is absent") + try { + const blockedKey = ["storage_recovery", "sessions", broken.key[2], "info"] + expect(await handle.store.read(blockedKey)).toMatchObject({ blocked: true, reason: "historical_data_gap" }) + expect(await handle.store.readMany([broken.key, ["session_index", broken.key[2]]])).toEqual([ + undefined, + undefined, + ]) + expect(await handle.store.readMany([fixture.records[2].key, fixture.records[3].key])).toEqual([ + undefined, + undefined, + ]) + expect(await handle.store.read(["session_index", healthyID])).toMatchObject({ scopeID: "home" }) + expect(await handle.store.read(healthyKey)).toMatchObject({ futureOwner: { retained: true } }) + const state = await handle.store.read<{ backup: string }>(["storage_import", "info"]) + expect(await Bun.file(path.join(state.backup, "data", ...broken.key) + ".json").json()).toEqual(broken.value) + await Storage.provide(handle, async () => { + await StorageRecovery.load() + expect(() => StorageRecovery.assertRunnable(broken.key[2])).toThrow("quarantined") + expect(() => StorageRecovery.assertRunnable(healthyID)).not.toThrow() + }) + } finally { + await handle.store.close() + } + }, + 30000, +) diff --git a/script/pack-workspace.ts b/script/pack-workspace.ts index f6c0583a4..c07dc40a2 100644 --- a/script/pack-workspace.ts +++ b/script/pack-workspace.ts @@ -1,5 +1,6 @@ #!/usr/bin/env bun import { buildWatcher } from "../packages/runtime-local/script/build-watcher" +import { buildSqlite } from "../packages/harness/script/build-sqlite" import { cp, mkdir, mkdtemp, rm, chmod } from "node:fs/promises" import os from "node:os" import path from "node:path" @@ -68,6 +69,8 @@ export async function packWorkspace( const process = Bun.spawn(command, { cwd: sourceDirectory, stdout: "inherit", stderr: "inherit" }) if (await process.exited) throw new Error(`Package build failed: ${name}`) } + if (pkg.directory === "packages/harness" && target.os === "darwin") + await cp(await buildSqlite(), path.join(sourceDirectory, "dist/libsqlite3.dylib")) if (pkg.directory === "packages/runtime-local") { await stageWorkspaceSandbox(path.join(sourceDirectory, "dist"), target, options.assetsRoot) if (target.os === "linux") { @@ -104,8 +107,8 @@ export async function packWorkspace( engines: { bun: ">=1.3.14" }, } if (pkg.directory === "packages/product-runtime") (manifest.files as string[]).push("dist/schema") + if (["packages/harness", "packages/runtime-local"].includes(pkg.directory)) manifest.os = [target.os] if (pkg.directory === "packages/runtime-local") { - manifest.os = [target.os] manifest.cpu = [target.arch] } delete manifest.scripts diff --git a/test/script/release/workspace-sqlite.test.ts b/test/script/release/workspace-sqlite.test.ts new file mode 100644 index 000000000..f66f16a5f --- /dev/null +++ b/test/script/release/workspace-sqlite.test.ts @@ -0,0 +1,46 @@ +import { expect, test } from "bun:test" +import { mkdtemp, mkdir, realpath, rm } from "node:fs/promises" +import os from "node:os" +import path from "node:path" +import { pathToFileURL } from "node:url" +import { packWorkspace } from "../../../script/pack-workspace" + +test.skipIf(process.platform !== "darwin")( + "packed harness opens SQLite without host libraries", + async () => { + const directory = await realpath(await mkdtemp(path.join(os.tmpdir(), "synergy-sqlite-package-"))) + try { + const packed = await packWorkspace("packages/harness", path.join(directory, "archives")) + const unpacked = path.join(directory, "unpacked") + await mkdir(unpacked) + const extraction = Bun.spawn(["tar", "-xzf", packed.entry, "-C", unpacked], { stderr: "pipe" }) + expect(await extraction.exited, await new Response(extraction.stderr).text()).toBe(0) + expect(await Bun.file(path.join(unpacked, "package/dist/libsqlite3.dylib")).exists()).toBe(true) + expect(await Bun.file(path.join(unpacked, "package/package.json")).json()).toMatchObject({ os: ["darwin"] }) + const engine = pathToFileURL(path.join(unpacked, "package/dist/storage/sqlite-engine.js")).href + const script = ` + import { mock } from "bun:test"; + import fs from "node:fs"; + import { Database } from "bun:sqlite"; + const exists = fs.existsSync; + mock.module("node:fs", () => ({ ...fs, existsSync: (file) => + String(file).endsWith("libsqlite3.dylib") && !String(file).startsWith(${JSON.stringify(unpacked)}) + ? false : exists(file) + })); + const { initializeSqliteEngine } = await import(${JSON.stringify(engine)}); + initializeSqliteEngine(); + const db = new Database(":memory:"); + db.run("CREATE TABLE example (value TEXT)"); + db.run("INSERT INTO example VALUES ('packaged')"); + if (db.query("SELECT value FROM example").get().value !== "packaged") throw new Error("Write failed"); + db.close(); + ` + const child = Bun.spawn([process.execPath, "-e", script], { stdout: "pipe", stderr: "pipe" }) + const [code, stderr] = await Promise.all([child.exited, new Response(child.stderr).text()]) + expect(code, stderr).toBe(0) + } finally { + await rm(directory, { recursive: true, force: true }) + } + }, + 120_000, +)