From da6e9d9790ad2574f1b2ea67152f3a7d2eb71380 Mon Sep 17 00:00:00 2001 From: Johns <19662585+Rowdy@users.noreply.github.com> Date: Mon, 29 Jun 2026 20:04:23 +0200 Subject: [PATCH 1/4] feat(devonthink): add devonthink --- library/productivity/devonthink/.gitignore | 9 + library/productivity/devonthink/.golangci.yml | 12 + .../productivity/devonthink/.goreleaser.yaml | 51 + .../devonthink-local-only-phase5-skip.md | 6 + .../proofs/phase5-acceptance.json | 18 + .../proofs/phase5-skip.json | 15 + .../research/research.json | 206 ++ .../devonthink-smart-group-search-scope.json | 18 + .../devonthink/.printing-press.json | 57 + library/productivity/devonthink/AGENTS.md | 46 + library/productivity/devonthink/CHANGELOG.md | 4 + library/productivity/devonthink/LICENSE | 191 ++ library/productivity/devonthink/Makefile | 24 + library/productivity/devonthink/NOTICE | 11 + library/productivity/devonthink/README.md | 482 +++ library/productivity/devonthink/SKILL.md | 408 +++ .../devonthink/cmd/devonthink-pp-cli/main.go | 16 + .../devonthink/cmd/devonthink-pp-mcp/main.go | 30 + ...-001-feat-smart-group-search-scope-plan.md | 184 ++ library/productivity/devonthink/go.mod | 34 + library/productivity/devonthink/go.sum | 84 + .../devonthink/internal/cache/cache.go | 59 + .../devonthink/internal/cli/agent_context.go | 194 ++ .../devonthink/internal/cli/ai.go | 22 + .../devonthink/internal/cli/ai_ask.go | 205 ++ .../devonthink/internal/cli/ai_summarize.go | 200 ++ .../devonthink/internal/cli/analytics.go | 197 ++ .../devonthink/internal/cli/api_discovery.go | 109 + .../devonthink/internal/cli/batch.go | 22 + .../devonthink/internal/cli/batch_apply.go | 195 ++ .../devonthink/internal/cli/batch_plan.go | 216 ++ .../internal/cli/channel_workflow.go | 165 + .../devonthink/internal/cli/context_pack.go | 45 + .../internal/cli/context_pack_test.go | 20 + .../devonthink/internal/cli/data_source.go | 612 ++++ .../devonthink/internal/cli/deliver.go | 114 + .../devonthink/internal/cli/doctor.go | 505 +++ .../devonthink/internal/cli/export.go | 122 + .../devonthink/internal/cli/feedback.go | 228 ++ .../devonthink/internal/cli/graph.go | 22 + .../devonthink/internal/cli/graph_audit.go | 91 + .../devonthink/internal/cli/graph_links.go | 94 + .../devonthink/internal/cli/helpers.go | 1835 +++++++++++ .../devonthink/internal/cli/import.go | 109 + .../devonthink/internal/cli/ingest.go | 22 + .../devonthink/internal/cli/ingest_file.go | 200 ++ .../devonthink/internal/cli/ingest_url.go | 200 ++ .../internal/cli/inventory_export.go | 50 + .../internal/cli/inventory_export_test.go | 42 + .../devonthink/internal/cli/ledger.go | 22 + .../devonthink/internal/cli/ledger_list.go | 91 + .../devonthink/internal/cli/ledger_show.go | 83 + .../devonthink/internal/cli/mcp.go | 23 + .../devonthink/internal/cli/mcp_call.go | 195 ++ .../devonthink/internal/cli/mcp_schema.go | 79 + .../devonthink/internal/cli/mcp_tools.go | 79 + .../devonthink/internal/cli/media.go | 22 + .../devonthink/internal/cli/media_ocr.go | 195 ++ .../internal/cli/media_transcribe.go | 200 ++ .../devonthink/internal/cli/mirror.go | 22 + .../devonthink/internal/cli/mirror_search.go | 90 + .../devonthink/internal/cli/mirror_sync.go | 89 + .../devonthink/internal/cli/privacy_audit.go | 40 + .../internal/cli/privacy_audit_test.go | 20 + .../devonthink/internal/cli/profile.go | 351 +++ .../internal/cli/promoted_context.go | 105 + .../internal/cli/promoted_databases.go | 84 + .../internal/cli/promoted_groups.go | 99 + .../internal/cli/promoted_inventory.go | 105 + .../internal/cli/promoted_privacy.go | 102 + .../internal/cli/promoted_runtime.go | 84 + .../internal/cli/promoted_sheets.go | 98 + .../devonthink/internal/cli/promoted_tags.go | 89 + .../devonthink/internal/cli/records.go | 30 + .../internal/cli/records_content.go | 88 + .../devonthink/internal/cli/records_create.go | 220 ++ .../devonthink/internal/cli/records_get.go | 83 + .../internal/cli/records_highlights.go | 83 + .../devonthink/internal/cli/records_lookup.go | 109 + .../devonthink/internal/cli/records_move.go | 205 ++ .../internal/cli/records_related.go | 90 + .../devonthink/internal/cli/records_search.go | 120 + .../devonthink/internal/cli/records_update.go | 220 ++ .../internal/cli/records_versions.go | 83 + .../devonthink/internal/cli/root.go | 327 ++ .../devonthink/internal/cli/root_test.go | 244 ++ .../devonthink/internal/cli/search.go | 304 ++ .../devonthink/internal/cli/selection.go | 22 + .../devonthink/internal/cli/selection_get.go | 79 + .../internal/cli/selection_snapshot.go | 89 + .../devonthink/internal/cli/sync.go | 1655 ++++++++++ .../devonthink/internal/cli/sync_hint.go | 123 + .../devonthink/internal/cli/sync_hint_test.go | 175 ++ .../internal/cli/sync_numeric_id_test.go | 47 + .../devonthink/internal/cli/tail.go | 167 + .../devonthink/internal/cli/version.go | 25 + .../devonthink/internal/cli/which.go | 222 ++ .../devonthink/internal/cli/which_test.go | 98 + .../devonthink/internal/client/client.go | 845 +++++ .../devonthink/internal/client/client_test.go | 72 + .../client_verify_short_circuit_test.go | 169 + .../internal/client/local_devonthink.go | 1076 +++++++ .../internal/client/smart_group_scope.go | 155 + .../internal/client/smart_group_scope_test.go | 59 + .../internal/cliutil/cliutil_test.go | 839 +++++ .../devonthink/internal/cliutil/duration.go | 41 + .../internal/cliutil/duration_test.go | 56 + .../internal/cliutil/extractnumber.go | 67 + .../internal/cliutil/extractnumber_test.go | 117 + .../devonthink/internal/cliutil/fanout.go | 202 ++ .../devonthink/internal/cliutil/jwtshape.go | 96 + .../internal/cliutil/jwtshape_test.go | 128 + .../devonthink/internal/cliutil/odata_date.go | 37 + .../internal/cliutil/odata_date_test.go | 62 + .../devonthink/internal/cliutil/probe.go | 104 + .../devonthink/internal/cliutil/ratelimit.go | 207 ++ .../devonthink/internal/cliutil/text.go | 50 + .../devonthink/internal/cliutil/verifyenv.go | 93 + .../devonthink/internal/config/config.go | 165 + .../internal/mcp/cobratree/classify.go | 115 + .../internal/mcp/cobratree/cli_path.go | 35 + .../internal/mcp/cobratree/names.go | 25 + .../internal/mcp/cobratree/shellout.go | 161 + .../internal/mcp/cobratree/shellout_test.go | 228 ++ .../internal/mcp/cobratree/typemap.go | 76 + .../internal/mcp/cobratree/walker.go | 66 + .../devonthink/internal/mcp/tools.go | 1230 ++++++++ .../devonthink/internal/mcp/tools_test.go | 320 ++ .../devonthink/internal/store/extras.go | 28 + .../internal/store/schema_version_test.go | 1760 +++++++++++ .../devonthink/internal/store/store.go | 2786 +++++++++++++++++ .../internal/store/upsert_batch_test.go | 1042 ++++++ .../devonthink/internal/types/types.go | 128 + .../devonthink/tools-manifest.json | 918 ++++++ 134 files changed, 28634 insertions(+) create mode 100644 library/productivity/devonthink/.gitignore create mode 100644 library/productivity/devonthink/.golangci.yml create mode 100644 library/productivity/devonthink/.goreleaser.yaml create mode 100644 library/productivity/devonthink/.manuscripts/20260629T175310Z-b4705e85/proofs/devonthink-local-only-phase5-skip.md create mode 100644 library/productivity/devonthink/.manuscripts/20260629T175310Z-b4705e85/proofs/phase5-acceptance.json create mode 100644 library/productivity/devonthink/.manuscripts/20260629T175310Z-b4705e85/proofs/phase5-skip.json create mode 100644 library/productivity/devonthink/.manuscripts/20260629T175310Z-b4705e85/research/research.json create mode 100644 library/productivity/devonthink/.printing-press-patches/devonthink-smart-group-search-scope.json create mode 100644 library/productivity/devonthink/.printing-press.json create mode 100644 library/productivity/devonthink/AGENTS.md create mode 100644 library/productivity/devonthink/CHANGELOG.md create mode 100644 library/productivity/devonthink/LICENSE create mode 100644 library/productivity/devonthink/Makefile create mode 100644 library/productivity/devonthink/NOTICE create mode 100644 library/productivity/devonthink/README.md create mode 100644 library/productivity/devonthink/SKILL.md create mode 100644 library/productivity/devonthink/cmd/devonthink-pp-cli/main.go create mode 100644 library/productivity/devonthink/cmd/devonthink-pp-mcp/main.go create mode 100644 library/productivity/devonthink/docs/plans/2026-06-22-001-feat-smart-group-search-scope-plan.md create mode 100644 library/productivity/devonthink/go.mod create mode 100644 library/productivity/devonthink/go.sum create mode 100644 library/productivity/devonthink/internal/cache/cache.go create mode 100644 library/productivity/devonthink/internal/cli/agent_context.go create mode 100644 library/productivity/devonthink/internal/cli/ai.go create mode 100644 library/productivity/devonthink/internal/cli/ai_ask.go create mode 100644 library/productivity/devonthink/internal/cli/ai_summarize.go create mode 100644 library/productivity/devonthink/internal/cli/analytics.go create mode 100644 library/productivity/devonthink/internal/cli/api_discovery.go create mode 100644 library/productivity/devonthink/internal/cli/batch.go create mode 100644 library/productivity/devonthink/internal/cli/batch_apply.go create mode 100644 library/productivity/devonthink/internal/cli/batch_plan.go create mode 100644 library/productivity/devonthink/internal/cli/channel_workflow.go create mode 100644 library/productivity/devonthink/internal/cli/context_pack.go create mode 100644 library/productivity/devonthink/internal/cli/context_pack_test.go create mode 100644 library/productivity/devonthink/internal/cli/data_source.go create mode 100644 library/productivity/devonthink/internal/cli/deliver.go create mode 100644 library/productivity/devonthink/internal/cli/doctor.go create mode 100644 library/productivity/devonthink/internal/cli/export.go create mode 100644 library/productivity/devonthink/internal/cli/feedback.go create mode 100644 library/productivity/devonthink/internal/cli/graph.go create mode 100644 library/productivity/devonthink/internal/cli/graph_audit.go create mode 100644 library/productivity/devonthink/internal/cli/graph_links.go create mode 100644 library/productivity/devonthink/internal/cli/helpers.go create mode 100644 library/productivity/devonthink/internal/cli/import.go create mode 100644 library/productivity/devonthink/internal/cli/ingest.go create mode 100644 library/productivity/devonthink/internal/cli/ingest_file.go create mode 100644 library/productivity/devonthink/internal/cli/ingest_url.go create mode 100644 library/productivity/devonthink/internal/cli/inventory_export.go create mode 100644 library/productivity/devonthink/internal/cli/inventory_export_test.go create mode 100644 library/productivity/devonthink/internal/cli/ledger.go create mode 100644 library/productivity/devonthink/internal/cli/ledger_list.go create mode 100644 library/productivity/devonthink/internal/cli/ledger_show.go create mode 100644 library/productivity/devonthink/internal/cli/mcp.go create mode 100644 library/productivity/devonthink/internal/cli/mcp_call.go create mode 100644 library/productivity/devonthink/internal/cli/mcp_schema.go create mode 100644 library/productivity/devonthink/internal/cli/mcp_tools.go create mode 100644 library/productivity/devonthink/internal/cli/media.go create mode 100644 library/productivity/devonthink/internal/cli/media_ocr.go create mode 100644 library/productivity/devonthink/internal/cli/media_transcribe.go create mode 100644 library/productivity/devonthink/internal/cli/mirror.go create mode 100644 library/productivity/devonthink/internal/cli/mirror_search.go create mode 100644 library/productivity/devonthink/internal/cli/mirror_sync.go create mode 100644 library/productivity/devonthink/internal/cli/privacy_audit.go create mode 100644 library/productivity/devonthink/internal/cli/privacy_audit_test.go create mode 100644 library/productivity/devonthink/internal/cli/profile.go create mode 100644 library/productivity/devonthink/internal/cli/promoted_context.go create mode 100644 library/productivity/devonthink/internal/cli/promoted_databases.go create mode 100644 library/productivity/devonthink/internal/cli/promoted_groups.go create mode 100644 library/productivity/devonthink/internal/cli/promoted_inventory.go create mode 100644 library/productivity/devonthink/internal/cli/promoted_privacy.go create mode 100644 library/productivity/devonthink/internal/cli/promoted_runtime.go create mode 100644 library/productivity/devonthink/internal/cli/promoted_sheets.go create mode 100644 library/productivity/devonthink/internal/cli/promoted_tags.go create mode 100644 library/productivity/devonthink/internal/cli/records.go create mode 100644 library/productivity/devonthink/internal/cli/records_content.go create mode 100644 library/productivity/devonthink/internal/cli/records_create.go create mode 100644 library/productivity/devonthink/internal/cli/records_get.go create mode 100644 library/productivity/devonthink/internal/cli/records_highlights.go create mode 100644 library/productivity/devonthink/internal/cli/records_lookup.go create mode 100644 library/productivity/devonthink/internal/cli/records_move.go create mode 100644 library/productivity/devonthink/internal/cli/records_related.go create mode 100644 library/productivity/devonthink/internal/cli/records_search.go create mode 100644 library/productivity/devonthink/internal/cli/records_update.go create mode 100644 library/productivity/devonthink/internal/cli/records_versions.go create mode 100644 library/productivity/devonthink/internal/cli/root.go create mode 100644 library/productivity/devonthink/internal/cli/root_test.go create mode 100644 library/productivity/devonthink/internal/cli/search.go create mode 100644 library/productivity/devonthink/internal/cli/selection.go create mode 100644 library/productivity/devonthink/internal/cli/selection_get.go create mode 100644 library/productivity/devonthink/internal/cli/selection_snapshot.go create mode 100644 library/productivity/devonthink/internal/cli/sync.go create mode 100644 library/productivity/devonthink/internal/cli/sync_hint.go create mode 100644 library/productivity/devonthink/internal/cli/sync_hint_test.go create mode 100644 library/productivity/devonthink/internal/cli/sync_numeric_id_test.go create mode 100644 library/productivity/devonthink/internal/cli/tail.go create mode 100644 library/productivity/devonthink/internal/cli/version.go create mode 100644 library/productivity/devonthink/internal/cli/which.go create mode 100644 library/productivity/devonthink/internal/cli/which_test.go create mode 100644 library/productivity/devonthink/internal/client/client.go create mode 100644 library/productivity/devonthink/internal/client/client_test.go create mode 100644 library/productivity/devonthink/internal/client/client_verify_short_circuit_test.go create mode 100644 library/productivity/devonthink/internal/client/local_devonthink.go create mode 100644 library/productivity/devonthink/internal/client/smart_group_scope.go create mode 100644 library/productivity/devonthink/internal/client/smart_group_scope_test.go create mode 100644 library/productivity/devonthink/internal/cliutil/cliutil_test.go create mode 100644 library/productivity/devonthink/internal/cliutil/duration.go create mode 100644 library/productivity/devonthink/internal/cliutil/duration_test.go create mode 100644 library/productivity/devonthink/internal/cliutil/extractnumber.go create mode 100644 library/productivity/devonthink/internal/cliutil/extractnumber_test.go create mode 100644 library/productivity/devonthink/internal/cliutil/fanout.go create mode 100644 library/productivity/devonthink/internal/cliutil/jwtshape.go create mode 100644 library/productivity/devonthink/internal/cliutil/jwtshape_test.go create mode 100644 library/productivity/devonthink/internal/cliutil/odata_date.go create mode 100644 library/productivity/devonthink/internal/cliutil/odata_date_test.go create mode 100644 library/productivity/devonthink/internal/cliutil/probe.go create mode 100644 library/productivity/devonthink/internal/cliutil/ratelimit.go create mode 100644 library/productivity/devonthink/internal/cliutil/text.go create mode 100644 library/productivity/devonthink/internal/cliutil/verifyenv.go create mode 100644 library/productivity/devonthink/internal/config/config.go create mode 100644 library/productivity/devonthink/internal/mcp/cobratree/classify.go create mode 100644 library/productivity/devonthink/internal/mcp/cobratree/cli_path.go create mode 100644 library/productivity/devonthink/internal/mcp/cobratree/names.go create mode 100644 library/productivity/devonthink/internal/mcp/cobratree/shellout.go create mode 100644 library/productivity/devonthink/internal/mcp/cobratree/shellout_test.go create mode 100644 library/productivity/devonthink/internal/mcp/cobratree/typemap.go create mode 100644 library/productivity/devonthink/internal/mcp/cobratree/walker.go create mode 100644 library/productivity/devonthink/internal/mcp/tools.go create mode 100644 library/productivity/devonthink/internal/mcp/tools_test.go create mode 100644 library/productivity/devonthink/internal/store/extras.go create mode 100644 library/productivity/devonthink/internal/store/schema_version_test.go create mode 100644 library/productivity/devonthink/internal/store/store.go create mode 100644 library/productivity/devonthink/internal/store/upsert_batch_test.go create mode 100644 library/productivity/devonthink/internal/types/types.go create mode 100644 library/productivity/devonthink/tools-manifest.json diff --git a/library/productivity/devonthink/.gitignore b/library/productivity/devonthink/.gitignore new file mode 100644 index 0000000000..9873aea591 --- /dev/null +++ b/library/productivity/devonthink/.gitignore @@ -0,0 +1,9 @@ +/DEVONthink-cli +/devonthink-pp-cli +/devonthink-pp-mcp +/devonthink-inventory.json +/dogfood-results.json +/workflow-verify-report.json +/dist/ +/.cache/ +.DS_Store diff --git a/library/productivity/devonthink/.golangci.yml b/library/productivity/devonthink/.golangci.yml new file mode 100644 index 0000000000..5397226df9 --- /dev/null +++ b/library/productivity/devonthink/.golangci.yml @@ -0,0 +1,12 @@ +linters: + enable: + - errorlint + - govet + - ineffassign + - staticcheck + - unused + +formatters: + enable: + - gofmt + - goimports diff --git a/library/productivity/devonthink/.goreleaser.yaml b/library/productivity/devonthink/.goreleaser.yaml new file mode 100644 index 0000000000..a8b46538a5 --- /dev/null +++ b/library/productivity/devonthink/.goreleaser.yaml @@ -0,0 +1,51 @@ +version: 2 +project_name: devonthink-pp-cli +changelog: + disable: true +builds: + - id: devonthink-pp-cli + main: ./cmd/devonthink-pp-cli + binary: devonthink-pp-cli + env: + - CGO_ENABLED=0 + ldflags: + - -s -w -X github.com/mvanhorn/printing-press-library/library/productivity/devonthink/internal/cli.version={{ .Version }} + targets: + - darwin_amd64 + - darwin_arm64 + - linux_amd64 + - linux_arm64 + - windows_amd64 + - windows_arm64 + - id: devonthink-pp-mcp + main: ./cmd/devonthink-pp-mcp + binary: devonthink-pp-mcp + env: + - CGO_ENABLED=0 + ldflags: + - -s -w -X main.version={{ .Version }} + targets: + - darwin_amd64 + - darwin_arm64 + - linux_amd64 + - linux_arm64 + - windows_amd64 + - windows_arm64 +archives: + - formats: [tar.gz] + name_template: "{{ .ProjectName }}_{{ .Version }}_{{ .Os }}_{{ .Arch }}" + format_overrides: + - goos: windows + formats: [zip] +checksum: + name_template: checksums.txt +brews: + - name: devonthink-pp-cli + repository: + owner: rowdy + name: homebrew-tap + homepage: "https://github.com/mvanhorn/printing-press-library" + description: "Local-first DEVONthink automation with safer shell workflows than raw AppleScript or MCP alone." + install: | + bin.install "devonthink-pp-cli" + bin.install "devonthink-pp-mcp" diff --git a/library/productivity/devonthink/.manuscripts/20260629T175310Z-b4705e85/proofs/devonthink-local-only-phase5-skip.md b/library/productivity/devonthink/.manuscripts/20260629T175310Z-b4705e85/proofs/devonthink-local-only-phase5-skip.md new file mode 100644 index 0000000000..4d71ce1890 --- /dev/null +++ b/library/productivity/devonthink/.manuscripts/20260629T175310Z-b4705e85/proofs/devonthink-local-only-phase5-skip.md @@ -0,0 +1,6 @@ +# DEVONthink Phase 5 Skip + +- Decision: skip live public dogfood. +- Reason: DEVONthink runs only on this local Mac or the user's own LAN, and live results would expose personal database names or record metadata in public publish artifacts. +- Contract: the CLI remains local-first and wraps the official local DEVONthink MCP/automation surface; Smart Groups are search scopes only, not workflow policy. +- Verification still run in this session: `go test ./...`, `go build ./cmd/devonthink-pp-cli`, `devonthink-pp-cli records search --help`, and `cli-printing-press publish validate`. diff --git a/library/productivity/devonthink/.manuscripts/20260629T175310Z-b4705e85/proofs/phase5-acceptance.json b/library/productivity/devonthink/.manuscripts/20260629T175310Z-b4705e85/proofs/phase5-acceptance.json new file mode 100644 index 0000000000..64a2aeddf8 --- /dev/null +++ b/library/productivity/devonthink/.manuscripts/20260629T175310Z-b4705e85/proofs/phase5-acceptance.json @@ -0,0 +1,18 @@ +{ + "schema_version": 1, + "api_name": "devonthink", + "cli_name": "devonthink-pp-cli", + "run_id": "20260629T175310Z-b4705e85", + "status": "skip", + "level": "none", + "skip_reason": "lan-unreachable-from-generation-host", + "auth_context": { + "type": "none", + "api_key_available": false, + "browser_session_available": false, + "local_network_only": true + }, + "artifact_type": "phase5-acceptance", + "acceptance_kind": "skip", + "note": "DEVONthink live behavior is local-machine only; public library CI cannot access the user app or local MCP server." +} diff --git a/library/productivity/devonthink/.manuscripts/20260629T175310Z-b4705e85/proofs/phase5-skip.json b/library/productivity/devonthink/.manuscripts/20260629T175310Z-b4705e85/proofs/phase5-skip.json new file mode 100644 index 0000000000..3410b2f290 --- /dev/null +++ b/library/productivity/devonthink/.manuscripts/20260629T175310Z-b4705e85/proofs/phase5-skip.json @@ -0,0 +1,15 @@ +{ + "schema_version": 1, + "api_name": "devonthink", + "cli_name": "devonthink-pp-cli", + "run_id": "20260629T175310Z-b4705e85", + "status": "skip", + "level": "none", + "skip_reason": "lan-unreachable-from-generation-host", + "auth_context": { + "type": "none", + "api_key_available": false, + "browser_session_available": false, + "local_network_only": true + } +} diff --git a/library/productivity/devonthink/.manuscripts/20260629T175310Z-b4705e85/research/research.json b/library/productivity/devonthink/.manuscripts/20260629T175310Z-b4705e85/research/research.json new file mode 100644 index 0000000000..e10e010771 --- /dev/null +++ b/library/productivity/devonthink/.manuscripts/20260629T175310Z-b4705e85/research/research.json @@ -0,0 +1,206 @@ +{ + "api_name": "devonthink", + "novelty_score": 8, + "alternatives": [ + { + "name": "DEVONthink official MCP server", + "url": "https://www.devontechnologies.com/blog/20260526-devonthink-mcp-server", + "language": "Swift", + "stars": 0, + "command_count": 59 + }, + { + "name": "Rowdy/DEVONthink-ai-maintenance", + "url": "https://github.com/Rowdy/DEVONthink-ai-maintenance", + "language": "Codex plugin", + "stars": 0, + "command_count": 0 + } + ], + "gaps": [ + "Official MCP is a local agent tool surface, not a shell-native CLI with stable --agent, --select, profiles, delivery sinks, and Unix pipelines.", + "Maintenance workflows need a stable read-only inventory/search contract without making workflow policy part of the core CLI.", + "DEVONthink is local-only and personal-data-bearing, so public live dogfood cannot safely publish real record names or database contents." + ], + "patterns": [ + "Wrap official local MCP tools instead of directly manipulating DEVONthink database packages.", + "Keep DEVONthink access local to this Mac or the user's own LAN.", + "Expose read-only scopes and inventory for downstream maintenance plugins while keeping action workflow policy outside the CLI." + ], + "recommendation": "proceed", + "researched_at": "2026-06-29T17:53:10Z", + "novel_features": [ + { + "name": "Smart Group scoped search", + "command": "records search", + "description": "Scope a normal DEVONthink query to a Smart Group by UUID, exact name, or DEVONthink path while preserving normal search output.", + "rationale": "Combines human-friendly Smart Group names with agent-safe JSON envelopes and explicit scope metadata.", + "example": "devonthink-pp-cli records search \"tags:waiting/rueckerstattung\" --smart-group \"Offene Rückerstattungen\" --agent --select uuid,name,item_link,tags,databaseName", + "why_it_matters": "Use this when a downstream tool needs a stable dynamic search scope without treating Smart Groups as workflow policy.", + "group": "Agent-native plumbing" + }, + { + "name": "Context pack", + "command": "context pack", + "description": "Build a compact evidence packet from records, selections, highlights, links, and related items.", + "rationale": "Requires local DEVONthink structure plus token-budgeted output shaping for agent handoff.", + "example": "devonthink-pp-cli context pack --query \"project alpha\" --token-budget 6000 --agent", + "why_it_matters": "Use this when an agent needs enough DEVONthink context to reason without dumping whole documents.", + "group": "Local state that compounds" + }, + { + "name": "Privacy audit", + "command": "privacy audit", + "description": "Preview database scope, content-size budget, and cloud/MCP exposure before a handoff.", + "rationale": "Applies local-first DEVONthink safety checks before content leaves the machine.", + "example": "devonthink-pp-cli privacy audit --query \"kind:pdf\" --agent", + "why_it_matters": "Use this before exporting or sharing DEVONthink-derived context with another tool.", + "group": "Local safety" + }, + { + "name": "Batch planning ledger", + "command": "batch plan", + "description": "Stage multi-record edits as validated dry-run plans before applying them.", + "rationale": "Separates planning, proof, and apply state for safer local automation.", + "example": "devonthink-pp-cli batch plan --dry-run --agent", + "why_it_matters": "Use this when a script needs reviewable intent before any DEVONthink mutation.", + "group": "Local safety" + }, + { + "name": "Maintenance inventory export", + "command": "inventory export", + "description": "Export DEVONthink databases, groups, tags, and document metadata for maintenance plugins.", + "rationale": "Provides a stable downstream contract without embedding maintenance policy in the CLI.", + "example": "devonthink-pp-cli inventory export --format maintenance --query \"kind:document\" --limit 500 --agent --select databases,documents", + "why_it_matters": "Use this when structure-audit or inbox-triage tooling needs a stable local inventory contract.", + "group": "Agent-native plumbing" + }, + { + "name": "Knowledge graph audit", + "command": "graph audit", + "description": "Detect orphans, broken links, unresolved wiki links, weak hubs, and tag-only clusters.", + "rationale": "Requires merging DEVONthink item links, wiki links, mentions, tags, parents, and local mirror edges.", + "example": "devonthink-pp-cli graph audit --limit 50 --agent", + "why_it_matters": "Use this when DEVONthink should behave like a maintained knowledge graph instead of a folder pile.", + "group": "Local state that compounds" + }, + { + "name": "MCP passthrough", + "command": "mcp call", + "description": "Call DEVONthink's official local MCP tools from scripts when the local MCP server is enabled.", + "rationale": "Keeps new official MCP tools reachable before the CLI has promoted commands for them.", + "example": "devonthink-pp-cli mcp call search_records --args '{\"query\":\"kind:pdf\",\"limit\":5}' --agent", + "why_it_matters": "Use this when the official MCP exposes a new read tool before the CLI adds a first-class command.", + "group": "Agent-native plumbing" + } + ], + "novel_features_built": [ + { + "name": "Smart Group scoped search", + "command": "records search", + "description": "Scope a normal DEVONthink query to a Smart Group by UUID, exact name, or DEVONthink path while preserving normal search output.", + "rationale": "Combines human-friendly Smart Group names with agent-safe JSON envelopes and explicit scope metadata.", + "example": "devonthink-pp-cli records search \"tags:waiting/rueckerstattung\" --smart-group \"Offene Rückerstattungen\" --agent --select uuid,name,item_link,tags,databaseName", + "why_it_matters": "Use this when a downstream tool needs a stable dynamic search scope without treating Smart Groups as workflow policy.", + "group": "Agent-native plumbing" + }, + { + "name": "Context pack", + "command": "context pack", + "description": "Build a compact evidence packet from records, selections, highlights, links, and related items.", + "rationale": "Requires local DEVONthink structure plus token-budgeted output shaping for agent handoff.", + "example": "devonthink-pp-cli context pack --query \"project alpha\" --token-budget 6000 --agent", + "why_it_matters": "Use this when an agent needs enough DEVONthink context to reason without dumping whole documents.", + "group": "Local state that compounds" + }, + { + "name": "Privacy audit", + "command": "privacy audit", + "description": "Preview database scope, content-size budget, and cloud/MCP exposure before a handoff.", + "rationale": "Applies local-first DEVONthink safety checks before content leaves the machine.", + "example": "devonthink-pp-cli privacy audit --query \"kind:pdf\" --agent", + "why_it_matters": "Use this before exporting or sharing DEVONthink-derived context with another tool.", + "group": "Local safety" + }, + { + "name": "Batch planning ledger", + "command": "batch plan", + "description": "Stage multi-record edits as validated dry-run plans before applying them.", + "rationale": "Separates planning, proof, and apply state for safer local automation.", + "example": "devonthink-pp-cli batch plan --dry-run --agent", + "why_it_matters": "Use this when a script needs reviewable intent before any DEVONthink mutation.", + "group": "Local safety" + }, + { + "name": "Maintenance inventory export", + "command": "inventory export", + "description": "Export DEVONthink databases, groups, tags, and document metadata for maintenance plugins.", + "rationale": "Provides a stable downstream contract without embedding maintenance policy in the CLI.", + "example": "devonthink-pp-cli inventory export --format maintenance --query \"kind:document\" --limit 500 --agent --select databases,documents", + "why_it_matters": "Use this when structure-audit or inbox-triage tooling needs a stable local inventory contract.", + "group": "Agent-native plumbing" + }, + { + "name": "Knowledge graph audit", + "command": "graph audit", + "description": "Detect orphans, broken links, unresolved wiki links, weak hubs, and tag-only clusters.", + "rationale": "Requires merging DEVONthink item links, wiki links, mentions, tags, parents, and local mirror edges.", + "example": "devonthink-pp-cli graph audit --limit 50 --agent", + "why_it_matters": "Use this when DEVONthink should behave like a maintained knowledge graph instead of a folder pile.", + "group": "Local state that compounds" + }, + { + "name": "MCP passthrough", + "command": "mcp call", + "description": "Call DEVONthink's official local MCP tools from scripts when the local MCP server is enabled.", + "rationale": "Keeps new official MCP tools reachable before the CLI has promoted commands for them.", + "example": "devonthink-pp-cli mcp call search_records --args '{\"query\":\"kind:pdf\",\"limit\":5}' --agent", + "why_it_matters": "Use this when the official MCP exposes a new read tool before the CLI adds a first-class command.", + "group": "Agent-native plumbing" + } + ], + "narrative": { + "display_name": "DEVONthink", + "headline": "Local-first DEVONthink automation with safer shell workflows than raw AppleScript or MCP alone.", + "value_prop": "Use DEVONthink as the local source of truth while giving agents and scripts stable, compact CLI output. The CLI wraps the official local MCP surface, adds search scopes, inventory export, context packing, local mirrors, and safety-oriented workflow primitives.", + "when_to_use": "Use this CLI for local DEVONthink inventory, scoped record search, context packs, and read-only maintenance handoffs. It is designed for this Mac or the user's own LAN, not internet-hosted DEVONthink access.", + "anti_triggers": [ + "Do not use this CLI to access DEVONthink over the public internet.", + "Do not treat Smart Groups as action workflow policy; they are search scopes only.", + "Do not directly edit files inside DEVONthink database packages." + ], + "quickstart": [ + { + "command": "devonthink-pp-cli runtime doctor --json", + "comment": "Check local runtime readiness without touching DEVONthink content." + }, + { + "command": "devonthink-pp-cli records search \"kind:pdf\" --limit 5 --agent --select uuid,name,item_link", + "comment": "Find records while keeping agent output compact." + }, + { + "command": "devonthink-pp-cli selection snapshot --agent", + "comment": "Capture the current GUI selection as a repeatable workflow seed." + } + ], + "recipes": [ + { + "title": "Search within a Smart Group", + "command": "devonthink-pp-cli records search \"tags:waiting/rueckerstattung\" --smart-group \"Offene Rückerstattungen\" --agent --select uuid,name,item_link,tags,databaseName", + "explanation": "Scopes a normal query to a Smart Group and returns normal search rows plus meta.scope." + }, + { + "title": "Feed the maintenance plugin", + "command": "devonthink-pp-cli inventory export --format maintenance --query \"kind:document\" --limit 500 --agent --select databases.name,documents.name,documents.tags", + "explanation": "Produces stable inventory JSON for structure-audit and inbox-triage workflows." + } + ], + "trigger_phrases": [ + "search DEVONthink", + "scope DEVONthink search to a Smart Group", + "export DEVONthink maintenance inventory", + "build a DEVONthink context pack", + "use devonthink" + ] + } +} diff --git a/library/productivity/devonthink/.printing-press-patches/devonthink-smart-group-search-scope.json b/library/productivity/devonthink/.printing-press-patches/devonthink-smart-group-search-scope.json new file mode 100644 index 0000000000..d2058acd98 --- /dev/null +++ b/library/productivity/devonthink/.printing-press-patches/devonthink-smart-group-search-scope.json @@ -0,0 +1,18 @@ +{ + "schema_version": 2, + "id": "devonthink-smart-group-search-scope", + "applied_at": "2026-06-29", + "base_run_id": "20260629T175310Z-b4705e85", + "base_printing_press_version": "4.27.0", + "summary": "Add read-only Smart Group scope resolution for records search.", + "reason": "Downstream maintenance workflows need to scope searches by Smart Group UUID, exact name, or DEVONthink path while keeping Smart Groups out of action policy.", + "files": [ + "internal/client/smart_group_scope.go", + "internal/client/smart_group_scope_test.go", + "internal/cli/records_search.go", + "internal/cli/helpers.go", + "README.md", + "SKILL.md" + ], + "validated_outcome": "go test ./... and publish validate passed for the promoted CLI." +} diff --git a/library/productivity/devonthink/.printing-press.json b/library/productivity/devonthink/.printing-press.json new file mode 100644 index 0000000000..b7197a7948 --- /dev/null +++ b/library/productivity/devonthink/.printing-press.json @@ -0,0 +1,57 @@ +{ + "schema_version": 1, + "generated_at": "2026-06-29T17:56:26.526338Z", + "printing_press_version": "4.27.0", + "api_name": "devonthink", + "cli_name": "devonthink-pp-cli", + "creator": { + "handle": "Rowdy", + "name": "Johns" + }, + "contributors": [], + "printer": "Rowdy", + "printer_name": "Johns", + "spec_url": "https://www.devontechnologies.com/blog/20260526-devonthink-mcp-server", + "run_id": "20260629T175310Z-b4705e85", + "category": "productivity", + "description": "Local-first DEVONthink automation with safer shell workflows than raw AppleScript or MCP alone.", + "auth_type": "none", + "novel_features": [ + { + "name": "Smart Group scoped search", + "command": "records search", + "description": "Scope a normal DEVONthink query to a Smart Group by UUID, exact name, or DEVONthink path while preserving normal search output." + }, + { + "name": "Context pack", + "command": "context pack", + "description": "Build a compact evidence packet from records, selections, highlights, links, and related items." + }, + { + "name": "Privacy audit", + "command": "privacy audit", + "description": "Preview database scope, content-size budget, and cloud/MCP exposure before a handoff." + }, + { + "name": "Batch planning ledger", + "command": "batch plan", + "description": "Stage multi-record edits as validated dry-run plans before applying them." + }, + { + "name": "Maintenance inventory export", + "command": "inventory export", + "description": "Export DEVONthink databases, groups, tags, and document metadata for maintenance plugins." + }, + { + "name": "Knowledge graph audit", + "command": "graph audit", + "description": "Detect orphans, broken links, unresolved wiki links, weak hubs, and tag-only clusters." + }, + { + "name": "MCP passthrough", + "command": "mcp call", + "description": "Call DEVONthink's official local MCP tools from scripts when the local MCP server is enabled." + } + ], + "documentation_url": "https://www.devontechnologies.com/blog/20260526-devonthink-mcp-server" +} diff --git a/library/productivity/devonthink/AGENTS.md b/library/productivity/devonthink/AGENTS.md new file mode 100644 index 0000000000..9a7c6992e9 --- /dev/null +++ b/library/productivity/devonthink/AGENTS.md @@ -0,0 +1,46 @@ +# DEVONthink Printed CLI Agent Guide + +This directory is a generated `devonthink-pp-cli` printed CLI. It was produced by [CLI Printing Press](https://github.com/mvanhorn/cli-printing-press), so treat systemic fixes as upstream Printing Press fixes first. Keep local edits narrow and document why a generated-tree patch belongs here. + +## Local Operating Contract + +Start by asking the generated CLI for current runtime truth: + +```bash +devonthink-pp-cli doctor --json +devonthink-pp-cli agent-context --pretty +``` + +Use runtime discovery instead of relying on a copied command list: + +```bash +devonthink-pp-cli which "" --json +devonthink-pp-cli --help +``` + +Add `--agent` to command invocations for JSON, compact output, non-interactive defaults, no color, and confirmation-safe scripting: + +```bash +devonthink-pp-cli --agent +``` + +Before running an unfamiliar command that may mutate remote state, inspect its help and prefer a dry run: + +```bash +devonthink-pp-cli --help +devonthink-pp-cli --dry-run --agent +``` + +Use `--yes --no-input` only after the target, arguments, and side effects are clear. + +For install, auth, examples, and longer product guidance, read `README.md` and `SKILL.md`. This file intentionally stays small so repo-local agents get invariant local guidance without duplicating the generated docs. + +## Release Ledger + +`CHANGELOG.md` and `.printing-press-release.json` are the public library's per-CLI release ledger. Fresh prints may carry blank skeletons, but the final `YYYY.M.N` CLI release version is assigned only after a publish PR merges in `mvanhorn/printing-press-library`. Do not hand-bump those files or edit `var version = ...` for release bookkeeping; preserve existing ledger files on reprint and let the library workflow stamp the next release. + +## Local Customizations + +This directory is **generated output** -- a fresh print can overwrite the whole tree, so ad-hoc hand-edits don't survive on their own. If you modify the generated code, record each change under `.printing-press-patches/` (parallel to `.printing-press.json`) so a regen carries the intent forward instead of silently dropping it. + +The entry shape, and the altitude to write it at -- a durable reprint-guard, not a changelog -- live in the source catalog's `AGENTS.md`, which is the single source of truth; this guide intentionally doesn't duplicate them. diff --git a/library/productivity/devonthink/CHANGELOG.md b/library/productivity/devonthink/CHANGELOG.md new file mode 100644 index 0000000000..e75331a55f --- /dev/null +++ b/library/productivity/devonthink/CHANGELOG.md @@ -0,0 +1,4 @@ +# Changelog + +This file is maintained by printing-press-library release automation. Do not hand-edit release sections in normal PRs. + diff --git a/library/productivity/devonthink/LICENSE b/library/productivity/devonthink/LICENSE new file mode 100644 index 0000000000..d0ef9d5af2 --- /dev/null +++ b/library/productivity/devonthink/LICENSE @@ -0,0 +1,191 @@ + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to the Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by the Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding any notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + Copyright 2026 rowdy and contributors + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/library/productivity/devonthink/Makefile b/library/productivity/devonthink/Makefile new file mode 100644 index 0000000000..0cdeda838f --- /dev/null +++ b/library/productivity/devonthink/Makefile @@ -0,0 +1,24 @@ +.PHONY: build test lint install clean + +build: + go build -o bin/devonthink-pp-cli ./cmd/devonthink-pp-cli + +test: + go test ./... + +lint: + golangci-lint run + +install: + go install ./cmd/devonthink-pp-cli + +clean: + rm -rf bin/ + +build-mcp: + go build -o bin/devonthink-pp-mcp ./cmd/devonthink-pp-mcp + +install-mcp: + go install ./cmd/devonthink-pp-mcp + +build-all: build build-mcp diff --git a/library/productivity/devonthink/NOTICE b/library/productivity/devonthink/NOTICE new file mode 100644 index 0000000000..e3f1ea98f1 --- /dev/null +++ b/library/productivity/devonthink/NOTICE @@ -0,0 +1,11 @@ +devonthink-pp-cli +Copyright 2026 rowdy and contributors + +Created by (@Rowdy). + +This CLI was generated by CLI Printing Press (https://github.com/mvanhorn/cli-printing-press) +by Matt Van Horn and Trevin Chow. The Non-Obvious Insight, domain archetype detection, +workflow commands, and behavioral insight commands were produced by the printing press's +creative vision engine. + +CLI Printing Press is licensed separately under the MIT License. diff --git a/library/productivity/devonthink/README.md b/library/productivity/devonthink/README.md new file mode 100644 index 0000000000..33d7af6534 --- /dev/null +++ b/library/productivity/devonthink/README.md @@ -0,0 +1,482 @@ +# DEVONthink CLI + +**Local-first DEVONthink automation with safer shell workflows than raw AppleScript or MCP alone.** + +This CLI treats DEVONthink as a local knowledge database, not a cloud API. It wraps core record/search operations, adds a SQLite mirror for repeatable analysis, and provides stable inventory, graph, batch, and ledger contracts for higher-level maintenance plugins. + +## Install + +The recommended path installs both the `devonthink-pp-cli` binary and the `pp-devonthink` agent skill (Claude Code, Codex, Cursor, Gemini CLI, GitHub Copilot, and other agents supported by the upstream [`skills`](https://github.com/vercel-labs/skills) CLI) in one shot: + +```bash +npx -y @mvanhorn/printing-press-library install devonthink +``` + +For CLI only (no skill): + +```bash +npx -y @mvanhorn/printing-press-library install devonthink --cli-only +``` + +For skill only — installs the skill into the same agents as the default command above, but skips the CLI binary (use this to update or reinstall just the skill): + +```bash +npx -y @mvanhorn/printing-press-library install devonthink --skill-only +``` + +To constrain the skill install to one or more specific agents (repeatable — agent names match the [`skills`](https://github.com/vercel-labs/skills) CLI): + +```bash +npx -y @mvanhorn/printing-press-library install devonthink --agent claude-code +npx -y @mvanhorn/printing-press-library install devonthink --agent claude-code --agent codex +``` + +### Without Node (Go fallback) + +If `npx` isn't available (no Node, offline), install the CLI directly via Go (requires Go 1.26.4 or newer): + +```bash +go install github.com/mvanhorn/printing-press-library/library/productivity/devonthink/cmd/devonthink-pp-cli@latest +``` + +This installs the CLI only — no skill. + +### Pre-built binary + +Download a pre-built binary for your platform from the [latest release](https://github.com/mvanhorn/printing-press-library/releases/tag/devonthink-current). On macOS, clear the Gatekeeper quarantine: `xattr -d com.apple.quarantine `. On Unix, mark it executable: `chmod +x `. + + +## Install for Hermes + +Install the CLI binary first. The installer writes binaries to a per-user managed bin directory by default: `$HOME/.local/bin` on macOS/Linux and `%LOCALAPPDATA%\Programs\PrintingPress\bin` on Windows. + +```bash +npx -y @mvanhorn/printing-press-library install devonthink --cli-only +``` + +Then install the focused Hermes skill. + +From the Hermes CLI: + +```bash +hermes skills install mvanhorn/printing-press-library/cli-skills/pp-devonthink --force +``` + +Inside a Hermes chat session: + +```bash +/skills install mvanhorn/printing-press-library/cli-skills/pp-devonthink --force +``` + +Restart the Hermes session or gateway if the newly installed skill is not visible immediately. + +## Install for OpenClaw +Install both the CLI binary and the focused OpenClaw skill. The installer defaults binaries to a per-user bin directory (`$HOME/.local/bin` on macOS/Linux, `%LOCALAPPDATA%\Programs\PrintingPress\bin` on Windows): + +```bash +npx -y @mvanhorn/printing-press-library install devonthink --agent openclaw +``` + +Restart the OpenClaw session or gateway if the newly installed skill is not visible immediately. + +## Use with Claude Desktop + +This CLI ships an [MCPB](https://github.com/modelcontextprotocol/mcpb) bundle — Claude Desktop's standard format for one-click MCP extension installs (no JSON config required). + +To install: + +1. Download the `.mcpb` for your platform from the [latest release](https://github.com/mvanhorn/printing-press-library/releases/tag/devonthink-current). +2. Double-click the `.mcpb` file. Claude Desktop opens and walks you through the install. + +Requires Claude Desktop 1.0.0 or later. Pre-built bundles ship for macOS Apple Silicon (`darwin-arm64`) and Windows (`amd64`, `arm64`); for other platforms, use the manual config below. + +
+Manual JSON config (advanced) + +If you can't use the MCPB bundle (older Claude Desktop, unsupported platform), install the MCP binary and configure it manually. + + +```bash +go install github.com/mvanhorn/printing-press-library/library/productivity/devonthink/cmd/devonthink-pp-mcp@latest +``` + +Add to your Claude Desktop config (`~/Library/Application Support/Claude/claude_desktop_config.json`): + +```json +{ + "mcpServers": { + "devonthink": { + "command": "devonthink-pp-mcp" + } + } +} +``` + +
+ +## Authentication + +Default operation uses local macOS automation and requires no API key. Optional official MCP passthrough uses DEVONthink's local MCP server when you enable it in DEVONthink; keep it bound to localhost or your own LAN and set any bearer token through DEVONthink's MCP settings. + +## Quick Start + +```bash +# Check local runtime readiness without touching DEVONthink. +devonthink-pp-cli runtime doctor --json + +# Preview the stable inventory contract consumed by maintenance workflows. +devonthink-pp-cli inventory export --format maintenance --query "kind:document" --limit 500 --output devonthink-inventory.json + +# Find records while keeping agent output compact. +devonthink-pp-cli records search "kind:pdf" --limit 5 --agent --select uuid,name,item_link + +# Scope a normal search to a Smart Group by UUID, exact name, or DEVONthink path. +devonthink-pp-cli records search "tags:waiting/rueckerstattung" --smart-group "Offene Rückerstattungen" --agent --select uuid,name,item_link,tags,databaseName + +# Capture the current GUI selection as a repeatable workflow seed. +devonthink-pp-cli selection snapshot --agent + +# Build a bounded evidence packet for local reasoning. +devonthink-pp-cli context pack --query "project alpha" --token-budget 6000 --agent + +``` + +## Unique Features + +These capabilities aren't available in any other tool for this API. + +### Local state that compounds +- **`context pack`** — Build a compact evidence packet from records, selections, highlights, links, and related items. + + _Use this when an agent needs enough DEVONthink context to reason without dumping whole documents._ + + ```bash + devonthink-pp-cli context pack --query "project alpha" --token-budget 6000 --agent --select markdown,records + ``` +- **`graph audit`** — Detect orphans, broken links, unresolved wiki links, weak hubs, and tag-only clusters. + + _Use this when DEVONthink should behave like a maintained knowledge graph instead of a folder pile._ + + ```bash + devonthink-pp-cli graph audit --database Research --agent --select issues,type,count,samples + ``` +- **`mirror search`** — Query a local SQLite mirror for repeatable fast analysis without repeated app calls. + + _Use this for repeated analysis, dashboards, and low-token agent workflows._ + + ```bash + devonthink-pp-cli mirror search "tag:tax" --limit 20 --agent --select uuid,name,path + ``` + +### Local-first safety +- **`privacy audit`** — Preview what a workflow may expose before content leaves the local machine. + + _Use this before sending DEVONthink-derived context to an external model or shared MCP endpoint._ + + ```bash + devonthink-pp-cli privacy audit --query "invoice" --limit 10 --agent + ``` +- **`agent-context`** — Emit an agent contract that enforces local-machine and own-LAN DEVONthink access only. + + _Use this before handing DEVONthink access to an agent that must avoid remote control paths._ + + ```bash + devonthink-pp-cli agent-context --local-only --agent + ``` + +### Safe automation +- **`batch plan`** — Stage multi-record edits as validated dry-run plans before applying them. + + _Use this for multi-record writes where each target must be checked before mutation._ + + ```bash + devonthink-pp-cli batch plan --from selection --add-tag reviewed --move-to /Archive --agent + ``` +- **`ledger list`** — Review CLI-driven mutation plans, applies, target proofs, and rollback hints. + + _Use this to audit or explain what recent automation did to DEVONthink._ + + ```bash + devonthink-pp-cli ledger list --since 7d --agent --select time,action,count,status + ``` +- **`selection snapshot`** — Turn the current GUI selection into a reusable JSON workflow seed. + + _Use this when the human has curated records in the GUI and wants an agent-safe handoff._ + + ```bash + devonthink-pp-cli selection snapshot --note "review these PDFs" --agent + ``` + +### Agent-native plumbing +- **`inventory export`** — Export DEVONthink databases, groups, tags, and document metadata for maintenance plugins. + + _Use this when structure-audit or inbox-triage tooling needs a stable local inventory contract._ + + ```bash + devonthink-pp-cli inventory export --format maintenance --query "kind:document" --limit 500 --output devonthink-inventory.json --agent --select databases,documents + ``` +- **`mcp call`** — Call DEVONthink's official local MCP tools from scripts when the local MCP server is enabled. + + _Use this when the official MCP exposes a new tool before the CLI has a promoted command._ + + ```bash + devonthink-pp-cli mcp call search_records --args '{"query":"kind:pdf","limit":5}' --agent + ``` + +## Recipes + + +### Compact search for an agent + +```bash +devonthink-pp-cli records search "tags:review AND kind:pdf" --limit 10 --agent --select uuid,name,item_link,tags +``` + +Returns only the fields an agent needs to pick the next record. + +### Search within a Smart Group + +```bash +devonthink-pp-cli records search "tags:waiting/rueckerstattung" \ + --smart-group "Offene Rückerstattungen" \ + --agent \ + --select uuid,name,item_link,tags,databaseName +``` + +Smart Groups are search scopes only. They do not define action workflow policy; downstream maintenance tools should still apply their own review and transition rules. + +### Feed the maintenance plugin + +```bash +devonthink-pp-cli inventory export --format maintenance --query "kind:document" --limit 500 --output devonthink-inventory.json --agent --select databases.name,documents.name,documents.tags +``` + +Produces the stable inventory JSON that structure-audit and inbox-triage workflows can consume. + +### Create a local evidence packet + +```bash +devonthink-pp-cli context pack --query "family invoices" --token-budget 5000 --agent --select markdown,records.item_link +``` + +Builds a bounded context bundle without dumping entire records. + +### Plan a safe tag cleanup + +```bash +devonthink-pp-cli batch plan --query "tags:todo" --add-tag reviewed --dry-run --agent +``` + +Stages a batch change for review before any record is mutated. + +### Audit graph health + +```bash +devonthink-pp-cli graph audit --database Research --agent --select issues,type,count,samples +``` + +Finds link and organization gaps from the local mirror. + +## Usage + +Run `devonthink-pp-cli --help` for the full command reference and flag list. + +## Commands + +### ai + +DEVONthink AI and summary helpers + +- **`devonthink-pp-cli ai ask`** - Ask DEVONthink AI about selected local records with explicit cloud-use warnings +- **`devonthink-pp-cli ai summarize`** - Summarize records or highlights + +### batch + +Dry-run-first multi-record mutation plans + +- **`devonthink-pp-cli batch apply`** - Apply a previously reviewed local JSON plan +- **`devonthink-pp-cli batch plan`** - Stage multi-record changes as a local JSON plan + +### context + +Agent context bundles + +- **`devonthink-pp-cli context`** - Build a compact local context pack from records, selection, or search + +### databases + +Open DEVONthink databases + +- **`devonthink-pp-cli databases`** - List open databases + +### graph + +Links, mentions, and knowledge graph health + +- **`devonthink-pp-cli graph audit`** - Detect orphans, unresolved wiki links, weak hubs, and tag-only clusters +- **`devonthink-pp-cli graph links`** - List item links, wiki links, mentions, and unresolved wiki names + +### groups + +DEVONthink groups and folders + +- **`devonthink-pp-cli groups`** - Render a bounded group tree + +### ingest + +File and URL ingestion + +- **`devonthink-pp-cli ingest file`** - Import or index a file or folder +- **`devonthink-pp-cli ingest url`** - Capture a URL as Markdown, HTML, PDF, bookmark, or webarchive + +### inventory + +Stable inventory export contracts + +- **`devonthink-pp-cli inventory`** - Export databases, groups, tags, and selected document metadata for downstream tools + +### ledger + +Local operation ledger + +- **`devonthink-pp-cli ledger list`** - List recent CLI operation ledger entries +- **`devonthink-pp-cli ledger show`** - Show one ledger entry with target proofs and rollback hints + +### mcp + +Optional local official MCP passthrough + +- **`devonthink-pp-cli mcp call`** - Call a local official DEVONthink MCP tool by name +- **`devonthink-pp-cli mcp schema`** - Emit cached MCP tool schemas +- **`devonthink-pp-cli mcp tools`** - List official DEVONthink MCP tools when local MCP HTTP is enabled + +### media + +OCR and transcription + +- **`devonthink-pp-cli media ocr`** - OCR an image or scanned PDF +- **`devonthink-pp-cli media transcribe`** - Transcribe audio, video, image, or PDF content + +### mirror + +Local SQLite mirror + +- **`devonthink-pp-cli mirror search`** - Search the local mirror with FTS +- **`devonthink-pp-cli mirror sync`** - Refresh the local SQLite mirror from open DEVONthink databases + +### privacy + +Local privacy and exposure reports + +- **`devonthink-pp-cli privacy`** - Preview database scope, content-size budget, and cloud/MCP exposure before handoff + +### records + +DEVONthink records + +- **`devonthink-pp-cli records content`** - Extract text content with length and redaction controls +- **`devonthink-pp-cli records create`** - Create a record or group after validating destination +- **`devonthink-pp-cli records get`** - Get record metadata +- **`devonthink-pp-cli records highlights`** - Extract highlights and annotations +- **`devonthink-pp-cli records lookup`** - Look up records by exact name, URL, path, filename, location, or comment +- **`devonthink-pp-cli records move`** - Move, duplicate, replicate, or trash a record with dry-run proof +- **`devonthink-pp-cli records related`** - Find related records using DEVONthink similarity +- **`devonthink-pp-cli records search`** - Search records using DEVONthink query syntax or local mirror fallback +- **`devonthink-pp-cli records update`** - Update record text, properties, tags, comment, URL, aliases, or rating +- **`devonthink-pp-cli records versions`** - List saved record versions + +### runtime + +Local DEVONthink runtime health + +- **`devonthink-pp-cli runtime`** - Check DEVONthink app, AppleScript, optional MCP, and local mirror readiness + +### selection + +Current DEVONthink GUI selection + +- **`devonthink-pp-cli selection get`** - Return currently selected records +- **`devonthink-pp-cli selection snapshot`** - Capture the current selection as a reusable workflow seed + +### sheets + +DEVONthink sheets + +- **`devonthink-pp-cli sheets `** - Read a sheet as structured rows + +### tags + +Tag taxonomy and hygiene + +- **`devonthink-pp-cli tags`** - Analyze tags for duplicates, case drift, action tags, and maintenance tags + + +## Output Formats + +```bash +# Human-readable table (default in terminal, JSON when piped) +devonthink-pp-cli databases + +# JSON for scripting and agents +devonthink-pp-cli databases --json + +# Filter to specific fields +devonthink-pp-cli databases --json --select id,name,status + +# Dry run — show the request without sending +devonthink-pp-cli databases --dry-run + +# Agent mode — JSON + compact + no prompts in one flag +devonthink-pp-cli databases --agent +``` + +## Agent Usage + +This CLI is designed for AI agent consumption: + +- **Non-interactive** - never prompts, every input is a flag +- **Pipeable** - `--json` output to stdout, errors to stderr +- **Filterable** - `--select id,name` returns only fields you need +- **Previewable** - `--dry-run` shows the request without sending +- **Explicit retries** - add `--idempotent` to create retries when a no-op success is acceptable +- **Confirmable** - `--yes` for explicit confirmation of destructive actions +- **Piped input** - write commands can accept structured input when their help lists `--stdin` +- **Offline-friendly** - sync/search commands can use the local SQLite store when available +- **Agent-safe by default** - no colors or formatting unless `--human-friendly` is set + +Exit codes: `0` success, `2` usage error, `3` not found, `5` API error, `7` rate limited, `10` config error. + +## Health Check + +```bash +devonthink-pp-cli doctor +``` + +Verifies configuration and connectivity to the API. + +## Configuration + +Config file: `~/.config/devonthink-pp-cli/config.toml` + +Static request headers can be configured under `headers`; per-command header overrides take precedence. + +## Troubleshooting +**Not found errors (exit code 3)** +- Check the resource ID is correct +- Run the `list` command to see available items + +### API-specific +- **doctor reports DEVONthink is not running** — Open DEVONthink locally, then rerun devonthink-pp-cli doctor. +- **MCP passthrough commands fail** — Enable DEVONthink Settings > AI > MCP and verify the local endpoint or use native CLI commands instead. +- **mirror search returns no rows** — Run devonthink-pp-cli mirror sync before querying the local mirror. + +## Sources & Inspiration + +This CLI was built by studying these projects and resources: + +- [**dvcrn/devonthink-cli**](https://github.com/dvcrn/devonthink-cli) — TypeScript (15 stars) +- [**2b3pro/Devonthink-MCP-CLI**](https://github.com/2b3pro/Devonthink-MCP-CLI) — JavaScript (3 stars) +- [**TomBener/dtx**](https://github.com/TomBener/dtx) — TypeScript (3 stars) +- [**fenrick/devonthink-cli**](https://github.com/fenrick/devonthink-cli) — Swift (2 stars) + +Generated by [CLI Printing Press](https://github.com/mvanhorn/cli-printing-press) diff --git a/library/productivity/devonthink/SKILL.md b/library/productivity/devonthink/SKILL.md new file mode 100644 index 0000000000..3acc95d70d --- /dev/null +++ b/library/productivity/devonthink/SKILL.md @@ -0,0 +1,408 @@ +--- +name: pp-devonthink +description: "Local-first DEVONthink automation with safer shell workflows than raw AppleScript or MCP alone. Trigger phrases: `search DEVONthink`, `pack DEVONthink context`, `export DEVONthink maintenance inventory`, `audit DEVONthink links`, `batch tag DEVONthink records`, `use devonthink`, `run devonthink`." +author: "rowdy" +license: "Apache-2.0" +argument-hint: " [args] | install cli|mcp" +allowed-tools: "Read Bash" +metadata: + openclaw: + requires: + bins: + - devonthink-pp-cli + install: + - kind: go + bins: [devonthink-pp-cli] + module: github.com/mvanhorn/printing-press-library/library/productivity/devonthink/cmd/devonthink-pp-cli +--- + +# DEVONthink — Printing Press CLI + +## Prerequisites: Install the CLI + +This skill drives the `devonthink-pp-cli` binary. **You must verify the CLI is installed before invoking any command from this skill.** If it is missing, install it first: + +1. Install via the Printing Press installer. It defaults binaries to `$HOME/.local/bin` on macOS/Linux and `%LOCALAPPDATA%\Programs\PrintingPress\bin` on Windows: + ```bash + npx -y @mvanhorn/printing-press-library install devonthink --cli-only + ``` +2. Verify: `devonthink-pp-cli --version` +3. Ensure the reported install directory is on `$PATH` for the agent/runtime that will invoke this skill. + +If the `npx` install fails (no Node, offline, etc.), fall back to a direct Go install (requires Go 1.26.4 or newer). This installs into `$GOPATH/bin` (default `$HOME/go/bin`), so add that directory to `$PATH` instead: + +```bash +go install github.com/mvanhorn/printing-press-library/library/productivity/devonthink/cmd/devonthink-pp-cli@latest +``` + +If `--version` reports "command not found" after install, the runtime cannot see the binary directory on `$PATH`. Do not proceed with skill commands until verification succeeds. + +This CLI treats DEVONthink as a local knowledge database, not a cloud API. It wraps core record/search operations, adds a SQLite mirror for repeatable analysis, and provides stable inventory, graph, batch, and ledger contracts for higher-level maintenance plugins. + +## When to Use This CLI + +Use this CLI for local DEVONthink search, context preparation, inventory export, safe batch edits, and shell automation. Prefer it when an agent needs deterministic JSON and compact outputs instead of a GUI or raw MCP chat interaction. Let dedicated maintenance plugins own filing policy and recurring inbox triage decisions. + +## Anti-triggers + +Do not use this CLI for: +- Do not use this CLI to expose DEVONthink over the public internet. +- Do not use this CLI as the canonical store for documents; DEVONthink remains canonical. +- Do not put filing-model policy or recurring inbox triage decisions in the core CLI; use a maintenance plugin on top of inventory export. +- Do not use write commands for bulk changes without a dry-run batch plan. +- Do not use semantic search unless a local mirror/index has been built and the embedding provider is intentionally configured. + +## Unique Capabilities + +These capabilities aren't available in any other tool for this API. + +### Local state that compounds +- **`context pack`** — Build a compact evidence packet from records, selections, highlights, links, and related items. + + _Use this when an agent needs enough DEVONthink context to reason without dumping whole documents._ + + ```bash + devonthink-pp-cli context pack --query "project alpha" --token-budget 6000 --agent --select markdown,records + ``` +- **`graph audit`** — Detect orphans, broken links, unresolved wiki links, weak hubs, and tag-only clusters. + + _Use this when DEVONthink should behave like a maintained knowledge graph instead of a folder pile._ + + ```bash + devonthink-pp-cli graph audit --database Research --agent --select issues,type,count,samples + ``` +- **`mirror search`** — Query a local SQLite mirror for repeatable fast analysis without repeated app calls. + + _Use this for repeated analysis, dashboards, and low-token agent workflows._ + + ```bash + devonthink-pp-cli mirror search "tag:tax" --limit 20 --agent --select uuid,name,path + ``` + +### Local-first safety +- **`privacy audit`** — Preview what a workflow may expose before content leaves the local machine. + + _Use this before sending DEVONthink-derived context to an external model or shared MCP endpoint._ + + ```bash + devonthink-pp-cli privacy audit --query "invoice" --limit 10 --agent + ``` +- **`agent-context`** — Emit an agent contract that enforces local-machine and own-LAN DEVONthink access only. + + _Use this before handing DEVONthink access to an agent that must avoid remote control paths._ + + ```bash + devonthink-pp-cli agent-context --local-only --agent + ``` + +### Safe automation +- **`batch plan`** — Stage multi-record edits as validated dry-run plans before applying them. + + _Use this for multi-record writes where each target must be checked before mutation._ + + ```bash + devonthink-pp-cli batch plan --from selection --add-tag reviewed --move-to /Archive --agent + ``` +- **`ledger list`** — Review CLI-driven mutation plans, applies, target proofs, and rollback hints. + + _Use this to audit or explain what recent automation did to DEVONthink._ + + ```bash + devonthink-pp-cli ledger list --since 7d --agent --select time,action,count,status + ``` +- **`selection snapshot`** — Turn the current GUI selection into a reusable JSON workflow seed. + + _Use this when the human has curated records in the GUI and wants an agent-safe handoff._ + + ```bash + devonthink-pp-cli selection snapshot --note "review these PDFs" --agent + ``` + +### Agent-native plumbing +- **`inventory export`** — Export DEVONthink databases, groups, tags, and document metadata for maintenance plugins. + + _Use this when structure-audit or inbox-triage tooling needs a stable local inventory contract._ + + ```bash + devonthink-pp-cli inventory export --format maintenance --query "kind:document" --limit 500 --output devonthink-inventory.json --agent --select databases,documents + ``` +- **`mcp call`** — Call DEVONthink's official local MCP tools from scripts when the local MCP server is enabled. + + _Use this when the official MCP exposes a new tool before the CLI has a promoted command._ + + ```bash + devonthink-pp-cli mcp call search_records --args '{"query":"kind:pdf","limit":5}' --agent + ``` + +## Command Reference + +**ai** — DEVONthink AI and summary helpers + +- `devonthink-pp-cli ai ask` — Ask DEVONthink AI about selected local records with explicit cloud-use warnings +- `devonthink-pp-cli ai summarize` — Summarize records or highlights + +**batch** — Dry-run-first multi-record mutation plans + +- `devonthink-pp-cli batch apply` — Apply a previously reviewed local JSON plan +- `devonthink-pp-cli batch plan` — Stage multi-record changes as a local JSON plan + +**context** — Agent context bundles + +- `devonthink-pp-cli context` — Build a compact local context pack from records, selection, or search + +**databases** — Open DEVONthink databases + +- `devonthink-pp-cli databases` — List open databases + +**graph** — Links, mentions, and knowledge graph health + +- `devonthink-pp-cli graph audit` — Detect orphans, unresolved wiki links, weak hubs, and tag-only clusters +- `devonthink-pp-cli graph links` — List item links, wiki links, mentions, and unresolved wiki names + +**groups** — DEVONthink groups and folders + +- `devonthink-pp-cli groups` — Render a bounded group tree + +**ingest** — File and URL ingestion + +- `devonthink-pp-cli ingest file` — Import or index a file or folder +- `devonthink-pp-cli ingest url` — Capture a URL as Markdown, HTML, PDF, bookmark, or webarchive + +**inventory** — Stable inventory export contracts + +- `devonthink-pp-cli inventory` — Export databases, groups, tags, and selected document metadata for downstream tools + +**ledger** — Local operation ledger + +- `devonthink-pp-cli ledger list` — List recent CLI operation ledger entries +- `devonthink-pp-cli ledger show` — Show one ledger entry with target proofs and rollback hints + +**mcp** — Optional local official MCP passthrough + +- `devonthink-pp-cli mcp call` — Call a local official DEVONthink MCP tool by name +- `devonthink-pp-cli mcp schema` — Emit cached MCP tool schemas +- `devonthink-pp-cli mcp tools` — List official DEVONthink MCP tools when local MCP HTTP is enabled + +**media** — OCR and transcription + +- `devonthink-pp-cli media ocr` — OCR an image or scanned PDF +- `devonthink-pp-cli media transcribe` — Transcribe audio, video, image, or PDF content + +**mirror** — Local SQLite mirror + +- `devonthink-pp-cli mirror search` — Search the local mirror with FTS +- `devonthink-pp-cli mirror sync` — Refresh the local SQLite mirror from open DEVONthink databases + +**privacy** — Local privacy and exposure reports + +- `devonthink-pp-cli privacy` — Preview database scope, content-size budget, and cloud/MCP exposure before handoff + +**records** — DEVONthink records + +- `devonthink-pp-cli records content` — Extract text content with length and redaction controls +- `devonthink-pp-cli records create` — Create a record or group after validating destination +- `devonthink-pp-cli records get` — Get record metadata +- `devonthink-pp-cli records highlights` — Extract highlights and annotations +- `devonthink-pp-cli records lookup` — Look up records by exact name, URL, path, filename, location, or comment +- `devonthink-pp-cli records move` — Move, duplicate, replicate, or trash a record with dry-run proof +- `devonthink-pp-cli records related` — Find related records using DEVONthink similarity +- `devonthink-pp-cli records search` — Search records using DEVONthink query syntax or local mirror fallback +- `devonthink-pp-cli records update` — Update record text, properties, tags, comment, URL, aliases, or rating +- `devonthink-pp-cli records versions` — List saved record versions + +**runtime** — Local DEVONthink runtime health + +- `devonthink-pp-cli runtime` — Check DEVONthink app, AppleScript, optional MCP, and local mirror readiness + +**selection** — Current DEVONthink GUI selection + +- `devonthink-pp-cli selection get` — Return currently selected records +- `devonthink-pp-cli selection snapshot` — Capture the current selection as a reusable workflow seed + +**sheets** — DEVONthink sheets + +- `devonthink-pp-cli sheets ` — Read a sheet as structured rows + +**tags** — Tag taxonomy and hygiene + +- `devonthink-pp-cli tags` — Analyze tags for duplicates, case drift, action tags, and maintenance tags + + +### Finding the right command + +When you know what you want to do but not which command does it, ask the CLI directly: + +```bash +devonthink-pp-cli which "" +``` + +`which` resolves a natural-language capability query to the best matching command from this CLI's curated feature index. Exit code `0` means at least one match; exit code `2` means no confident match — fall back to `--help` or use a narrower query. + +## Recipes + +### Compact search for an agent + +```bash +devonthink-pp-cli records search "tags:review AND kind:pdf" --limit 10 --agent --select uuid,name,item_link,tags +``` + +Returns only the fields an agent needs to pick the next record. + +### Search within a Smart Group + +```bash +devonthink-pp-cli records search "tags:waiting/rueckerstattung" \ + --smart-group "Offene Rückerstattungen" \ + --agent \ + --select uuid,name,item_link,tags,databaseName +``` + +Smart Groups are search scopes only. They are not action workflow policy; use maintenance workflow rules for triage or lifecycle decisions. + +### Feed the maintenance plugin + +```bash +devonthink-pp-cli inventory export --format maintenance --query "kind:document" --limit 500 --output devonthink-inventory.json --agent --select databases.name,documents.name,documents.tags +``` + +Produces the stable inventory JSON that structure-audit and inbox-triage workflows can consume. + +### Create a local evidence packet + +```bash +devonthink-pp-cli context pack --query "family invoices" --token-budget 5000 --agent --select markdown,records.item_link +``` + +Builds a bounded context bundle without dumping entire records. + +### Plan a safe tag cleanup + +```bash +devonthink-pp-cli batch plan --query "tags:todo" --add-tag reviewed --dry-run --agent +``` + +Stages a batch change for review before any record is mutated. + +### Audit graph health + +```bash +devonthink-pp-cli graph audit --database Research --agent --select issues,type,count,samples +``` + +Finds link and organization gaps from the local mirror. + +## Auth Setup + +Default operation uses local macOS automation and requires no API key. Optional official MCP passthrough uses DEVONthink's local MCP server when you enable it in DEVONthink; keep it bound to localhost or your own LAN and set any bearer token through DEVONthink's MCP settings. + +Run `devonthink-pp-cli doctor` to verify setup. + +## Agent Mode + +Add `--agent` to any command. Expands to: `--json --compact --no-input --no-color --yes`. + +- **Pipeable** — JSON on stdout, errors on stderr +- **Filterable** — `--select` keeps a subset of fields. Dotted paths descend into nested structures; arrays traverse element-wise. Critical for keeping context small on verbose APIs: + + ```bash + devonthink-pp-cli databases --agent --select id,name,status + ``` +- **Previewable** — `--dry-run` shows the request without sending +- **Offline-friendly** — sync/search commands can use the local SQLite store when available +- **Non-interactive** — never prompts, every input is a flag +- **Explicit retries** — use `--idempotent` only when an already-existing create should count as success + +### Response envelope + +Commands that read from the local store or the API wrap output in a provenance envelope: + +```json +{ + "meta": {"source": "live" | "local", "synced_at": "...", "reason": "..."}, + "results": +} +``` + +Parse `.results` for data and `.meta.source` to know whether it's live or local. A human-readable `N results (live)` summary is printed to stderr only when stdout is a terminal AND no machine-format flag (`--json`, `--csv`, `--compact`, `--quiet`, `--plain`, `--select`) is set — piped/agent consumers and explicit-format runs get pure JSON on stdout. + +## Agent Feedback + +When you (or the agent) notice something off about this CLI, record it: + +``` +devonthink-pp-cli feedback "the --since flag is inclusive but docs say exclusive" +devonthink-pp-cli feedback --stdin < notes.txt +devonthink-pp-cli feedback list --json --limit 10 +``` + +Entries are stored locally at `~/.local/share/devonthink-pp-cli/feedback.jsonl`. They are never POSTed unless `DEVONTHINK_FEEDBACK_ENDPOINT` is set AND either `--send` is passed or `DEVONTHINK_FEEDBACK_AUTO_SEND=true`. Default behavior is local-only. + +Write what *surprised* you, not a bug report. Short, specific, one line: that is the part that compounds. + +## Output Delivery + +Every command accepts `--deliver `. The output goes to the named sink in addition to (or instead of) stdout, so agents can route command results without hand-piping. Three sinks are supported: + +| Sink | Effect | +|------|--------| +| `stdout` | Default; write to stdout only | +| `file:` | Atomically write output to `` (tmp + rename) | +| `webhook:` | POST the output body to the URL (`application/json` or `application/x-ndjson` when `--compact`) | + +Unknown schemes are refused with a structured error naming the supported set. Webhook failures return non-zero and log the URL + HTTP status on stderr. + +## Named Profiles + +A profile is a saved set of flag values, reused across invocations. Use it when a scheduled agent calls the same command every run with the same configuration - HeyGen's "Beacon" pattern. + +``` +devonthink-pp-cli profile save briefing --json +devonthink-pp-cli --profile briefing databases +devonthink-pp-cli profile list --json +devonthink-pp-cli profile show briefing +devonthink-pp-cli profile delete briefing --yes +``` + +Explicit flags always win over profile values; profile values win over defaults. `agent-context` lists all available profiles under `available_profiles` so introspecting agents discover them at runtime. + +## Exit Codes + +| Code | Meaning | +|------|---------| +| 0 | Success | +| 2 | Usage error (wrong arguments) | +| 3 | Resource not found | +| 5 | API error (upstream issue) | +| 7 | Rate limited (wait and retry) | +| 10 | Config error | + +## Argument Parsing + +Parse `$ARGUMENTS`: + +1. **Empty, `help`, or `--help`** → show `devonthink-pp-cli --help` output +2. **Starts with `install`** → ends with `mcp` → MCP installation; otherwise → see Prerequisites above +3. **Anything else** → Direct Use (execute as CLI command with `--agent`) + +## MCP Server Installation + +1. Install the MCP server: + ```bash + go install github.com/mvanhorn/printing-press-library/library/productivity/devonthink/cmd/devonthink-pp-mcp@latest + ``` +2. Register with Claude Code: + ```bash + claude mcp add devonthink-pp-mcp -- devonthink-pp-mcp + ``` +3. Verify: `claude mcp list` + +## Direct Use + +1. Check if installed: `which devonthink-pp-cli` + If not found, offer to install (see Prerequisites at the top of this skill). +2. Match the user query to the best command from the Unique Capabilities and Command Reference above. +3. Execute with the `--agent` flag: + ```bash + devonthink-pp-cli [subcommand] [args] --agent + ``` +4. If ambiguous, drill into subcommand help: `devonthink-pp-cli --help`. diff --git a/library/productivity/devonthink/cmd/devonthink-pp-cli/main.go b/library/productivity/devonthink/cmd/devonthink-pp-cli/main.go new file mode 100644 index 0000000000..6744d2096c --- /dev/null +++ b/library/productivity/devonthink/cmd/devonthink-pp-cli/main.go @@ -0,0 +1,16 @@ +// Copyright 2026 rowdy and contributors. Licensed under Apache-2.0. See LICENSE. +// Generated by CLI Printing Press (https://github.com/mvanhorn/cli-printing-press). DO NOT EDIT. + +package main + +import ( + "os" + + "github.com/mvanhorn/printing-press-library/library/productivity/devonthink/internal/cli" +) + +func main() { + if err := cli.Execute(); err != nil { + os.Exit(cli.ExitCode(err)) + } +} diff --git a/library/productivity/devonthink/cmd/devonthink-pp-mcp/main.go b/library/productivity/devonthink/cmd/devonthink-pp-mcp/main.go new file mode 100644 index 0000000000..3aca1747b9 --- /dev/null +++ b/library/productivity/devonthink/cmd/devonthink-pp-mcp/main.go @@ -0,0 +1,30 @@ +// Copyright 2026 rowdy and contributors. Licensed under Apache-2.0. See LICENSE. +// Generated by CLI Printing Press (https://github.com/mvanhorn/cli-printing-press). DO NOT EDIT. + +package main + +import ( + "fmt" + "os" + + "github.com/mark3labs/mcp-go/server" + mcptools "github.com/mvanhorn/printing-press-library/library/productivity/devonthink/internal/mcp" +) + +// version is the printed MCP server's version, overridable at build time via ldflags. +var version = "0.0.0-dev" + +func main() { + s := server.NewMCPServer( + "Devonthink", + version, + server.WithToolCapabilities(false), + ) + + mcptools.RegisterTools(s) + + if err := server.ServeStdio(s); err != nil { + fmt.Fprintf(os.Stderr, "MCP server error: %v\n", err) + os.Exit(1) + } +} diff --git a/library/productivity/devonthink/docs/plans/2026-06-22-001-feat-smart-group-search-scope-plan.md b/library/productivity/devonthink/docs/plans/2026-06-22-001-feat-smart-group-search-scope-plan.md new file mode 100644 index 0000000000..97b18fd365 --- /dev/null +++ b/library/productivity/devonthink/docs/plans/2026-06-22-001-feat-smart-group-search-scope-plan.md @@ -0,0 +1,184 @@ +--- +title: "feat: Add Smart Group scope resolution to records search" +type: "feat" +date: "2026-06-22" +--- + +# feat: Add Smart Group scope resolution to records search + +## Summary + +Add `--smart-group` to `devonthink-pp-cli records search` so users and agents can scope a normal DEVONthink query to a Smart Group by UUID, exact name, or DEVONthink path. The command remains read-only, delegates the final record search through `search_records` with `group_uuid`, and enriches the normal provenance envelope with `meta.scope`. + +--- + +## Problem Frame + +Downstream tools need a stable CLI contract for searches such as reimbursement follow-up lists without embedding raw MCP calls or hand-resolving Smart Group UUIDs. The current command already accepts `--group` and maps it to MCP `group_uuid`, but Smart Groups are easier for humans to name and safer for agents to treat as a scoped search source than as a workflow-policy surface. + +--- + +## Requirements + +- R1. `records search --smart-group ` accepts a Smart Group UUID and uses it as the search scope. +- R2. The same flag resolves an exact Smart Group name or DEVONthink path to one Smart Group UUID before running the record search. +- R3. Ambiguous exact-name matches fail clearly and list enough candidate context for the user or agent to disambiguate. +- R4. The final record search still calls the existing `search_records` path with `group_uuid`; Smart Group resolution must not create a separate result source. +- R5. `--agent`, `--json`, and `--select` preserve the existing result envelope and field-selection behavior. +- R6. Successful Smart Group scoped JSON output includes `meta.scope.type`, `meta.scope.input`, and `meta.scope.uuid`. +- R7. The feature remains read-only and documents Smart Groups as search scopes only, not action-workflow policy. +- R8. Generated-tree local customization intent is recorded under `.printing-press-patches/` so a future reprint can preserve the change. +- R9. Smart Group scoping fails clearly when the caller forces local cached data because cached records cannot reproduce DEVONthink Smart Group membership. + +--- + +## High-Level Technical Design + +```mermaid +flowchart TB + A["records search query + flags"] --> B{"--smart-group set?"} + B -->|no| C["existing records search path"] + B -->|yes| D["resolve Smart Group input"] + D --> E{"unique Smart Group?"} + E -->|no match| F["clear not-found error"] + E -->|duplicate name| G["clear ambiguous-name error"] + E -->|yes| H["set group_uuid to resolved UUID"] + H --> I["call search_records through existing read strategy"] + I --> J["apply limit and output filtering"] + J --> K["wrap results with meta.source and meta.scope"] +``` + +--- + +## Key Technical Decisions + +- KTD1. Keep Smart Group scoping on `records search`, not in maintenance workflows: this preserves the CLI as a local data access layer while leaving filing policy and action semantics to downstream tools. +- KTD2. Resolve by official MCP read tools before the final search: `search_records` already returns Smart Groups with `kind:smartgroup`, and the local adapter already routes reads through the official local MCP bridge. +- KTD3. Treat `--group` and `--smart-group` as mutually exclusive scopes: allowing both would make the final `group_uuid` source ambiguous. +- KTD4. Extend the provenance envelope with optional scope metadata: `wrapWithProvenance` is already the stable place where `meta.source` is emitted, and applying `--select` before wrapping keeps `meta.scope` intact for agent consumers. +- KTD5. Prefer exact matching over fuzzy matching: exact name and normalized path matches are deterministic, while fuzzy matching would make ambiguous Smart Groups harder to diagnose. +- KTD6. Treat Smart Group scope as live-only: local cached reads may still support ordinary `records search`, but Smart Group membership is dynamic DEVONthink state that must come from live local MCP. + +--- + +## Implementation Units + +### U1. Smart Group Resolution Helper + +- **Goal:** Add a read-only resolver that turns the `--smart-group` input into a unique Smart Group UUID and human-readable candidate context. +- **Requirements:** R1, R2, R3, R7 +- **Dependencies:** none +- **Files:** `internal/client/local_devonthink.go`, `internal/client/smart_group_scope.go`, `internal/client/smart_group_scope_test.go` +- **Approach:** Add a small client-side helper that uses existing local MCP read access. Direct UUID input should be accepted and verified as a Smart Group when record properties are available. Name and path input should search Smart Groups, filter exact matches, and return a typed not-found or ambiguous error with candidate `name`, `location`, `databaseName`, and `uuid` fields. +- **Patterns to follow:** Existing `localRecordSearch` MCP call shape in `internal/client/local_devonthink.go`; typed error and exit-code style in `internal/cli/helpers.go`. +- **Test scenarios:** + - Happy path: UUID-shaped input for a Smart Group returns that UUID and records the original input. + - Happy path: exact name input matching one Smart Group returns its UUID. + - Happy path: path input matching `location + name` returns its UUID. + - Edge case: two Smart Groups with the same exact name return an ambiguous error with both candidate UUIDs. + - Error path: a UUID that resolves to a non-Smart Group returns a clear not-Smart-Group error. + - Error path: a name/path with no Smart Group match returns a clear not-found error. +- **Verification:** Resolver tests prove deterministic UUID, name, path, duplicate, and no-match behavior without mutating DEVONthink. + +### U2. Records Search Flag and Search Delegation + +- **Goal:** Wire `--smart-group` into `records search` while preserving the existing query, limit, output, and source-selection behavior. +- **Requirements:** R1, R4, R5, R6, R7, R9 +- **Dependencies:** U1 +- **Files:** `internal/cli/records_search.go`, `internal/cli/records_search_test.go`, `internal/cli/helpers.go`, `internal/cli/root_test.go` +- **Approach:** Add a `--smart-group` string flag. When present, resolve it before `resolveReadWithStrategy`, set the existing search parameter that becomes MCP `group_uuid`, and attach scope metadata to the provenance value. Reject simultaneous `--group` and `--smart-group` as a usage error. Reject `--smart-group` with `--data-source local` because cached data cannot reproduce dynamic Smart Group membership. Keep `--select` applied to results before wrapping so `meta.scope` remains available. +- **Patterns to follow:** Current `records_search.go` output pipeline; `DataProvenance` and `wrapWithProvenance` in `internal/cli/helpers.go`; field projection tests in `internal/cli/root_test.go`. +- **Test scenarios:** + - Happy path: `records search "tags:waiting/rueckerstattung" --smart-group "Offene Rückerstattungen" --agent --select uuid,name,item_link,tags,databaseName` emits JSON with `meta.source`, `meta.scope`, and filtered result rows. + - Happy path: resolved Smart Group UUID is sent to the final search as the existing group scope parameter. + - Edge case: `--select` does not remove `meta.scope`. + - Error path: `--group` and `--smart-group` together exit as usage error. + - Error path: ambiguous Smart Group name under `--agent` produces a machine-readable error envelope with candidate context plus clear stderr. + - Error path: `--data-source local --smart-group ` exits before search with a clear unsupported-scope error. +- **Verification:** Command tests pin the downstream JSON contract and prove the final search still follows the existing read strategy. + +### U3. Agent and MCP Discovery Parity + +- **Goal:** Make the new flag discoverable to agents that inspect the CLI or its local MCP wrapper. +- **Requirements:** R5, R7 +- **Dependencies:** U2 +- **Files:** `internal/cli/agent_context.go`, `internal/mcp/tools.go`, `internal/mcp/tools_test.go`, `tools-manifest.json`, `internal/cli/which.go`, `internal/cli/which_test.go` +- **Approach:** Ensure Cobra flag introspection exposes `--smart-group` in `agent-context`. Update the CLI-provided MCP `records_search` tool registration and manifest if those surfaces do not derive the new flag automatically. Update `which` only if capability discovery cannot otherwise lead agents to `records search`. +- **Patterns to follow:** `records_search` manual registration in `internal/mcp/tools.go`; existing `whichIndex` test coverage in `internal/cli/which_test.go`. +- **Test scenarios:** + - Happy path: `agent-context` lists `smart-group` under `records search`. + - Happy path: the CLI MCP server advertises the Smart Group search scope input for `records_search`. + - Edge case: existing `--group` discovery remains present and unchanged. +- **Verification:** Agent-facing discovery surfaces show the new scope option without changing command names. + +### U4. Documentation and Reprint Guard + +- **Goal:** Document the Smart Group scope contract and preserve the local generated-tree customization intent. +- **Requirements:** R6, R7, R8 +- **Dependencies:** U2, U3 +- **Files:** `README.md`, `SKILL.md`, `.printing-press-patches/2026-06-22-smart-group-search-scope.md` +- **Approach:** Add the downstream contract example to the compact search guidance and state that Smart Groups scope search only. Record the generated-tree patch rationale under `.printing-press-patches/` because the repo is printed output and future reprints can overwrite hand edits. +- **Patterns to follow:** Existing agent recipe style in `README.md` and `SKILL.md`; local customization rule in `AGENTS.md`. +- **Test scenarios:** + - Documentation example uses `--smart-group`, `--agent`, and `--select uuid,name,item_link,tags,databaseName`. + - Documentation states Smart Groups are search scopes only and do not imply maintenance action policy. + - Patch ledger entry names the local generated-code intent and the files/classes of behavior to preserve. +- **Verification:** Docs give downstream tools the exact contract and future maintainers can recover the change during reprint. + +--- + +## Acceptance Examples + +- AE1. Given a unique Smart Group named `Offene Rückerstattungen`, when an agent runs: + + ```bash + devonthink-pp-cli records search "tags:waiting/rueckerstattung" \ + --smart-group "Offene Rückerstattungen" \ + --agent \ + --select uuid,name,item_link,tags,databaseName + ``` + + Then stdout is valid JSON with normal `results` rows and `meta.scope.type == "smart_group"`, `meta.scope.input == "Offene Rückerstattungen"`, and `meta.scope.uuid` set. + +- AE2. Given two Smart Groups with the same exact name, when the user resolves by name, then the command fails before search and reports the ambiguous candidates. +- AE3. Given `--select uuid,name`, when Smart Group scope is active, then result rows are filtered but `meta.scope` remains present. +- AE4. Given an existing Smart Group UUID, when passed to `--smart-group`, then the command scopes search with that UUID without requiring the caller to know the Smart Group path. + +--- + +## Scope Boundaries + +- In scope: resolving Smart Group UUID, exact name, and DEVONthink path for `records search`. +- In scope: preserving read-only CLI and agent JSON behavior. +- Out of scope: changing Smart Group definitions, executing maintenance actions, or making Smart Groups action workflow policies. +- Out of scope: fuzzy Smart Group matching, interactive disambiguation, and new maintenance-plugin commands. + +### Deferred to Follow-Up Work + +- Consider adding Smart Group-aware scopes to `context pack` only after `records search` has a stable resolver contract. +- Consider upstreaming the resolver pattern into Printing Press templates if future printed local-app CLIs need named dynamic scopes. + +--- + +## System-Wide Impact + +This changes an exported CLI contract and agent-facing discovery surfaces. Human users gain a friendlier search scope, while agents gain a deterministic bridge from curated DEVONthink Smart Groups to compact JSON search results. The local-only boundary remains unchanged: the CLI still talks to DEVONthink on this Mac through local automation and the official local MCP bridge. + +--- + +## Risks & Dependencies + +- **MCP availability:** Smart Group name/path resolution depends on the local official DEVONthink MCP server exposing readable record metadata. Mitigate by failing clearly when the resolver cannot read Smart Groups. +- **Duplicate names:** DEVONthink allows same-name Smart Groups across databases or folders. Mitigate by treating name duplicates as errors and asking callers to use a path or UUID. +- **Generated tree drift:** Manual edits may be overwritten by future Printing Press reprints. Mitigate with a `.printing-press-patches/` entry that records the behavioral intent. +- **Local fallback semantics:** The existing local SQLite fallback does not apply endpoint filters. Smart Group scoping should require live local MCP resolution and should not pretend cached data can reproduce Smart Group membership. + +--- + +## Sources & Research + +- `internal/cli/records_search.go` currently builds `/records/search` params, supports `--group`, and wraps JSON output with provenance. +- `internal/client/local_devonthink.go` maps the search `group` parameter to MCP `search_records` `group_uuid`. +- `internal/cli/helpers.go` owns `DataProvenance`, `wrapWithProvenance`, `--select` projection behavior, and typed CLI errors. +- `README.md` and `SKILL.md` already document compact agent search recipes and the response envelope. +- Runtime discovery showed the installed CLI has no `--smart-group` flag yet, while read-only MCP `search_records` can return Smart Groups via `kind:smartgroup`. diff --git a/library/productivity/devonthink/go.mod b/library/productivity/devonthink/go.mod new file mode 100644 index 0000000000..3ae8f47351 --- /dev/null +++ b/library/productivity/devonthink/go.mod @@ -0,0 +1,34 @@ +module github.com/mvanhorn/printing-press-library/library/productivity/devonthink + +go 1.26 + +toolchain go1.26.4 + +require ( + github.com/pelletier/go-toml/v2 v2.2.4 + github.com/spf13/cobra v1.9.1 +) + +require modernc.org/sqlite v1.37.0 + +require ( + github.com/mark3labs/mcp-go v0.47.0 + github.com/spf13/pflag v1.0.6 +) + +require ( + github.com/dustin/go-humanize v1.0.1 // indirect + github.com/google/jsonschema-go v0.4.2 // indirect + github.com/google/uuid v1.6.0 // indirect + github.com/inconshreveable/mousetrap v1.1.0 // indirect + github.com/mattn/go-isatty v0.0.20 // indirect + github.com/ncruces/go-strftime v0.1.9 // indirect + github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec // indirect + github.com/spf13/cast v1.7.1 // indirect + github.com/yosida95/uritemplate/v3 v3.0.2 // indirect + golang.org/x/exp v0.0.0-20250305212735-054e65f0b394 // indirect + golang.org/x/sys v0.31.0 // indirect + modernc.org/libc v1.62.1 // indirect + modernc.org/mathutil v1.7.1 // indirect + modernc.org/memory v1.9.1 // indirect +) diff --git a/library/productivity/devonthink/go.sum b/library/productivity/devonthink/go.sum new file mode 100644 index 0000000000..5d8eb6ef91 --- /dev/null +++ b/library/productivity/devonthink/go.sum @@ -0,0 +1,84 @@ +github.com/cpuguy83/go-md2man/v2 v2.0.6/go.mod h1:oOW0eioCTA6cOiMLiUPZOpcVxMig6NIQQ7OS05n1F4g= +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY= +github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto= +github.com/frankban/quicktest v1.14.6 h1:7Xjx+VpznH+oBnejlPUj8oUpdxnVs4f8XU8WnHkI4W8= +github.com/frankban/quicktest v1.14.6/go.mod h1:4ptaffx2x8+WTWXmUCuVU6aPUX1/Mz7zb5vbUoiM6w0= +github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= +github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU= +github.com/google/jsonschema-go v0.4.2 h1:tmrUohrwoLZZS/P3x7ex0WAVknEkBZM46iALbcqoRA8= +github.com/google/jsonschema-go v0.4.2/go.mod h1:r5quNTdLOYEz95Ru18zA0ydNbBuYoo9tgaYcxEYhJVE= +github.com/google/pprof v0.0.0-20250317173921-a4b03ec1a45e h1:ijClszYn+mADRFY17kjQEVQ1XRhq2/JR1M3sGqeJoxs= +github.com/google/pprof v0.0.0-20250317173921-a4b03ec1a45e/go.mod h1:boTsfXsheKC2y+lKOCMpSfarhxDeIzfZG1jqGcPl3cA= +github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= +github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= +github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= +github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= +github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= +github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= +github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= +github.com/mark3labs/mcp-go v0.47.0 h1:h44yeM3DduDyQgzImYWu4pt6VRkqP/0p/95AGhWngnA= +github.com/mark3labs/mcp-go v0.47.0/go.mod h1:JKTC7R2LLVagkEWK7Kwu7DbmA6iIvnNAod6yrHiQMag= +github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY= +github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= +github.com/ncruces/go-strftime v0.1.9 h1:bY0MQC28UADQmHmaF5dgpLmImcShSi2kHU9XLdhx/f4= +github.com/ncruces/go-strftime v0.1.9/go.mod h1:Fwc5htZGVVkseilnfgOVb9mKy6w1naJmn9CehxcKcls= +github.com/pelletier/go-toml/v2 v2.2.4 h1:mye9XuhQ6gvn5h28+VilKrrPoQVanw5PMw/TB0t5Ec4= +github.com/pelletier/go-toml/v2 v2.2.4/go.mod h1:2gIqNv+qfxSVS7cM2xJQKtLSTLUE9V8t9Stt+h56mCY= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec h1:W09IVJc94icq4NjY3clb7Lk8O1qJ8BdBEF8z0ibU0rE= +github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo= +github.com/rogpeppe/go-internal v1.9.0 h1:73kH8U+JUqXU8lRuOHeVHaa/SZPifC7BkcraZVejAe8= +github.com/rogpeppe/go-internal v1.9.0/go.mod h1:WtVeX8xhTBvf0smdhujwtBcq4Qrzq/fJaraNFVN+nFs= +github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= +github.com/spf13/cast v1.7.1 h1:cuNEagBQEHWN1FnbGEjCXL2szYEXqfJPbP2HNUaca9Y= +github.com/spf13/cast v1.7.1/go.mod h1:ancEpBxwJDODSW/UG4rDrAqiKolqNNh2DX3mk86cAdo= +github.com/spf13/cobra v1.9.1 h1:CXSaggrXdbHK9CF+8ywj8Amf7PBRmPCOJugH954Nnlo= +github.com/spf13/cobra v1.9.1/go.mod h1:nDyEzZ8ogv936Cinf6g1RU9MRY64Ir93oCnqb9wxYW0= +github.com/spf13/pflag v1.0.6 h1:jFzHGLGAlb3ruxLB8MhbI6A8+AQX/2eW4qeyNZXNp2o= +github.com/spf13/pflag v1.0.6/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= +github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg= +github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= +github.com/yosida95/uritemplate/v3 v3.0.2 h1:Ed3Oyj9yrmi9087+NczuL5BwkIc4wvTb5zIM+UJPGz4= +github.com/yosida95/uritemplate/v3 v3.0.2/go.mod h1:ILOh0sOhIJR3+L/8afwt/kE++YT040gmv5BQTMR2HP4= +golang.org/x/exp v0.0.0-20250305212735-054e65f0b394 h1:nDVHiLt8aIbd/VzvPWN6kSOPE7+F/fNFDSXLVYkE/Iw= +golang.org/x/exp v0.0.0-20250305212735-054e65f0b394/go.mod h1:sIifuuw/Yco/y6yb6+bDNfyeQ/MdPUy/hKEMYQV17cM= +golang.org/x/mod v0.24.0 h1:ZfthKaKaT4NrhGVZHO1/WDTwGES4De8KtWO0SIbNJMU= +golang.org/x/mod v0.24.0/go.mod h1:IXM97Txy2VM4PJ3gI61r1YEk/gAj6zAHN3AdZt6S9Ww= +golang.org/x/sync v0.12.0 h1:MHc5BpPuC30uJk597Ri8TV3CNZcTLu6B6z4lJy+g6Jw= +golang.org/x/sync v0.12.0/go.mod h1:1dzgHSNfp02xaA81J2MS99Qcpr2w7fw1gpm99rleRqA= +golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.31.0 h1:ioabZlmFYtWhL+TRYpcnNlLwhyxaM9kWTDEmfnprqik= +golang.org/x/sys v0.31.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k= +golang.org/x/tools v0.31.0 h1:0EedkvKDbh+qistFTd0Bcwe/YLh4vHwWEkiI0toFIBU= +golang.org/x/tools v0.31.0/go.mod h1:naFTU+Cev749tSJRXJlna0T3WxKvb1kWEx15xA4SdmQ= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +modernc.org/cc/v4 v4.25.2 h1:T2oH7sZdGvTaie0BRNFbIYsabzCxUQg8nLqCdQ2i0ic= +modernc.org/cc/v4 v4.25.2/go.mod h1:uVtb5OGqUKpoLWhqwNQo/8LwvoiEBLvZXIQ/SmO6mL0= +modernc.org/ccgo/v4 v4.25.1 h1:TFSzPrAGmDsdnhT9X2UrcPMI3N/mJ9/X9ykKXwLhDsU= +modernc.org/ccgo/v4 v4.25.1/go.mod h1:njjuAYiPflywOOrm3B7kCB444ONP5pAVr8PIEoE0uDw= +modernc.org/fileutil v1.3.0 h1:gQ5SIzK3H9kdfai/5x41oQiKValumqNTDXMvKo62HvE= +modernc.org/fileutil v1.3.0/go.mod h1:XatxS8fZi3pS8/hKG2GH/ArUogfxjpEKs3Ku3aK4JyQ= +modernc.org/gc/v2 v2.6.5 h1:nyqdV8q46KvTpZlsw66kWqwXRHdjIlJOhG6kxiV/9xI= +modernc.org/gc/v2 v2.6.5/go.mod h1:YgIahr1ypgfe7chRuJi2gD7DBQiKSLMPgBQe9oIiito= +modernc.org/libc v1.62.1 h1:s0+fv5E3FymN8eJVmnk0llBe6rOxCu/DEU+XygRbS8s= +modernc.org/libc v1.62.1/go.mod h1:iXhATfJQLjG3NWy56a6WVU73lWOcdYVxsvwCgoPljuo= +modernc.org/mathutil v1.7.1 h1:GCZVGXdaN8gTqB1Mf/usp1Y/hSqgI2vAGGP4jZMCxOU= +modernc.org/mathutil v1.7.1/go.mod h1:4p5IwJITfppl0G4sUEDtCr4DthTaT47/N3aT6MhfgJg= +modernc.org/memory v1.9.1 h1:V/Z1solwAVmMW1yttq3nDdZPJqV1rM05Ccq6KMSZ34g= +modernc.org/memory v1.9.1/go.mod h1:/JP4VbVC+K5sU2wZi9bHoq2MAkCnrt2r98UGeSK7Mjw= +modernc.org/opt v0.1.4 h1:2kNGMRiUjrp4LcaPuLY2PzUfqM/w9N23quVwhKt5Qm8= +modernc.org/opt v0.1.4/go.mod h1:03fq9lsNfvkYSfxrfUhZCWPk1lm4cq4N+Bh//bEtgns= +modernc.org/sortutil v1.2.1 h1:+xyoGf15mM3NMlPDnFqrteY07klSFxLElE2PVuWIJ7w= +modernc.org/sortutil v1.2.1/go.mod h1:7ZI3a3REbai7gzCLcotuw9AC4VZVpYMjDzETGsSMqJE= +modernc.org/sqlite v1.37.0 h1:s1TMe7T3Q3ovQiK2Ouz4Jwh7dw4ZDqbebSDTlSJdfjI= +modernc.org/sqlite v1.37.0/go.mod h1:5YiWv+YviqGMuGw4V+PNplcyaJ5v+vQd7TQOgkACoJM= +modernc.org/strutil v1.2.1 h1:UneZBkQA+DX2Rp35KcM69cSsNES9ly8mQWD71HKlOA0= +modernc.org/strutil v1.2.1/go.mod h1:EHkiggD70koQxjVdSBM3JKM7k6L0FbGE5eymy9i3B9A= +modernc.org/token v1.1.0 h1:Xl7Ap9dKaEs5kLoOQeQmPWevfnk/DM5qcLcYlA8ys6Y= +modernc.org/token v1.1.0/go.mod h1:UGzOrNV1mAFSEB63lOFHIpNRUVMvYTc6yu1SMY/XTDM= diff --git a/library/productivity/devonthink/internal/cache/cache.go b/library/productivity/devonthink/internal/cache/cache.go new file mode 100644 index 0000000000..e4cacb6d5a --- /dev/null +++ b/library/productivity/devonthink/internal/cache/cache.go @@ -0,0 +1,59 @@ +// Copyright 2026 rowdy and contributors. Licensed under Apache-2.0. See LICENSE. +// Generated by CLI Printing Press (https://github.com/mvanhorn/cli-printing-press). DO NOT EDIT. + +// Package cache provides a file-based response cache with optional embedded database backend. +// The default implementation stores JSON responses as flat files in ~/.cache// with a TTL. +// For higher-throughput or concurrent-write scenarios, replace the file backend with an +// embedded database such as bolt (go.etcd.io/bbolt), badger (github.com/dgraph-io/badger), +// or sqlite (modernc.org/sqlite). +package cache + +import ( + "crypto/sha256" + "encoding/hex" + "encoding/json" + "os" + "path/filepath" + "time" +) + +// Store is a key-value cache backed by the filesystem. +type Store struct { + Dir string + TTL time.Duration +} + +// New creates a file-based cache store. +func New(dir string, ttl time.Duration) *Store { + return &Store{Dir: dir, TTL: ttl} +} + +// Get retrieves a cached value. Returns nil if not found or expired. +func (s *Store) Get(key string) (json.RawMessage, bool) { + path := s.path(key) + info, err := os.Stat(path) + if err != nil || time.Since(info.ModTime()) > s.TTL { + return nil, false + } + data, err := os.ReadFile(path) // #nosec G304 -- path is derived from the cache store root and cache key. + if err != nil { + return nil, false + } + return json.RawMessage(data), true +} + +// Set stores a value in the cache. +func (s *Store) Set(key string, value json.RawMessage) { + _ = os.MkdirAll(s.Dir, 0o700) + _ = os.WriteFile(s.path(key), []byte(value), 0o600) +} + +// Clear removes all cached entries. +func (s *Store) Clear() error { + return os.RemoveAll(s.Dir) +} + +func (s *Store) path(key string) string { + h := sha256.Sum256([]byte(key)) + return filepath.Join(s.Dir, hex.EncodeToString(h[:8])+".json") +} diff --git a/library/productivity/devonthink/internal/cli/agent_context.go b/library/productivity/devonthink/internal/cli/agent_context.go new file mode 100644 index 0000000000..04e7a8a1dc --- /dev/null +++ b/library/productivity/devonthink/internal/cli/agent_context.go @@ -0,0 +1,194 @@ +// Copyright 2026 rowdy and contributors. Licensed under Apache-2.0. See LICENSE. +// Generated by CLI Printing Press (https://github.com/mvanhorn/cli-printing-press). DO NOT EDIT. + +package cli + +import ( + "encoding/json" + "os" + "sort" + + "github.com/spf13/cobra" + "github.com/spf13/pflag" +) + +// agentContextSchemaVersion is bumped on any breaking change to the JSON +// shape emitted by `agent-context`. Agents should check this before +// parsing. Shape at v3 adds kind-aware auth env var metadata. +const agentContextSchemaVersion = "3" + +// agentContext is the structured description of this CLI consumed by AI +// agents. Inspired by Cloudflare's /cdn-cgi/explorer/api runtime endpoint +// (2026-04-13 Wrangler post): agents can introspect the live CLI without +// parsing --help or reading source. +type agentContext struct { + SchemaVersion string `json:"schema_version"` + CLI agentContextCLI `json:"cli"` + Auth agentContextAuth `json:"auth"` + Discovery *agentContextDiscovery `json:"discovery,omitempty"` + Commands []agentContextCommand `json:"commands"` + AvailableProfiles []string `json:"available_profiles"` + FeedbackEndpointConfigured bool `json:"feedback_endpoint_configured"` +} + +type agentContextCLI struct { + Name string `json:"name"` + Description string `json:"description"` + Version string `json:"version"` + LocalOnly bool `json:"local_only"` +} + +type agentContextAuth struct { + Mode string `json:"mode"` + EnvVars []agentContextAuthEnvVar `json:"env_vars"` +} + +type agentContextAuthEnvVar struct { + Name string `json:"name"` + Kind string `json:"kind"` + Required bool `json:"required"` + Sensitive bool `json:"sensitive"` + Description string `json:"description,omitempty"` +} + +type agentContextDiscovery struct { + Source string `json:"source"` + TargetURL string `json:"target_url,omitempty"` + EntryCount int `json:"entry_count,omitempty"` + APIEntryCount int `json:"api_entry_count,omitempty"` + Reachability string `json:"reachability,omitempty"` + Protocols []string `json:"protocols,omitempty"` + AuthCandidates []string `json:"auth_candidates,omitempty"` + Protections []string `json:"protections,omitempty"` + GenerationHints []string `json:"generation_hints,omitempty"` + Warnings []string `json:"warnings,omitempty"` + CandidateCommands []string `json:"candidate_commands,omitempty"` +} + +type agentContextCommand struct { + Name string `json:"name"` + Use string `json:"use,omitempty"` + Short string `json:"short,omitempty"` + Annotations map[string]string `json:"annotations,omitempty"` + Flags []agentContextFlag `json:"flags,omitempty"` + Subcommands []agentContextCommand `json:"subcommands,omitempty"` +} + +type agentContextFlag struct { + Name string `json:"name"` + Type string `json:"type"` + Usage string `json:"usage,omitempty"` + Default string `json:"default,omitempty"` +} + +func newAgentContextCmd(rootCmd *cobra.Command) *cobra.Command { + var localOnly bool + var pretty bool + cmd := &cobra.Command{ + Use: "agent-context", + Short: "Emit structured JSON describing this CLI for agents", + Annotations: map[string]string{"mcp:read-only": "true"}, + Long: `Outputs a machine-readable description of commands, flags, and auth so +agents can introspect this CLI at runtime without parsing --help or +reading source. Schema is versioned via schema_version.`, + RunE: func(cmd *cobra.Command, args []string) error { + ctx := buildAgentContext(rootCmd) + ctx.CLI.LocalOnly = localOnly || ctx.CLI.LocalOnly + enc := json.NewEncoder(os.Stdout) + if pretty { + enc.SetIndent("", " ") + } + return enc.Encode(ctx) + }, + } + cmd.Flags().BoolVar(&localOnly, "local-only", false, "assert that DEVONthink access must stay on this Mac or the user's own LAN") + cmd.Flags().BoolVar(&pretty, "pretty", false, "indent JSON output for human reading") + return cmd +} + +func buildAgentContext(rootCmd *cobra.Command) agentContext { + envVars := []agentContextAuthEnvVar{} + authMode := "none" + if authMode == "" { + authMode = "none" + } + profiles := ListProfileNames() + if profiles == nil { + profiles = []string{} + } + return agentContext{ + SchemaVersion: agentContextSchemaVersion, + CLI: agentContextCLI{ + Name: "devonthink-pp-cli", + Description: "Local-first DEVONthink automation with safer shell workflows than raw AppleScript or MCP alone.", + Version: rootCmd.Version, + LocalOnly: true, + }, + Auth: agentContextAuth{ + Mode: authMode, + EnvVars: envVars, + }, + Discovery: buildAgentDiscoveryContext(), + Commands: collectAgentCommands(rootCmd), + AvailableProfiles: profiles, + FeedbackEndpointConfigured: FeedbackEndpointConfigured(), + } +} + +func buildAgentDiscoveryContext() *agentContextDiscovery { + return nil +} + +// collectAgentCommands walks the cobra tree from the given command and +// returns its direct children (skipping the agent-context command itself +// to avoid self-reference). Each child is recursed into if it has +// subcommands. Flags are captured via VisitAll. Output is sorted by +// command name for stable diffs across regenerations. +// +// Cobra's Hidden flag suppresses listing in --help but does not gate +// agent discovery. Raw resource parents are Hidden so --help stays +// curated and the `api` browser populates; the agent-context surface +// must still enumerate them and their endpoints so agents can call any +// action a CLI user could. +func collectAgentCommands(c *cobra.Command) []agentContextCommand { + children := c.Commands() + sort.Slice(children, func(i, j int) bool { return children[i].Name() < children[j].Name() }) + + out := make([]agentContextCommand, 0, len(children)) + for _, sub := range children { + if sub.Name() == "agent-context" { + continue + } + entry := agentContextCommand{ + Name: sub.Name(), + Use: sub.Use, + Short: sub.Short, + } + // Surface Cobra annotations (e.g., pp:endpoint, mcp:read-only) so + // agents and the live-dogfood classifier can detect destructive-at-auth + // endpoints without parsing source. Empty maps are stripped via + // omitempty in the struct tag. + if len(sub.Annotations) > 0 { + entry.Annotations = make(map[string]string, len(sub.Annotations)) + for k, v := range sub.Annotations { + entry.Annotations[k] = v + } + } + sub.Flags().VisitAll(func(f *pflag.Flag) { + entry.Flags = append(entry.Flags, agentContextFlag{ + Name: f.Name, + Type: f.Value.Type(), + Usage: f.Usage, + Default: f.DefValue, + }) + }) + sort.Slice(entry.Flags, func(i, j int) bool { + return entry.Flags[i].Name < entry.Flags[j].Name + }) + if len(sub.Commands()) > 0 { + entry.Subcommands = collectAgentCommands(sub) + } + out = append(out, entry) + } + return out +} diff --git a/library/productivity/devonthink/internal/cli/ai.go b/library/productivity/devonthink/internal/cli/ai.go new file mode 100644 index 0000000000..b7452dc8ea --- /dev/null +++ b/library/productivity/devonthink/internal/cli/ai.go @@ -0,0 +1,22 @@ +// Copyright 2026 rowdy and contributors. Licensed under Apache-2.0. See LICENSE. +// Generated by CLI Printing Press (https://github.com/mvanhorn/cli-printing-press). DO NOT EDIT. + +package cli + +import ( + "github.com/spf13/cobra" +) + +func newAiCmd(flags *rootFlags) *cobra.Command { + cmd := &cobra.Command{ + Use: "ai", + Short: "DEVONthink AI and summary helpers", + Hidden: true, + Annotations: map[string]string{"mcp:read-only": "true"}, + RunE: parentNoSubcommandRunE(flags), + } + + cmd.AddCommand(newAiAskCmd(flags)) + cmd.AddCommand(newAiSummarizeCmd(flags)) + return cmd +} diff --git a/library/productivity/devonthink/internal/cli/ai_ask.go b/library/productivity/devonthink/internal/cli/ai_ask.go new file mode 100644 index 0000000000..d0fe1870c8 --- /dev/null +++ b/library/productivity/devonthink/internal/cli/ai_ask.go @@ -0,0 +1,205 @@ +// Copyright 2026 rowdy and contributors. Licensed under Apache-2.0. See LICENSE. +// Generated by CLI Printing Press (https://github.com/mvanhorn/cli-printing-press). DO NOT EDIT. + +package cli + +import ( + "encoding/json" + "fmt" + "io" + "os" + + "github.com/spf13/cobra" +) + +func newAiAskCmd(flags *rootFlags) *cobra.Command { + var bodyUuid string + var bodySelection bool + var bodyDryRun bool + var stdinBody bool + + cmd := &cobra.Command{ + Use: "ask ", + Short: "Ask DEVONthink AI about selected local records with explicit cloud-use warnings", + Example: " devonthink-pp-cli ai ask \"Summarize the selected records\" --selection --dry-run", + Annotations: map[string]string{"pp:endpoint": "ai.ask", "pp:method": "POST", "pp:path": "/ai/ask"}, + RunE: func(cmd *cobra.Command, args []string) error { + if len(args) == 0 { + return cmd.Help() + } + if !stdinBody { + } + c, err := flags.newClient() + if err != nil { + return err + } + + path := "/ai/ask" + params := map[string]string{} + params["prompt"] = args[0] + var body map[string]any + if stdinBody { + stdinData, err := io.ReadAll(os.Stdin) + if err != nil { + return fmt.Errorf("reading stdin: %w", err) + } + var jsonBody map[string]any + if err := json.Unmarshal(stdinData, &jsonBody); err != nil { + return fmt.Errorf("parsing stdin JSON: %w", err) + } + body = jsonBody + } else { + body = map[string]any{} + if bodyUuid != "" { + body["uuid"] = bodyUuid + } + if cmd.Flags().Changed("selection") { + body["selection"] = bodySelection + } + if cmd.Flags().Changed("dry-run") { + body["dry_run"] = bodyDryRun + } + } + data, statusCode, err := c.PostWithParams(cmd.Context(), path, params, body) + if err != nil { + return classifyAPIError(err, flags) + } + // Inspect the mutate response body for a partial-failure-shaped + // field (e.g. Google Ads `partialFailureError`). Several Google + // APIs return 200 OK with a partial-failure field when some + // operations in the batch failed; ignoring it silently swallows + // real failures. Detection runs before output-mode selection so + // the exit code is consistent regardless of how stdout is + // rendered. --dry-run short-circuits because no real request + // was sent. + var partialFailure *partialFailureReport + if !flags.dryRun && statusCode >= 200 && statusCode < 300 { + partialFailure = detectPartialFailure(data) + if partialFailure != nil { + fmt.Fprintf(os.Stderr, "warning: partial failure detected in %s response: %s\n", "ai", partialFailure.Message) + if len(partialFailure.ResourceNames) > 0 { + fmt.Fprintf(os.Stderr, " succeeded: %d operation(s)\n", len(partialFailure.ResourceNames)) + } + } + } + if !flags.dryRun && statusCode >= 200 && statusCode < 300 && (partialFailure == nil || flags.allowPartialFailure) { + writeMutationResponseToStore(cmd.Context(), "ai", data, "") + } + if wantsHumanTable(cmd.OutOrStdout(), flags) { + // Check if response contains an array (directly or wrapped in "data") + var items []map[string]any + if json.Unmarshal(data, &items) == nil && len(items) > 0 { + if err := printAutoTable(cmd.OutOrStdout(), items); err != nil { + fmt.Fprintf(os.Stderr, "warning: table rendering failed, falling back to JSON: %v\n", err) + } else { + if partialFailure != nil && !flags.allowPartialFailure { + return partialFailureErr(fmt.Errorf("partial failure in %s response: %s", "ai", partialFailure.Message)) + } + return nil + } + } else { + var wrapped struct { + Data []map[string]any `json:"data"` + } + if json.Unmarshal(data, &wrapped) == nil && len(wrapped.Data) > 0 { + if err := printAutoTable(cmd.OutOrStdout(), wrapped.Data); err != nil { + fmt.Fprintf(os.Stderr, "warning: table rendering failed, falling back to JSON: %v\n", err) + } else { + if partialFailure != nil && !flags.allowPartialFailure { + return partialFailureErr(fmt.Errorf("partial failure in %s response: %s", "ai", partialFailure.Message)) + } + return nil + } + } + } + } + if flags.asJSON || (!isTerminal(cmd.OutOrStdout()) && !flags.csv && !flags.quiet && !flags.plain) { + if flags.quiet { + if partialFailure != nil && !flags.allowPartialFailure { + return partialFailureErr(fmt.Errorf("partial failure in %s response: %s", "ai", partialFailure.Message)) + } + return nil + } + envelope := map[string]any{ + "action": "post", + "resource": "ai", + "path": path, + "status": statusCode, + "success": statusCode >= 200 && statusCode < 300 && (partialFailure == nil || flags.allowPartialFailure), + } + if partialFailure != nil { + envelope["partial_failure"] = partialFailure + } + if flags.dryRun { + envelope["dry_run"] = true + envelope["status"] = 0 + envelope["success"] = false + } + // Verify-mode synthetic envelope detection runs against RAW data + // (before --compact/--select filtering) so the sentinel field is + // guaranteed to be visible even if the operator passes a filter + // flag that would otherwise strip it. Surfaces a top-level + // verify_noop signal + flips success to false. Mirrors the dry_run + // shape above. + if len(data) > 0 { + var rawParsed any + if err := json.Unmarshal(data, &rawParsed); err == nil { + if m, ok := rawParsed.(map[string]any); ok { + if v, ok := m["__pp_verify_synthetic__"].(bool); ok && v { + envelope["verify_noop"] = true + envelope["success"] = false + } + } + } + } + // Apply --compact and --select to the API response before wrapping. + // --select wins when both are set: explicit field choice trumps the + // generic high-gravity allow-list. Otherwise --compact still applies + // when --agent is on but the user did not name fields. + filtered := data + if flags.selectFields != "" { + filtered = filterFields(filtered, flags.selectFields) + } else if flags.compact { + filtered = compactFields(filtered) + } + if len(filtered) > 0 { + var parsed any + if err := json.Unmarshal(filtered, &parsed); err == nil { + envelope["data"] = parsed + } + } + envelopeJSON, err := json.Marshal(envelope) + if err != nil { + return err + } + if perr := printOutput(cmd.OutOrStdout(), json.RawMessage(envelopeJSON), true); perr != nil { + return perr + } + if partialFailure != nil && !flags.allowPartialFailure { + return partialFailureErr(fmt.Errorf("partial failure in %s response: %s", "ai", partialFailure.Message)) + } + return nil + } + // Fall-through for mutate paths that did not hit the table or + // asJSON branches: --quiet, --csv, --plain, and default terminal + // raw output. printOutputWithFlags renders the body, then the + // typed partial-failure exit fires unless --allow-partial-failure + // downgrades it. Without this guard a partial failure would exit + // 0 for these output modes — the exact silent-swallow regression + // the surrounding patch is preventing for asJSON / piped output. + if perr := printOutputWithFlags(cmd.OutOrStdout(), data, flags); perr != nil { + return perr + } + if partialFailure != nil && !flags.allowPartialFailure { + return partialFailureErr(fmt.Errorf("partial failure in %s response: %s", "ai", partialFailure.Message)) + } + return nil + }, + } + cmd.Flags().StringVar(&bodyUuid, "uuid", "", "Record UUID to attach") + cmd.Flags().BoolVar(&bodySelection, "selection", false, "Attach the current selection") + cmd.Flags().BoolVar(&bodyDryRun, "dry-run", false, "Preview attachments without sending") + cmd.Flags().BoolVar(&stdinBody, "stdin", false, "Read request body as JSON from stdin") + + return cmd +} diff --git a/library/productivity/devonthink/internal/cli/ai_summarize.go b/library/productivity/devonthink/internal/cli/ai_summarize.go new file mode 100644 index 0000000000..b4f14f85de --- /dev/null +++ b/library/productivity/devonthink/internal/cli/ai_summarize.go @@ -0,0 +1,200 @@ +// Copyright 2026 rowdy and contributors. Licensed under Apache-2.0. See LICENSE. +// Generated by CLI Printing Press (https://github.com/mvanhorn/cli-printing-press). DO NOT EDIT. + +package cli + +import ( + "encoding/json" + "fmt" + "io" + "os" + + "github.com/spf13/cobra" +) + +func newAiSummarizeCmd(flags *rootFlags) *cobra.Command { + var bodyHighlights bool + var bodyDryRun bool + var stdinBody bool + + cmd := &cobra.Command{ + Use: "summarize ", + Short: "Summarize records or highlights", + Example: " devonthink-pp-cli ai summarize 550e8400-e29b-41d4-a716-446655440000", + Annotations: map[string]string{"pp:endpoint": "ai.summarize", "pp:method": "POST", "pp:path": "/ai/summarize"}, + RunE: func(cmd *cobra.Command, args []string) error { + if len(args) == 0 { + return cmd.Help() + } + if !stdinBody { + } + c, err := flags.newClient() + if err != nil { + return err + } + + path := "/ai/summarize" + params := map[string]string{} + params["uuid"] = args[0] + var body map[string]any + if stdinBody { + stdinData, err := io.ReadAll(os.Stdin) + if err != nil { + return fmt.Errorf("reading stdin: %w", err) + } + var jsonBody map[string]any + if err := json.Unmarshal(stdinData, &jsonBody); err != nil { + return fmt.Errorf("parsing stdin JSON: %w", err) + } + body = jsonBody + } else { + body = map[string]any{} + if cmd.Flags().Changed("highlights") { + body["highlights"] = bodyHighlights + } + if cmd.Flags().Changed("dry-run") { + body["dry_run"] = bodyDryRun + } + } + data, statusCode, err := c.PostWithParams(cmd.Context(), path, params, body) + if err != nil { + return classifyAPIError(err, flags) + } + // Inspect the mutate response body for a partial-failure-shaped + // field (e.g. Google Ads `partialFailureError`). Several Google + // APIs return 200 OK with a partial-failure field when some + // operations in the batch failed; ignoring it silently swallows + // real failures. Detection runs before output-mode selection so + // the exit code is consistent regardless of how stdout is + // rendered. --dry-run short-circuits because no real request + // was sent. + var partialFailure *partialFailureReport + if !flags.dryRun && statusCode >= 200 && statusCode < 300 { + partialFailure = detectPartialFailure(data) + if partialFailure != nil { + fmt.Fprintf(os.Stderr, "warning: partial failure detected in %s response: %s\n", "ai", partialFailure.Message) + if len(partialFailure.ResourceNames) > 0 { + fmt.Fprintf(os.Stderr, " succeeded: %d operation(s)\n", len(partialFailure.ResourceNames)) + } + } + } + if !flags.dryRun && statusCode >= 200 && statusCode < 300 && (partialFailure == nil || flags.allowPartialFailure) { + writeMutationResponseToStore(cmd.Context(), "ai", data, "") + } + if wantsHumanTable(cmd.OutOrStdout(), flags) { + // Check if response contains an array (directly or wrapped in "data") + var items []map[string]any + if json.Unmarshal(data, &items) == nil && len(items) > 0 { + if err := printAutoTable(cmd.OutOrStdout(), items); err != nil { + fmt.Fprintf(os.Stderr, "warning: table rendering failed, falling back to JSON: %v\n", err) + } else { + if partialFailure != nil && !flags.allowPartialFailure { + return partialFailureErr(fmt.Errorf("partial failure in %s response: %s", "ai", partialFailure.Message)) + } + return nil + } + } else { + var wrapped struct { + Data []map[string]any `json:"data"` + } + if json.Unmarshal(data, &wrapped) == nil && len(wrapped.Data) > 0 { + if err := printAutoTable(cmd.OutOrStdout(), wrapped.Data); err != nil { + fmt.Fprintf(os.Stderr, "warning: table rendering failed, falling back to JSON: %v\n", err) + } else { + if partialFailure != nil && !flags.allowPartialFailure { + return partialFailureErr(fmt.Errorf("partial failure in %s response: %s", "ai", partialFailure.Message)) + } + return nil + } + } + } + } + if flags.asJSON || (!isTerminal(cmd.OutOrStdout()) && !flags.csv && !flags.quiet && !flags.plain) { + if flags.quiet { + if partialFailure != nil && !flags.allowPartialFailure { + return partialFailureErr(fmt.Errorf("partial failure in %s response: %s", "ai", partialFailure.Message)) + } + return nil + } + envelope := map[string]any{ + "action": "post", + "resource": "ai", + "path": path, + "status": statusCode, + "success": statusCode >= 200 && statusCode < 300 && (partialFailure == nil || flags.allowPartialFailure), + } + if partialFailure != nil { + envelope["partial_failure"] = partialFailure + } + if flags.dryRun { + envelope["dry_run"] = true + envelope["status"] = 0 + envelope["success"] = false + } + // Verify-mode synthetic envelope detection runs against RAW data + // (before --compact/--select filtering) so the sentinel field is + // guaranteed to be visible even if the operator passes a filter + // flag that would otherwise strip it. Surfaces a top-level + // verify_noop signal + flips success to false. Mirrors the dry_run + // shape above. + if len(data) > 0 { + var rawParsed any + if err := json.Unmarshal(data, &rawParsed); err == nil { + if m, ok := rawParsed.(map[string]any); ok { + if v, ok := m["__pp_verify_synthetic__"].(bool); ok && v { + envelope["verify_noop"] = true + envelope["success"] = false + } + } + } + } + // Apply --compact and --select to the API response before wrapping. + // --select wins when both are set: explicit field choice trumps the + // generic high-gravity allow-list. Otherwise --compact still applies + // when --agent is on but the user did not name fields. + filtered := data + if flags.selectFields != "" { + filtered = filterFields(filtered, flags.selectFields) + } else if flags.compact { + filtered = compactFields(filtered) + } + if len(filtered) > 0 { + var parsed any + if err := json.Unmarshal(filtered, &parsed); err == nil { + envelope["data"] = parsed + } + } + envelopeJSON, err := json.Marshal(envelope) + if err != nil { + return err + } + if perr := printOutput(cmd.OutOrStdout(), json.RawMessage(envelopeJSON), true); perr != nil { + return perr + } + if partialFailure != nil && !flags.allowPartialFailure { + return partialFailureErr(fmt.Errorf("partial failure in %s response: %s", "ai", partialFailure.Message)) + } + return nil + } + // Fall-through for mutate paths that did not hit the table or + // asJSON branches: --quiet, --csv, --plain, and default terminal + // raw output. printOutputWithFlags renders the body, then the + // typed partial-failure exit fires unless --allow-partial-failure + // downgrades it. Without this guard a partial failure would exit + // 0 for these output modes — the exact silent-swallow regression + // the surrounding patch is preventing for asJSON / piped output. + if perr := printOutputWithFlags(cmd.OutOrStdout(), data, flags); perr != nil { + return perr + } + if partialFailure != nil && !flags.allowPartialFailure { + return partialFailureErr(fmt.Errorf("partial failure in %s response: %s", "ai", partialFailure.Message)) + } + return nil + }, + } + cmd.Flags().BoolVar(&bodyHighlights, "highlights", false, "Summarize highlights only") + cmd.Flags().BoolVar(&bodyDryRun, "dry-run", false, "Preview attachments without sending") + cmd.Flags().BoolVar(&stdinBody, "stdin", false, "Read request body as JSON from stdin") + + return cmd +} diff --git a/library/productivity/devonthink/internal/cli/analytics.go b/library/productivity/devonthink/internal/cli/analytics.go new file mode 100644 index 0000000000..0a4b3f5331 --- /dev/null +++ b/library/productivity/devonthink/internal/cli/analytics.go @@ -0,0 +1,197 @@ +// Copyright 2026 rowdy and contributors. Licensed under Apache-2.0. See LICENSE. +// Generated by CLI Printing Press (https://github.com/mvanhorn/cli-printing-press). DO NOT EDIT. + +package cli + +import ( + "encoding/json" + "fmt" + "io" + "sort" + "strings" + + "github.com/mvanhorn/printing-press-library/library/productivity/devonthink/internal/store" + "github.com/spf13/cobra" +) + +func newAnalyticsCmd(flags *rootFlags) *cobra.Command { + var resourceType string + var groupBy string + var dbPath string + var limit int + + cmd := &cobra.Command{ + Use: "analytics", + Short: "Run analytics queries on locally synced data", + Annotations: map[string]string{"mcp:read-only": "true"}, + Long: `Analyze locally synced data with count, group-by, and summary operations. +Data must be synced first with the sync command.`, + Example: ` # Count records by type + devonthink-pp-cli analytics --type messages + + # Group by a field + devonthink-pp-cli analytics --type messages --group-by author_id + + # Top 10 most frequent values + devonthink-pp-cli analytics --type messages --group-by channel_id --limit 10 --json`, + RunE: func(cmd *cobra.Command, args []string) error { + out := cmd.OutOrStdout() + if dbPath == "" { + dbPath = defaultDBPath("devonthink-pp-cli") + } + + db, err := store.OpenWithContext(cmd.Context(), dbPath) + if err != nil { + return fmt.Errorf("opening local database: %w\nRun 'devonthink-pp-cli sync' first.", err) + } + defer db.Close() + + maybeEmitSyncHints(cmd, db, resourceType, flags.maxAge) + + if resourceType == "" { + // Show summary of all resource types + status, err := db.Status() + if err != nil { + return fmt.Errorf("getting status: %w", err) + } + if flags.asJSON { + enc := json.NewEncoder(out) + enc.SetIndent("", " ") + return enc.Encode(status) + } + fmt.Fprintln(out, "Resource Type\tCount") + fmt.Fprintln(out, "-------------\t-----") + for rt, count := range status { + fmt.Fprintf(out, "%s\t%d\n", rt, count) + } + return nil + } + + if groupBy != "" { + return runGroupBy(out, db, resourceType, groupBy, limit, flags) + } + + count, err := db.Count(resourceType) + if err != nil { + return fmt.Errorf("counting: %w", err) + } + + if flags.asJSON { + result := map[string]any{"resource_type": resourceType, "count": count} + enc := json.NewEncoder(out) + enc.SetIndent("", " ") + return enc.Encode(result) + } + + fmt.Fprintf(out, "%s: %d records\n", resourceType, count) + return nil + }, + } + + cmd.Flags().StringVar(&resourceType, "type", "", "Resource type to analyze") + cmd.Flags().StringVar(&groupBy, "group-by", "", "Field to group by") + cmd.Flags().StringVar(&dbPath, "db", "", "Database path") + cmd.Flags().IntVar(&limit, "limit", 25, "Max groups to show") + + return cmd +} + +func runGroupBy(out io.Writer, db *store.Store, resourceType, field string, limit int, flags *rootFlags) error { + items, err := db.List(resourceType, 0) + if err != nil { + return err + } + + counts := make(map[string]int) + validFields := make(map[string]struct{}) + matchedAny := false + missingFieldCount := 0 + decodedRows := 0 + for _, item := range items { + var obj map[string]any + if err := json.Unmarshal(item, &obj); err != nil { + continue + } + decodedRows++ + for key := range obj { + validFields[key] = struct{}{} + } + raw, ok := resolvedGroupValue(obj, field) + if ok { + matchedAny = true + val := fmt.Sprintf("%v", raw) + counts[val]++ + } else { + missingFieldCount++ + } + } + if decodedRows > 0 && !matchedAny { + return fmt.Errorf("group-by field %q was not found in any %s record; valid group-by fields: %s", field, resourceType, formatGroupByFields(validFields)) + } + if missingFieldCount > 0 { + counts[""] += missingFieldCount + } + + type kv struct { + Key string `json:"value"` + Count int `json:"count"` + } + var sorted []kv + for k, v := range counts { + sorted = append(sorted, kv{k, v}) + } + sort.Slice(sorted, func(i, j int) bool { return sorted[i].Count > sorted[j].Count }) + if limit > 0 && len(sorted) > limit { + sorted = sorted[:limit] + } + + if flags.asJSON { + enc := json.NewEncoder(out) + enc.SetIndent("", " ") + return enc.Encode(sorted) + } + + fmt.Fprintf(out, "%s\tCount\n", field) + fmt.Fprintln(out, "---\t-----") + for _, kv := range sorted { + fmt.Fprintf(out, "%s\t%d\n", kv.Key, kv.Count) + } + return nil +} + +func resolvedGroupValue(obj map[string]any, field string) (any, bool) { + if val, ok := obj[field]; ok { + return val, true + } + if !strings.HasSuffix(field, "_id") { + if val, ok := obj[field+"_id"]; ok { + return val, true + } + } + return nil, false +} + +func formatGroupByFields(fields map[string]struct{}) string { + if len(fields) == 0 { + return "none" + } + var names []string + var aliases []string + for name := range fields { + names = append(names, name) + if strings.HasSuffix(name, "_id") { + alias := strings.TrimSuffix(name, "_id") + if alias != "" { + if _, exists := fields[alias]; !exists { + aliases = append(aliases, alias) + } + } + } + } + sort.Strings(names) + sort.Strings(aliases) + if len(aliases) == 0 { + return strings.Join(names, ", ") + } + return fmt.Sprintf("%s (aliases: %s)", strings.Join(names, ", "), strings.Join(aliases, ", ")) +} diff --git a/library/productivity/devonthink/internal/cli/api_discovery.go b/library/productivity/devonthink/internal/cli/api_discovery.go new file mode 100644 index 0000000000..63d285d6c6 --- /dev/null +++ b/library/productivity/devonthink/internal/cli/api_discovery.go @@ -0,0 +1,109 @@ +// Copyright 2026 rowdy and contributors. Licensed under Apache-2.0. See LICENSE. +// Generated by CLI Printing Press (https://github.com/mvanhorn/cli-printing-press). DO NOT EDIT. + +package cli + +import ( + "fmt" + "sort" + "strings" + + "github.com/spf13/cobra" +) + +func newAPICmd(flags *rootFlags) *cobra.Command { + cmd := &cobra.Command{ + Use: "api [interface]", + Short: "Browse all API endpoints by interface name", + Annotations: map[string]string{"mcp:read-only": "true"}, + Long: `Browse and call any API endpoint using the raw interface names. + +The friendly top-level commands cover the most common operations. +This command provides access to ALL endpoints for power users and +agents that need full API coverage. + +Run 'api' with no arguments to list all interfaces. +Run 'api ' to see that interface's methods.`, + Example: ` # List all available interfaces + devonthink-pp-cli api + + # Show methods for a specific interface + devonthink-pp-cli api `, + RunE: func(cmd *cobra.Command, args []string) error { + root := cmd.Root() + + if len(args) > 0 { + target := strings.ToLower(args[0]) + for _, child := range root.Commands() { + if child.Hidden && strings.ToLower(child.Name()) == target { + methods := child.Commands() + // JSON envelope: {interface, short, methods: [{name, short}, ...]}. + if flags.asJSON { + methodList := make([]map[string]any, 0, len(methods)) + for _, method := range methods { + methodList = append(methodList, map[string]any{ + "name": method.Name(), + "short": method.Short, + }) + } + return printJSONFiltered(cmd.OutOrStdout(), map[string]any{ + "interface": child.Name(), + "short": child.Short, + "methods": methodList, + }, flags) + } + if len(methods) == 0 { + return child.Help() + } + fmt.Fprintf(cmd.OutOrStdout(), "%s — %s\n\nMethods:\n", child.Name(), child.Short) + for _, method := range methods { + fmt.Fprintf(cmd.OutOrStdout(), " %-50s %s\n", child.Name()+" "+method.Name(), method.Short) + } + fmt.Fprintf(cmd.OutOrStdout(), "\nUse '%s-pp-cli %s --help' for details.\n", "devonthink", child.Name()) + return nil + } + } + return fmt.Errorf("interface %q not found. Run '%s-pp-cli api' to list all interfaces", args[0], "devonthink") + } + + // Pre-formatting human strings ahead of time would block the JSON + // path from emitting clean field values; build the typed slice and + // derive human format on print. + type ifaceEntry struct { + Name string `json:"name"` + Short string `json:"short"` + } + var ifaces []ifaceEntry + for _, child := range root.Commands() { + if child.Hidden { + ifaces = append(ifaces, ifaceEntry{Name: child.Name(), Short: child.Short}) + } + } + sort.Slice(ifaces, func(i, j int) bool { return ifaces[i].Name < ifaces[j].Name }) + + // JSON envelope: {interfaces: [...], note?: "..."}. + if flags.asJSON { + out := map[string]any{"interfaces": ifaces} + if len(ifaces) == 0 { + out["interfaces"] = []ifaceEntry{} + out["note"] = "No hidden API interfaces found." + } + return printJSONFiltered(cmd.OutOrStdout(), out, flags) + } + + if len(ifaces) == 0 { + fmt.Fprintln(cmd.OutOrStdout(), "No hidden API interfaces found.") + return nil + } + + fmt.Fprintf(cmd.OutOrStdout(), "Available API interfaces (%d):\n\n", len(ifaces)) + for _, e := range ifaces { + fmt.Fprintf(cmd.OutOrStdout(), " %-45s %s\n", e.Name, e.Short) + } + fmt.Fprintf(cmd.OutOrStdout(), "\nUse '%s-pp-cli api ' to see methods.\n", "devonthink") + return nil + }, + } + + return cmd +} diff --git a/library/productivity/devonthink/internal/cli/batch.go b/library/productivity/devonthink/internal/cli/batch.go new file mode 100644 index 0000000000..eac50dad71 --- /dev/null +++ b/library/productivity/devonthink/internal/cli/batch.go @@ -0,0 +1,22 @@ +// Copyright 2026 rowdy and contributors. Licensed under Apache-2.0. See LICENSE. +// Generated by CLI Printing Press (https://github.com/mvanhorn/cli-printing-press). DO NOT EDIT. + +package cli + +import ( + "github.com/spf13/cobra" +) + +func newBatchCmd(flags *rootFlags) *cobra.Command { + cmd := &cobra.Command{ + Use: "batch", + Short: "Dry-run-first multi-record mutation plans", + Hidden: true, + Annotations: map[string]string{"mcp:read-only": "true"}, + RunE: parentNoSubcommandRunE(flags), + } + + cmd.AddCommand(newBatchApplyCmd(flags)) + cmd.AddCommand(newBatchPlanCmd(flags)) + return cmd +} diff --git a/library/productivity/devonthink/internal/cli/batch_apply.go b/library/productivity/devonthink/internal/cli/batch_apply.go new file mode 100644 index 0000000000..2fcec9f63a --- /dev/null +++ b/library/productivity/devonthink/internal/cli/batch_apply.go @@ -0,0 +1,195 @@ +// Copyright 2026 rowdy and contributors. Licensed under Apache-2.0. See LICENSE. +// Generated by CLI Printing Press (https://github.com/mvanhorn/cli-printing-press). DO NOT EDIT. + +package cli + +import ( + "encoding/json" + "fmt" + "io" + "os" + + "github.com/spf13/cobra" +) + +func newBatchApplyCmd(flags *rootFlags) *cobra.Command { + var bodyYes bool + var stdinBody bool + + cmd := &cobra.Command{ + Use: "apply ", + Short: "Apply a previously reviewed local JSON plan", + Example: " devonthink-pp-cli batch apply plan.json --dry-run", + Annotations: map[string]string{"pp:endpoint": "batch.apply", "pp:method": "POST", "pp:path": "/batch/apply"}, + RunE: func(cmd *cobra.Command, args []string) error { + if len(args) == 0 { + return cmd.Help() + } + if !stdinBody { + } + c, err := flags.newClient() + if err != nil { + return err + } + + path := "/batch/apply" + params := map[string]string{} + params["plan"] = args[0] + var body map[string]any + if stdinBody { + stdinData, err := io.ReadAll(os.Stdin) + if err != nil { + return fmt.Errorf("reading stdin: %w", err) + } + var jsonBody map[string]any + if err := json.Unmarshal(stdinData, &jsonBody); err != nil { + return fmt.Errorf("parsing stdin JSON: %w", err) + } + body = jsonBody + } else { + body = map[string]any{} + if cmd.Flags().Changed("yes") { + body["yes"] = bodyYes + } + } + data, statusCode, err := c.PostWithParams(cmd.Context(), path, params, body) + if err != nil { + return classifyAPIError(err, flags) + } + // Inspect the mutate response body for a partial-failure-shaped + // field (e.g. Google Ads `partialFailureError`). Several Google + // APIs return 200 OK with a partial-failure field when some + // operations in the batch failed; ignoring it silently swallows + // real failures. Detection runs before output-mode selection so + // the exit code is consistent regardless of how stdout is + // rendered. --dry-run short-circuits because no real request + // was sent. + var partialFailure *partialFailureReport + if !flags.dryRun && statusCode >= 200 && statusCode < 300 { + partialFailure = detectPartialFailure(data) + if partialFailure != nil { + fmt.Fprintf(os.Stderr, "warning: partial failure detected in %s response: %s\n", "batch", partialFailure.Message) + if len(partialFailure.ResourceNames) > 0 { + fmt.Fprintf(os.Stderr, " succeeded: %d operation(s)\n", len(partialFailure.ResourceNames)) + } + } + } + if !flags.dryRun && statusCode >= 200 && statusCode < 300 && (partialFailure == nil || flags.allowPartialFailure) { + writeMutationResponseToStore(cmd.Context(), "batch", data, "") + } + if wantsHumanTable(cmd.OutOrStdout(), flags) { + // Check if response contains an array (directly or wrapped in "data") + var items []map[string]any + if json.Unmarshal(data, &items) == nil && len(items) > 0 { + if err := printAutoTable(cmd.OutOrStdout(), items); err != nil { + fmt.Fprintf(os.Stderr, "warning: table rendering failed, falling back to JSON: %v\n", err) + } else { + if partialFailure != nil && !flags.allowPartialFailure { + return partialFailureErr(fmt.Errorf("partial failure in %s response: %s", "batch", partialFailure.Message)) + } + return nil + } + } else { + var wrapped struct { + Data []map[string]any `json:"data"` + } + if json.Unmarshal(data, &wrapped) == nil && len(wrapped.Data) > 0 { + if err := printAutoTable(cmd.OutOrStdout(), wrapped.Data); err != nil { + fmt.Fprintf(os.Stderr, "warning: table rendering failed, falling back to JSON: %v\n", err) + } else { + if partialFailure != nil && !flags.allowPartialFailure { + return partialFailureErr(fmt.Errorf("partial failure in %s response: %s", "batch", partialFailure.Message)) + } + return nil + } + } + } + } + if flags.asJSON || (!isTerminal(cmd.OutOrStdout()) && !flags.csv && !flags.quiet && !flags.plain) { + if flags.quiet { + if partialFailure != nil && !flags.allowPartialFailure { + return partialFailureErr(fmt.Errorf("partial failure in %s response: %s", "batch", partialFailure.Message)) + } + return nil + } + envelope := map[string]any{ + "action": "post", + "resource": "batch", + "path": path, + "status": statusCode, + "success": statusCode >= 200 && statusCode < 300 && (partialFailure == nil || flags.allowPartialFailure), + } + if partialFailure != nil { + envelope["partial_failure"] = partialFailure + } + if flags.dryRun { + envelope["dry_run"] = true + envelope["status"] = 0 + envelope["success"] = false + } + // Verify-mode synthetic envelope detection runs against RAW data + // (before --compact/--select filtering) so the sentinel field is + // guaranteed to be visible even if the operator passes a filter + // flag that would otherwise strip it. Surfaces a top-level + // verify_noop signal + flips success to false. Mirrors the dry_run + // shape above. + if len(data) > 0 { + var rawParsed any + if err := json.Unmarshal(data, &rawParsed); err == nil { + if m, ok := rawParsed.(map[string]any); ok { + if v, ok := m["__pp_verify_synthetic__"].(bool); ok && v { + envelope["verify_noop"] = true + envelope["success"] = false + } + } + } + } + // Apply --compact and --select to the API response before wrapping. + // --select wins when both are set: explicit field choice trumps the + // generic high-gravity allow-list. Otherwise --compact still applies + // when --agent is on but the user did not name fields. + filtered := data + if flags.selectFields != "" { + filtered = filterFields(filtered, flags.selectFields) + } else if flags.compact { + filtered = compactFields(filtered) + } + if len(filtered) > 0 { + var parsed any + if err := json.Unmarshal(filtered, &parsed); err == nil { + envelope["data"] = parsed + } + } + envelopeJSON, err := json.Marshal(envelope) + if err != nil { + return err + } + if perr := printOutput(cmd.OutOrStdout(), json.RawMessage(envelopeJSON), true); perr != nil { + return perr + } + if partialFailure != nil && !flags.allowPartialFailure { + return partialFailureErr(fmt.Errorf("partial failure in %s response: %s", "batch", partialFailure.Message)) + } + return nil + } + // Fall-through for mutate paths that did not hit the table or + // asJSON branches: --quiet, --csv, --plain, and default terminal + // raw output. printOutputWithFlags renders the body, then the + // typed partial-failure exit fires unless --allow-partial-failure + // downgrades it. Without this guard a partial failure would exit + // 0 for these output modes — the exact silent-swallow regression + // the surrounding patch is preventing for asJSON / piped output. + if perr := printOutputWithFlags(cmd.OutOrStdout(), data, flags); perr != nil { + return perr + } + if partialFailure != nil && !flags.allowPartialFailure { + return partialFailureErr(fmt.Errorf("partial failure in %s response: %s", "batch", partialFailure.Message)) + } + return nil + }, + } + cmd.Flags().BoolVar(&bodyYes, "yes", false, "Confirm apply without an interactive prompt") + cmd.Flags().BoolVar(&stdinBody, "stdin", false, "Read request body as JSON from stdin") + + return cmd +} diff --git a/library/productivity/devonthink/internal/cli/batch_plan.go b/library/productivity/devonthink/internal/cli/batch_plan.go new file mode 100644 index 0000000000..69c0cf416e --- /dev/null +++ b/library/productivity/devonthink/internal/cli/batch_plan.go @@ -0,0 +1,216 @@ +// Copyright 2026 rowdy and contributors. Licensed under Apache-2.0. See LICENSE. +// Generated by CLI Printing Press (https://github.com/mvanhorn/cli-printing-press). DO NOT EDIT. + +package cli + +import ( + "encoding/json" + "fmt" + "io" + "os" + + "github.com/spf13/cobra" +) + +func newBatchPlanCmd(flags *rootFlags) *cobra.Command { + var bodyQuery string + var bodyFrom string + var bodyAddTag string + var bodyMoveTo string + var bodyOutput string + var bodyDryRun bool + var stdinBody bool + + cmd := &cobra.Command{ + Use: "plan", + Short: "Stage multi-record changes as a local JSON plan", + Example: " devonthink-pp-cli batch plan --query \"tags:todo\" --add-tag reviewed --dry-run --agent", + Annotations: map[string]string{"pp:endpoint": "batch.plan", "pp:method": "POST", "pp:path": "/batch/plan"}, + RunE: func(cmd *cobra.Command, args []string) error { + if !stdinBody { + } + c, err := flags.newClient() + if err != nil { + return err + } + + path := "/batch/plan" + params := map[string]string{} + var body map[string]any + if stdinBody { + stdinData, err := io.ReadAll(os.Stdin) + if err != nil { + return fmt.Errorf("reading stdin: %w", err) + } + var jsonBody map[string]any + if err := json.Unmarshal(stdinData, &jsonBody); err != nil { + return fmt.Errorf("parsing stdin JSON: %w", err) + } + body = jsonBody + } else { + body = map[string]any{} + if bodyQuery != "" { + body["query"] = bodyQuery + } + if bodyFrom != "" { + body["from"] = bodyFrom + } + if bodyAddTag != "" { + body["add_tag"] = bodyAddTag + } + if bodyMoveTo != "" { + body["move_to"] = bodyMoveTo + } + if bodyOutput != "" { + body["output"] = bodyOutput + } + if cmd.Flags().Changed("dry-run") { + body["dry_run"] = bodyDryRun + } + } + data, statusCode, err := c.PostWithParams(cmd.Context(), path, params, body) + if err != nil { + return classifyAPIError(err, flags) + } + // Inspect the mutate response body for a partial-failure-shaped + // field (e.g. Google Ads `partialFailureError`). Several Google + // APIs return 200 OK with a partial-failure field when some + // operations in the batch failed; ignoring it silently swallows + // real failures. Detection runs before output-mode selection so + // the exit code is consistent regardless of how stdout is + // rendered. --dry-run short-circuits because no real request + // was sent. + var partialFailure *partialFailureReport + if !flags.dryRun && statusCode >= 200 && statusCode < 300 { + partialFailure = detectPartialFailure(data) + if partialFailure != nil { + fmt.Fprintf(os.Stderr, "warning: partial failure detected in %s response: %s\n", "batch", partialFailure.Message) + if len(partialFailure.ResourceNames) > 0 { + fmt.Fprintf(os.Stderr, " succeeded: %d operation(s)\n", len(partialFailure.ResourceNames)) + } + } + } + if !flags.dryRun && statusCode >= 200 && statusCode < 300 && (partialFailure == nil || flags.allowPartialFailure) { + writeMutationResponseToStore(cmd.Context(), "batch", data, "") + } + if wantsHumanTable(cmd.OutOrStdout(), flags) { + // Check if response contains an array (directly or wrapped in "data") + var items []map[string]any + if json.Unmarshal(data, &items) == nil && len(items) > 0 { + if err := printAutoTable(cmd.OutOrStdout(), items); err != nil { + fmt.Fprintf(os.Stderr, "warning: table rendering failed, falling back to JSON: %v\n", err) + } else { + if partialFailure != nil && !flags.allowPartialFailure { + return partialFailureErr(fmt.Errorf("partial failure in %s response: %s", "batch", partialFailure.Message)) + } + return nil + } + } else { + var wrapped struct { + Data []map[string]any `json:"data"` + } + if json.Unmarshal(data, &wrapped) == nil && len(wrapped.Data) > 0 { + if err := printAutoTable(cmd.OutOrStdout(), wrapped.Data); err != nil { + fmt.Fprintf(os.Stderr, "warning: table rendering failed, falling back to JSON: %v\n", err) + } else { + if partialFailure != nil && !flags.allowPartialFailure { + return partialFailureErr(fmt.Errorf("partial failure in %s response: %s", "batch", partialFailure.Message)) + } + return nil + } + } + } + } + if flags.asJSON || (!isTerminal(cmd.OutOrStdout()) && !flags.csv && !flags.quiet && !flags.plain) { + if flags.quiet { + if partialFailure != nil && !flags.allowPartialFailure { + return partialFailureErr(fmt.Errorf("partial failure in %s response: %s", "batch", partialFailure.Message)) + } + return nil + } + envelope := map[string]any{ + "action": "post", + "resource": "batch", + "path": path, + "status": statusCode, + "success": statusCode >= 200 && statusCode < 300 && (partialFailure == nil || flags.allowPartialFailure), + } + if partialFailure != nil { + envelope["partial_failure"] = partialFailure + } + if flags.dryRun { + envelope["dry_run"] = true + envelope["status"] = 0 + envelope["success"] = false + } + // Verify-mode synthetic envelope detection runs against RAW data + // (before --compact/--select filtering) so the sentinel field is + // guaranteed to be visible even if the operator passes a filter + // flag that would otherwise strip it. Surfaces a top-level + // verify_noop signal + flips success to false. Mirrors the dry_run + // shape above. + if len(data) > 0 { + var rawParsed any + if err := json.Unmarshal(data, &rawParsed); err == nil { + if m, ok := rawParsed.(map[string]any); ok { + if v, ok := m["__pp_verify_synthetic__"].(bool); ok && v { + envelope["verify_noop"] = true + envelope["success"] = false + } + } + } + } + // Apply --compact and --select to the API response before wrapping. + // --select wins when both are set: explicit field choice trumps the + // generic high-gravity allow-list. Otherwise --compact still applies + // when --agent is on but the user did not name fields. + filtered := data + if flags.selectFields != "" { + filtered = filterFields(filtered, flags.selectFields) + } else if flags.compact { + filtered = compactFields(filtered) + } + if len(filtered) > 0 { + var parsed any + if err := json.Unmarshal(filtered, &parsed); err == nil { + envelope["data"] = parsed + } + } + envelopeJSON, err := json.Marshal(envelope) + if err != nil { + return err + } + if perr := printOutput(cmd.OutOrStdout(), json.RawMessage(envelopeJSON), true); perr != nil { + return perr + } + if partialFailure != nil && !flags.allowPartialFailure { + return partialFailureErr(fmt.Errorf("partial failure in %s response: %s", "batch", partialFailure.Message)) + } + return nil + } + // Fall-through for mutate paths that did not hit the table or + // asJSON branches: --quiet, --csv, --plain, and default terminal + // raw output. printOutputWithFlags renders the body, then the + // typed partial-failure exit fires unless --allow-partial-failure + // downgrades it. Without this guard a partial failure would exit + // 0 for these output modes — the exact silent-swallow regression + // the surrounding patch is preventing for asJSON / piped output. + if perr := printOutputWithFlags(cmd.OutOrStdout(), data, flags); perr != nil { + return perr + } + if partialFailure != nil && !flags.allowPartialFailure { + return partialFailureErr(fmt.Errorf("partial failure in %s response: %s", "batch", partialFailure.Message)) + } + return nil + }, + } + cmd.Flags().StringVar(&bodyQuery, "query", "", "Search query selecting records") + cmd.Flags().StringVar(&bodyFrom, "from", "", "Source selector such as selection or file") + cmd.Flags().StringVar(&bodyAddTag, "add-tag", "", "Tag to add") + cmd.Flags().StringVar(&bodyMoveTo, "move-to", "", "Destination group path or UUID") + cmd.Flags().StringVar(&bodyOutput, "output", "", "Plan output path") + cmd.Flags().BoolVar(&bodyDryRun, "dry-run", false, "Preview only") + cmd.Flags().BoolVar(&stdinBody, "stdin", false, "Read request body as JSON from stdin") + + return cmd +} diff --git a/library/productivity/devonthink/internal/cli/channel_workflow.go b/library/productivity/devonthink/internal/cli/channel_workflow.go new file mode 100644 index 0000000000..a00e76487a --- /dev/null +++ b/library/productivity/devonthink/internal/cli/channel_workflow.go @@ -0,0 +1,165 @@ +// Copyright 2026 rowdy and contributors. Licensed under Apache-2.0. See LICENSE. +// Generated by CLI Printing Press (https://github.com/mvanhorn/cli-printing-press). DO NOT EDIT. + +package cli + +import ( + "encoding/json" + "fmt" + "time" + + "github.com/mvanhorn/printing-press-library/library/productivity/devonthink/internal/store" + "github.com/spf13/cobra" +) + +func newWorkflowCmd(flags *rootFlags) *cobra.Command { + cmd := &cobra.Command{ + Use: "workflow", + Short: "Compound workflows that combine multiple API operations", + Annotations: map[string]string{"mcp:read-only": "true"}, + RunE: parentNoSubcommandRunE(flags), + } + cmd.AddCommand(newWorkflowArchiveCmd(flags)) + cmd.AddCommand(newWorkflowStatusCmd(flags)) + + return cmd +} +func newWorkflowArchiveCmd(flags *rootFlags) *cobra.Command { + var dbPath string + var full bool + + cmd := &cobra.Command{ + Use: "archive", + Short: "Sync all resources to local store for offline access and search", + Long: `Archive fetches all syncable resources from the API and stores them in a +local SQLite database. Supports incremental sync (only new data since last run) +and full resync. After archiving, use 'search' for instant full-text search.`, + Example: ` # Archive all resources + devonthink-pp-cli workflow archive + + # Full re-archive (ignore previous sync state) + devonthink-pp-cli workflow archive --full`, + RunE: func(cmd *cobra.Command, args []string) error { + c, err := flags.newClient() + if err != nil { + return err + } + c.NoCache = true + + if dbPath == "" { + dbPath = defaultDBPath("devonthink-pp-cli") + } + s, err := store.OpenWithContext(cmd.Context(), dbPath) + if err != nil { + return fmt.Errorf("opening store: %w", err) + } + defer s.Close() + + resources := []string{"databases", "ledger", "mirror", "records", "selection"} + totalSynced := 0 + syncEventWriter := cmd.OutOrStdout() + if flags.asJSON { + syncEventWriter = cmd.ErrOrStderr() + } + + // --full clears the cursor here because syncResource reads + // existingCursor unconditionally; its full param only gates the + // since filter, not cursor reset. Mirrors newSyncCmd's pattern. + if full { + for _, resource := range resources { + _ = s.SaveSyncState(resource, "", 0) + } + } + + for _, resource := range resources { + res := syncResource(cmd.Context(), c, s, resource, "", full, 100, false, nil, syncEventWriter) + if res.Err != nil { + fmt.Fprintf(cmd.ErrOrStderr(), " %s: error: %v\n", resource, res.Err) + continue + } + if res.Warn != nil { + fmt.Fprintf(cmd.ErrOrStderr(), " %s: warning: %v\n", resource, res.Warn) + continue + } + totalSynced += res.Count + fmt.Fprintf(cmd.ErrOrStderr(), " %s: %d synced\n", resource, res.Count) + } + + if flags.asJSON { + enc := json.NewEncoder(cmd.OutOrStdout()) + enc.SetIndent("", " ") + return enc.Encode(map[string]any{ + "resources_synced": len(resources), + "total_items": totalSynced, + "store_path": dbPath, + "timestamp": time.Now().UTC().Format(time.RFC3339), + }) + } + + fmt.Fprintf(cmd.OutOrStdout(), "Archived %d items across %d resources to %s\n", totalSynced, len(resources), dbPath) + return nil + }, + } + + cmd.Flags().StringVar(&dbPath, "db", "", "Database path (default: ~/.local/share/devonthink-pp-cli/data.db)") + cmd.Flags().BoolVar(&full, "full", false, "Full re-archive (ignore previous sync state)") + + return cmd +} + +func newWorkflowStatusCmd(flags *rootFlags) *cobra.Command { + var dbPath string + + cmd := &cobra.Command{ + Use: "status", + Short: "Show local archive status and sync state for all resources", + Annotations: map[string]string{"mcp:read-only": "true"}, + Example: ` # Show archive status + devonthink-pp-cli workflow status + + # Show status as JSON + devonthink-pp-cli workflow status --json`, + RunE: func(cmd *cobra.Command, args []string) error { + if dbPath == "" { + dbPath = defaultDBPath("devonthink-pp-cli") + } + s, err := store.OpenWithContext(cmd.Context(), dbPath) + if err != nil { + return fmt.Errorf("opening store: %w", err) + } + defer s.Close() + + status, err := s.Status() + if err != nil { + return err + } + + if flags.asJSON { + enc := json.NewEncoder(cmd.OutOrStdout()) + enc.SetIndent("", " ") + return enc.Encode(status) + } + + if len(status) == 0 { + fmt.Fprintln(cmd.OutOrStdout(), "No archived data. Run 'workflow archive' to sync.") + return nil + } + + fmt.Fprintln(cmd.OutOrStdout(), "Archive Status:") + total := 0 + for resource, count := range status { + fmt.Fprintf(cmd.OutOrStdout(), " %-30s %d items\n", resource, count) + total += count + } + fmt.Fprintf(cmd.OutOrStdout(), "\n Total: %d items\n", total) + fmt.Fprintf(cmd.OutOrStdout(), " Store: %s\n", dbPath) + return nil + }, + } + + cmd.Flags().StringVar(&dbPath, "db", "", "Database path") + + return cmd +} + +// defaultDBPath is defined in helpers.go diff --git a/library/productivity/devonthink/internal/cli/context_pack.go b/library/productivity/devonthink/internal/cli/context_pack.go new file mode 100644 index 0000000000..eb95f09ff6 --- /dev/null +++ b/library/productivity/devonthink/internal/cli/context_pack.go @@ -0,0 +1,45 @@ +// Copyright 2026 rowdy and contributors. Licensed under Apache-2.0. See LICENSE. +// Generated by CLI Printing Press (https://github.com/mvanhorn/cli-printing-press). DO NOT EDIT. + +package cli + +import ( + "github.com/spf13/cobra" +) + +func newNovelContextPackCmd(flags *rootFlags) *cobra.Command { + var flagLimit int + var flagQuery string + var flagTokenBudget int + + cmd := &cobra.Command{ + Use: "pack", + Short: "Build a compact evidence packet from records, selections, highlights, links, and related items.", + Annotations: map[string]string{"mcp:read-only": "true"}, + RunE: func(cmd *cobra.Command, args []string) error { + c, err := flags.newClient() + if err != nil { + return err + } + params := map[string]string{} + if flagQuery != "" { + params["query"] = formatCLIParamValue(flagQuery) + } + if flagLimit != 0 { + params["limit"] = formatCLIParamValue(flagLimit) + } + if flagTokenBudget != 0 { + params["token_budget"] = formatCLIParamValue(flagTokenBudget) + } + data, err := c.Get(cmd.Context(), "/context/pack", params) + if err != nil { + return classifyAPIError(err, flags) + } + return printOutputWithFlags(cmd.OutOrStdout(), data, flags) + }, + } + cmd.Flags().IntVar(&flagLimit, "limit", 10, "Maximum record metadata rows to include") + cmd.Flags().StringVar(&flagQuery, "query", "", "DEVONthink query or selection context to pack") + cmd.Flags().IntVar(&flagTokenBudget, "token-budget", 4000, "Approximate token budget for the context pack") + return cmd +} diff --git a/library/productivity/devonthink/internal/cli/context_pack_test.go b/library/productivity/devonthink/internal/cli/context_pack_test.go new file mode 100644 index 0000000000..d52867713c --- /dev/null +++ b/library/productivity/devonthink/internal/cli/context_pack_test.go @@ -0,0 +1,20 @@ +// Copyright 2026 rowdy and contributors. Licensed under Apache-2.0. See LICENSE. +// Generated by CLI Printing Press (https://github.com/mvanhorn/cli-printing-press). DO NOT EDIT. + +package cli + +import "testing" + +func TestNovelContextPackCommandDryRun(t *testing.T) { + payload := executeNovelDryRun(t, "context", "pack", "--query", "kind:pdf", "--limit", "2", "--token-budget", "1200") + if payload["path"] != "/context/pack" { + t.Fatalf("path = %v, want /context/pack", payload["path"]) + } + params, ok := payload["params"].(map[string]any) + if !ok { + t.Fatalf("params missing from payload: %#v", payload) + } + if params["query"] != "kind:pdf" || params["limit"] != "2" || params["token_budget"] != "1200" { + t.Fatalf("params = %#v", params) + } +} diff --git a/library/productivity/devonthink/internal/cli/data_source.go b/library/productivity/devonthink/internal/cli/data_source.go new file mode 100644 index 0000000000..fa5c0c87e6 --- /dev/null +++ b/library/productivity/devonthink/internal/cli/data_source.go @@ -0,0 +1,612 @@ +// Copyright 2026 rowdy and contributors. Licensed under Apache-2.0. See LICENSE. +// Generated by CLI Printing Press (https://github.com/mvanhorn/cli-printing-press). DO NOT EDIT. + +package cli + +import ( + "bytes" + "context" + "database/sql" + "encoding/json" + "errors" + "fmt" + "io" + "net" + "net/url" + "os" + "strings" + "time" + + "github.com/mvanhorn/printing-press-library/library/productivity/devonthink/internal/client" + "github.com/mvanhorn/printing-press-library/library/productivity/devonthink/internal/store" +) + +const networkFallbackReason = "api_unreachable" + +func unsupportedDataSourceError(strategy, requested string) error { + switch strategy { + case "local": + return fmt.Errorf("no live equivalent for this command (requested %q); use --data-source local or --data-source auto", requested) + case "live": + return fmt.Errorf("no local data source for this command (requested %q); use --data-source live or --data-source auto", requested) + default: + return fmt.Errorf("unsupported --data-source %q for strategy %q", requested, strategy) + } +} + +func validateDataSourceStrategy(flags *rootFlags, strategy string) error { + switch strategy { + case "", "auto": + return nil + case "local": + if flags.dataSource == "live" { + return unsupportedDataSourceError(strategy, flags.dataSource) + } + case "live": + if flags.dataSource == "local" { + return unsupportedDataSourceError(strategy, flags.dataSource) + } + } + return nil +} + +// isNetworkError returns true for errors caused by network connectivity issues +// (DNS, connection refused, timeout). HTTP 4xx/5xx errors are NOT network errors. +func isNetworkError(err error) bool { + if err == nil { + return false + } + var urlErr *url.Error + if As(err, &urlErr) { + // url.Error wraps the underlying network error + err = urlErr.Err + } + var netErr *net.OpError + if As(err, &netErr) { + return true + } + var dnsErr *net.DNSError + if As(err, &dnsErr) { + return true + } + // Check for common network error strings + msg := err.Error() + return strings.Contains(msg, "connection refused") || + strings.Contains(msg, "no such host") || + strings.Contains(msg, "network is unreachable") || + strings.Contains(msg, "i/o timeout") || + strings.Contains(msg, "TLS handshake timeout") +} + +// openStoreForRead opens the local SQLite store for reading. +// Returns nil, nil if the database file does not exist (no sync has been run). +func openStoreForRead(ctx context.Context, cliName string) (*store.Store, error) { + dbPath := defaultDBPath(cliName) + if _, err := os.Stat(dbPath); os.IsNotExist(err) { + return nil, nil + } + return store.OpenWithContext(ctx, dbPath) +} + +// localProvenance builds a DataProvenance for local data reads. +func localProvenance(db *store.Store, resourceType, reason string) DataProvenance { + prov := DataProvenance{ + Source: "local", + Reason: reason, + ResourceType: resourceType, + } + _, lastSynced, _, err := db.GetSyncState(resourceType) + if err == nil && !lastSynced.IsZero() { + prov.SyncedAt = &lastSynced + } + return prov +} + +func attachFreshness(prov DataProvenance, flags *rootFlags) DataProvenance { + if flags != nil { + prov.Freshness = flags.freshnessMeta + } + return prov +} + +// resolveRead dispatches a GET request to either the live API or local store +// based on the --data-source flag. Returns the response data and provenance metadata. +// +// Parameters: +// - c: the HTTP client for live API calls +// - flags: root flags containing dataSource setting +// - resourceType: the store resource type name (e.g., "links", "domains") +// - isList: true for list endpoints, false for get-by-ID endpoints +// - path: the API path (e.g., "/links" or "/links/abc123") +// - params: query parameters for the API call +// - headers: per-endpoint required headers (e.g. cal-api-version, Stripe-Version) +// baked in by the command template at codegen time. Pass nil when the endpoint +// declares no per-endpoint header overrides. Without this parameter, store-backed +// reads on per-endpoint-versioned APIs silently get the wrong response shape +// (cal-com retro #334 F1). +func resolveRead(ctx context.Context, c *client.Client, flags *rootFlags, resourceType string, isList bool, path string, params map[string]string, headers map[string]string, hintWriter io.Writer) (json.RawMessage, DataProvenance, error) { + return resolveReadWithStrategy(ctx, c, flags, "auto", resourceType, isList, path, params, headers, hintWriter) +} + +func resolveReadWithStrategy(ctx context.Context, c *client.Client, flags *rootFlags, strategy string, resourceType string, isList bool, path string, params map[string]string, headers map[string]string, hintWriter io.Writer) (json.RawMessage, DataProvenance, error) { + if err := validateDataSourceStrategy(flags, strategy); err != nil { + return nil, DataProvenance{}, err + } + if strategy == "local" { + data, prov, err := resolveLocal(ctx, flags, hintWriter, resourceType, isList, path, params, "strategy_local") + return data, attachFreshness(prov, flags), err + } + if strategy == "live" { + data, err := c.GetWithHeaders(ctx, path, params, headers) + if err != nil { + return nil, DataProvenance{}, err + } + return data, attachFreshness(DataProvenance{Source: "live"}, flags), nil + } + switch flags.dataSource { + case "local": + data, prov, err := resolveLocal(ctx, flags, hintWriter, resourceType, isList, path, params, "user_requested") + return data, attachFreshness(prov, flags), err + + case "live": + data, err := c.GetWithHeaders(ctx, path, params, headers) + if err != nil { + return nil, DataProvenance{}, err + } + return data, attachFreshness(DataProvenance{Source: "live"}, flags), nil + + default: // "auto" + data, err := c.GetWithHeaders(ctx, path, params, headers) + if err == nil { + writeThroughCache(ctx, resourceType, data) + return data, attachFreshness(DataProvenance{Source: "live"}, flags), nil + } + if !isNetworkError(err) { + // HTTP 4xx/5xx errors propagate — not a fallback case + return nil, DataProvenance{}, err + } + // Network error — try local fallback + fallbackData, fallbackProv, fallbackErr := resolveLocal(ctx, flags, hintWriter, resourceType, isList, path, params, networkFallbackReason) + if fallbackErr != nil { + return nil, DataProvenance{}, fmt.Errorf("API unreachable and no local data. Run 'devonthink-pp-cli sync' to enable offline access.\n\nOriginal error: %w", err) + } + return fallbackData, attachFreshness(fallbackProv, flags), nil + } +} + +// resolvePaginatedRead dispatches a paginated GET request to either the live API +// or local store. When local, skips pagination and returns all synced data. The +// headers argument carries per-endpoint required headers; pass nil when the +// endpoint declares no overrides. +func resolvePaginatedRead(ctx context.Context, c *client.Client, flags *rootFlags, resourceType string, path string, params map[string]string, headers map[string]string, fetchAll bool, cursorParam, paginationType, limitParam, nextCursorPath, hasMoreField string, hintWriter io.Writer) (json.RawMessage, DataProvenance, error) { + return resolvePaginatedReadWithStrategy(ctx, c, flags, "auto", resourceType, path, params, headers, fetchAll, cursorParam, paginationType, limitParam, nextCursorPath, hasMoreField, hintWriter) +} + +func resolvePaginatedReadWithStrategy(ctx context.Context, c *client.Client, flags *rootFlags, strategy string, resourceType string, path string, params map[string]string, headers map[string]string, fetchAll bool, cursorParam, paginationType, limitParam, nextCursorPath, hasMoreField string, hintWriter io.Writer) (json.RawMessage, DataProvenance, error) { + if err := validateDataSourceStrategy(flags, strategy); err != nil { + return nil, DataProvenance{}, err + } + if strategy == "local" { + data, prov, err := resolveLocal(ctx, flags, hintWriter, resourceType, true, path, params, "strategy_local") + return data, attachFreshness(prov, flags), err + } + if strategy == "live" { + data, err := paginatedGet(ctx, c, path, params, headers, fetchAll, cursorParam, paginationType, limitParam, nextCursorPath, hasMoreField) + if err != nil { + return nil, DataProvenance{}, err + } + return data, attachFreshness(DataProvenance{Source: "live"}, flags), nil + } + switch flags.dataSource { + case "local": + data, prov, err := resolveLocal(ctx, flags, hintWriter, resourceType, true, path, params, "user_requested") + return data, attachFreshness(prov, flags), err + + case "live": + data, err := paginatedGet(ctx, c, path, params, headers, fetchAll, cursorParam, paginationType, limitParam, nextCursorPath, hasMoreField) + if err != nil { + return nil, DataProvenance{}, err + } + return data, attachFreshness(DataProvenance{Source: "live"}, flags), nil + + default: // "auto" + data, err := paginatedGet(ctx, c, path, params, headers, fetchAll, cursorParam, paginationType, limitParam, nextCursorPath, hasMoreField) + if err == nil { + writeThroughCache(ctx, resourceType, data) + return data, attachFreshness(DataProvenance{Source: "live"}, flags), nil + } + if !isNetworkError(err) { + return nil, DataProvenance{}, err + } + fallbackData, fallbackProv, fallbackErr := resolveLocal(ctx, flags, hintWriter, resourceType, true, path, params, networkFallbackReason) + if fallbackErr != nil { + return nil, DataProvenance{}, fmt.Errorf("API unreachable and no local data. Run 'devonthink-pp-cli sync' to enable offline access.\n\nOriginal error: %w", err) + } + return fallbackData, attachFreshness(fallbackProv, flags), nil + } +} + +// listEnvelopeMetadataKeys are top-level keys that, when accompanying a +// list-wrapper array, suggest the response is a paginated list envelope +// rather than a detail object. Used by writeThroughCache to decide +// whether to upsert a single-object body. Any envelope key NOT in this +// set (and not a list wrapper itself) signals real per-row data, and +// the envelope is treated as a detail object even when one of its +// wrapper-named fields happens to be an empty array. +var listEnvelopeMetadataKeys = map[string]bool{ + // list wrappers themselves — must stay in sync with pageItemKeys + "results": true, "data": true, "items": true, + "records": true, "nodes": true, "entries": true, "features": true, + "Results": true, "Data": true, "Items": true, + "Records": true, "Nodes": true, "Entries": true, "Features": true, + // pagination cursors / tokens + "next_cursor": true, "nextCursor": true, + "next_page_token": true, "nextPageToken": true, + "page_token": true, "pageToken": true, + "end_cursor": true, "endCursor": true, + "start_cursor": true, "startCursor": true, + "cursor": true, "after": true, "before": true, + // has-more flags and page numbers + "has_more": true, "hasMore": true, "has_next": true, "hasNext": true, + "next_page": true, "previous_page": true, + "page": true, "page_size": true, "per_page": true, + // counts / totals + "total": true, "count": true, "size": true, "total_count": true, "totalCount": true, + // JSend / common status envelopes + "success": true, "status": true, "message": true, "error": true, + "errors": true, "Errors": true, "warnings": true, "Warnings": true, + // wrapper objects + "links": true, "meta": true, "pagination": true, + "response_metadata": true, "paging": true, + // links shape + "next": true, "prev": true, "previous": true, "first": true, "last": true, +} + +var writeThroughListWrapperKeys = []string{ + "data", "results", "items", "records", "nodes", "entries", "features", + "Data", "Results", "Items", "Records", "Nodes", "Entries", "Features", +} +var writeThroughNestedEnvelopeKeys = []string{"data", "Data", "result", "Result"} + +// writeThroughCache upserts live API results into the local SQLite store so +// FTS search covers everything the user has looked up — not just explicit syncs. +// Best-effort: failures are silently ignored (the live result already succeeded). +func writeThroughCache(ctx context.Context, resourceType string, data json.RawMessage) { + db, err := store.OpenWithContext(ctx, defaultDBPath("devonthink-pp-cli")) + if err != nil { + return + } + defer db.Close() + + // Collect items to upsert from various response shapes + var items []json.RawMessage + + // Try direct array first + if json.Unmarshal(data, &items) != nil || len(items) == 0 { + items = nil + // Try object — check for common envelope patterns (results, data, items) + var envelope map[string]json.RawMessage + if json.Unmarshal(data, &envelope) == nil { + matchedListEnvelope := false + if extracted, ok := extractWriteThroughListItems(envelope); ok { + matchedListEnvelope = true + items = extracted + } + if matchedListEnvelope && len(items) == 0 { + return + } + // Single object detail response: let UpsertBatch's existing + // resourceIDFieldOverrides mechanism resolve the primary key. + // Guarding on envelope["id"] dropped any API whose PK is named + // CertNo / sku / invoiceId / etc. on the floor (#1439). + // + // Treat the envelope as a list-shaped response only when EVERY + // top-level key is either a list-wrapper (results/data/items) + // holding a real array, or a known pagination-metadata key. + // A detail object that happens to carry an empty wrapper-named + // field alongside real data (e.g. {"id":"order","items":[], + // "status":"pending"}) must still cache as a single row. + if items == nil && !matchedListEnvelope && len(envelope) > 0 { + looksLikeListEnvelope := false + hasListWrapperArray := false + for _, key := range writeThroughListWrapperKeys { + raw, ok := envelope[key] + if !ok { + continue + } + // json.Unmarshal("null", &arr) succeeds with arr=nil, + // so require arr != nil to keep true empty arrays in + // the skip branch while letting scalar/null values fall + // through as regular field-name collisions. + var arr []json.RawMessage + if json.Unmarshal(raw, &arr) == nil && arr != nil { + hasListWrapperArray = true + break + } + } + if hasListWrapperArray { + looksLikeListEnvelope = true + for k := range envelope { + if !listEnvelopeMetadataKeys[k] { + looksLikeListEnvelope = false + break + } + } + } + if !looksLikeListEnvelope { + _, _, _ = db.UpsertBatch(resourceType, []json.RawMessage{data}) + return + } + } + } + } + + if len(items) > 0 { + _, _, _ = db.UpsertBatch(resourceType, items) + } +} + +type writeThroughArrayDecoder func(json.RawMessage) ([]json.RawMessage, bool) + +func extractWriteThroughListItems(envelope map[string]json.RawMessage) ([]json.RawMessage, bool) { + if items, ok := extractWriteThroughListWrapperItems(envelope, decodeWriteThroughNonEmptyArray); ok { + return items, true + } + + for _, key := range writeThroughNestedEnvelopeKeys { + raw, ok := envelope[key] + if !ok || isRawJSONNull(raw) { + continue + } + var inner map[string]json.RawMessage + if json.Unmarshal(raw, &inner) != nil { + continue + } + if items, ok := extractNestedWriteThroughListItems(inner); ok { + return items, true + } + } + + return extractWriteThroughSingleArraySibling(envelope, decodeWriteThroughNonEmptyArray) +} + +func extractNestedWriteThroughListItems(envelope map[string]json.RawMessage) ([]json.RawMessage, bool) { + if items, ok := extractWriteThroughListWrapperItems(envelope, decodeWriteThroughArray); ok { + return items, true + } + return extractWriteThroughSingleArraySibling(envelope, decodeWriteThroughArray) +} + +func extractWriteThroughListWrapperItems(envelope map[string]json.RawMessage, decode writeThroughArrayDecoder) ([]json.RawMessage, bool) { + for _, key := range writeThroughListWrapperKeys { + raw, ok := envelope[key] + if !ok { + continue + } + if items, ok := decode(raw); ok { + return items, true + } + } + return nil, false +} + +func extractWriteThroughSingleArraySibling(envelope map[string]json.RawMessage, decode writeThroughArrayDecoder) ([]json.RawMessage, bool) { + arrayCount := 0 + var arrayItems []json.RawMessage + for key, raw := range envelope { + if listEnvelopeMetadataKeys[key] { + continue + } + if candidate, ok := decode(raw); ok { + if !writeThroughArrayItemsAreObjects(candidate) { + continue + } + arrayItems = candidate + arrayCount++ + continue + } + return nil, false + } + if arrayCount == 1 { + return arrayItems, true + } + return nil, false +} + +func writeThroughArrayItemsAreObjects(items []json.RawMessage) bool { + if len(items) == 0 { + return true + } + var obj map[string]json.RawMessage + return json.Unmarshal(items[0], &obj) == nil +} + +func decodeWriteThroughNonEmptyArray(raw json.RawMessage) ([]json.RawMessage, bool) { + items, ok := decodeWriteThroughArray(raw) + if !ok || len(items) == 0 { + return nil, false + } + return items, true +} + +func decodeWriteThroughArray(raw json.RawMessage) ([]json.RawMessage, bool) { + var items []json.RawMessage + if json.Unmarshal(raw, &items) != nil || items == nil { + return nil, false + } + return items, true +} + +func isRawJSONNull(raw json.RawMessage) bool { + return strings.TrimSpace(string(raw)) == "null" +} + +func writeMutationResponseToStore(ctx context.Context, resourceType string, data json.RawMessage, responsePath string) { + items := mutationResponseEntityItems(resourceType, data, responsePath) + if len(items) == 0 { + return + } + + db, err := store.OpenWithContext(ctx, defaultDBPath("devonthink-pp-cli")) + if err != nil { + return + } + defer db.Close() + + _, _, _ = db.UpsertBatch(resourceType, items) +} + +func mutationResponseEntityItems(resourceType string, data json.RawMessage, responsePath string) []json.RawMessage { + if responsePath != "" { + if pathData, ok := mutationResponseAtPath(data, responsePath); ok { + data = pathData + } + } + + if items := mutationResponseItemsFromPayload(resourceType, data); len(items) > 0 { + return items + } + + data = mutationResponsePayload(data) + if items := mutationResponseItemsFromPayload(resourceType, data); len(items) > 0 { + return items + } + + if responsePath == "" { + var envelope map[string]json.RawMessage + if json.Unmarshal(data, &envelope) == nil { + if _, hasStatus := envelope["status"]; !hasStatus { + if raw, ok := envelope["data"]; ok { + return mutationResponseItemsFromPayload(resourceType, raw) + } + } + } + } + + return nil +} + +func mutationResponseItemsFromPayload(resourceType string, data json.RawMessage) []json.RawMessage { + if len(bytes.TrimSpace(data)) == 0 { + return nil + } + + var arr []json.RawMessage + if json.Unmarshal(data, &arr) == nil { + items := make([]json.RawMessage, 0, len(arr)) + for _, item := range arr { + if mutationResponseHasID(resourceType, item) { + items = append(items, item) + } + } + return items + } + + if mutationResponseHasID(resourceType, data) { + return []json.RawMessage{data} + } + return nil +} + +func mutationResponsePayload(data json.RawMessage) json.RawMessage { + var envelope struct { + Status string `json:"status"` + Data json.RawMessage `json:"data"` + } + if err := json.Unmarshal(data, &envelope); err != nil || envelope.Status == "" || envelope.Data == nil { + return data + } + switch envelope.Status { + case "success", "ok", "OK", "Success": + return envelope.Data + default: + return data + } +} + +func mutationResponseAtPath(data json.RawMessage, responsePath string) (json.RawMessage, bool) { + var root map[string]json.RawMessage + if err := json.Unmarshal(data, &root); err != nil { + return nil, false + } + return rawAtPath(root, strings.TrimPrefix(responsePath, "$.")) +} + +func mutationResponseHasID(resourceType string, data json.RawMessage) bool { + var obj map[string]any + if err := json.Unmarshal(data, &obj); err != nil || len(obj) == 0 { + return false + } + if synthetic, _ := obj["__pp_verify_synthetic__"].(bool); synthetic { + return false + } + return store.ExtractResourceID(resourceType, obj) != "" +} + +// resolveLocal reads data from the local SQLite store. +// Note: local reads return ALL synced data for the resource type. Endpoint-specific +// filters (query params, path scoping like /teams/{id}/users) are NOT applied locally. +// The provenance metadata includes "unscoped":true when params were present but not applied. +func resolveLocal(ctx context.Context, flags *rootFlags, hintWriter io.Writer, resourceType string, isList bool, path string, params map[string]string, reason string) (json.RawMessage, DataProvenance, error) { + db, err := openStoreForRead(ctx, "devonthink-pp-cli") + if err != nil { + return nil, DataProvenance{}, fmt.Errorf("opening local database: %w\nRun 'devonthink-pp-cli sync' first.", err) + } + if db == nil { + return nil, DataProvenance{}, fmt.Errorf("no local data. Run 'devonthink-pp-cli sync' first") + } + defer db.Close() + + if flags != nil { + emitSyncHints(hintWriter, db, resourceType, flags.maxAge) + } + + prov := localProvenance(db, resourceType, reason) + + // Warn if endpoint had filters that local reads can't reproduce + if len(params) > 0 { + fmt.Fprintf(os.Stderr, "warning: local data is unfiltered — endpoint filters are not applied to cached data\n") + } + + if isList { + raw, err := db.List(resourceType, 0) // 0 = no limit, return all synced data + if err != nil { + return nil, DataProvenance{}, fmt.Errorf("querying local store: %w", err) + } + // Filter out empty/invalid records (empty arrays, null, whitespace-only) + // that can end up in the store from pagination boundary artifacts. + var items []json.RawMessage + for _, r := range raw { + trimmed := strings.TrimSpace(string(r)) + if trimmed == "" || trimmed == "null" || trimmed == "[]" || trimmed == "{}" { + continue + } + items = append(items, r) + } + if len(items) == 0 { + return nil, DataProvenance{}, fmt.Errorf("no local data for %q. Run 'devonthink-pp-cli sync' first", resourceType) + } + // Marshal []json.RawMessage into a single JSON array + data, err := json.Marshal(items) + if err != nil { + return nil, DataProvenance{}, fmt.Errorf("marshaling local data: %w", err) + } + return data, prov, nil + } + + // Get by ID — extract the last path segment as the ID + parts := strings.Split(strings.TrimRight(path, "/"), "/") + id := parts[len(parts)-1] + + item, err := db.Get(resourceType, id) + if err != nil { + if errors.Is(err, sql.ErrNoRows) { + return nil, DataProvenance{}, fmt.Errorf("resource %q with ID %q not found in local store. Run 'devonthink-pp-cli sync' first", resourceType, id) + } + return nil, DataProvenance{}, fmt.Errorf("querying local store: %w", err) + } + return item, prov, nil +} + +// Ensure time import is used (compilation guard). +var _ = time.Now diff --git a/library/productivity/devonthink/internal/cli/deliver.go b/library/productivity/devonthink/internal/cli/deliver.go new file mode 100644 index 0000000000..76162acf99 --- /dev/null +++ b/library/productivity/devonthink/internal/cli/deliver.go @@ -0,0 +1,114 @@ +// Copyright 2026 rowdy and contributors. Licensed under Apache-2.0. See LICENSE. +// Generated by CLI Printing Press (https://github.com/mvanhorn/cli-printing-press). DO NOT EDIT. + +package cli + +import ( + "bytes" + "fmt" + "net/http" + "os" + "path/filepath" + "strings" + "time" +) + +// DeliverSink describes where command output should be routed when +// --deliver is set. Parsed from the sink specifier "scheme:target". +type DeliverSink struct { + Scheme string + Target string +} + +// ParseDeliverSink parses a --deliver value. Supported schemes: +// +// stdout -> default, no redirection +// file: -> write output atomically to +// webhook: -> POST output body to +// +// Returns an error for unknown schemes with a message naming the +// supported set, so agents see a structured refusal rather than a +// silent misroute. +func ParseDeliverSink(spec string) (DeliverSink, error) { + if spec == "" || spec == "stdout" { + return DeliverSink{Scheme: "stdout"}, nil + } + idx := strings.Index(spec, ":") + if idx == -1 { + return DeliverSink{}, fmt.Errorf("unknown --deliver sink %q: expected scheme:target (supported: stdout, file:, webhook:)", spec) + } + scheme := spec[:idx] + target := spec[idx+1:] + switch scheme { + case "file": + if target == "" { + return DeliverSink{}, fmt.Errorf("--deliver file: requires a path") + } + case "webhook": + if !strings.HasPrefix(target, "http://") && !strings.HasPrefix(target, "https://") { + return DeliverSink{}, fmt.Errorf("--deliver webhook: requires an http:// or https:// URL, got %q", target) + } + default: + return DeliverSink{}, fmt.Errorf("unknown --deliver scheme %q (supported: stdout, file, webhook)", scheme) + } + return DeliverSink{Scheme: scheme, Target: target}, nil +} + +// Deliver routes a captured output buffer to the configured sink. stdout +// is a no-op because the buffer has already been streamed to stdout via +// the MultiWriter set up in root.go. +func Deliver(sink DeliverSink, body []byte, compact bool) error { + switch sink.Scheme { + case "", "stdout": + return nil + case "file": + return deliverFile(sink.Target, body) + case "webhook": + return deliverWebhook(sink.Target, body, compact) + default: + return fmt.Errorf("unsupported deliver sink %q", sink.Scheme) + } +} + +func deliverFile(path string, body []byte) error { + // Atomic write: tmp + rename. Protects agents from seeing a partial + // file if the process is interrupted mid-write. + dir := filepath.Dir(path) + if dir != "" && dir != "." { + if err := os.MkdirAll(dir, 0o700); err != nil { + return fmt.Errorf("creating deliver dir: %w", err) + } + } + tmp := path + ".tmp" + if err := os.WriteFile(tmp, body, 0o600); err != nil { + return fmt.Errorf("writing deliver tmp: %w", err) + } + if err := os.Rename(tmp, path); err != nil { + return fmt.Errorf("replacing deliver file: %w", err) + } + return nil +} + +func deliverWebhook(url string, body []byte, compact bool) error { + contentType := "application/json" + if compact { + contentType = "application/x-ndjson" + } + req, err := http.NewRequest(http.MethodPost, url, bytes.NewReader(body)) + if err != nil { + return fmt.Errorf("building webhook request: %w", err) + } + req.Header.Set("Content-Type", contentType) + req.Header.Set("User-Agent", "devonthink-pp-cli/deliver") + + client := &http.Client{Timeout: 30 * time.Second} + resp, err := client.Do(req) + if err != nil { + return fmt.Errorf("posting to webhook: %w", err) + } + defer resp.Body.Close() + if resp.StatusCode >= 400 { + return fmt.Errorf("webhook returned %s", resp.Status) + } + return nil +} diff --git a/library/productivity/devonthink/internal/cli/doctor.go b/library/productivity/devonthink/internal/cli/doctor.go new file mode 100644 index 0000000000..ac0b0f0dd2 --- /dev/null +++ b/library/productivity/devonthink/internal/cli/doctor.go @@ -0,0 +1,505 @@ +// Copyright 2026 rowdy and contributors. Licensed under Apache-2.0. See LICENSE. +// Generated by CLI Printing Press (https://github.com/mvanhorn/cli-printing-press). DO NOT EDIT. + +package cli + +import ( + "context" + "database/sql" + "errors" + "fmt" + "io" + "os" + "strings" + "time" + + "github.com/mvanhorn/printing-press-library/library/productivity/devonthink/internal/client" + "github.com/mvanhorn/printing-press-library/library/productivity/devonthink/internal/cliutil" + "github.com/mvanhorn/printing-press-library/library/productivity/devonthink/internal/config" + "github.com/mvanhorn/printing-press-library/library/productivity/devonthink/internal/store" + "github.com/spf13/cobra" +) + +// looksLikeDoctorInterstitial reports whether the response body matches a known +// bot-detection challenge page (Cloudflare, Akamai, Vercel, AWS WAF, DataDome, +// PerimeterX). Only fires on the doctor probe — used to distinguish "transport +// reached the wall" from "transport failed entirely." Returns the vendor name +// when matched, or empty string when no match. +// +// Markers are anchored to or vendor-specific strings to avoid +// false-positives on benign content. For example, a recipe titled "Just A +// Moment of Pause Cookies" must NOT match the Cloudflare challenge marker; +// only "<title>just a moment" (the actual interstitial title) does. +func looksLikeDoctorInterstitial(body []byte) string { + if len(body) == 0 { + return "" + } + limit := len(body) + if limit > 8192 { + limit = 8192 + } + prefix := strings.ToLower(string(body[:limit])) + if !strings.Contains(prefix, "<title") { + // Every bot interstitial we recognize sets a <title>; bodies without + // one are body-only API responses, not challenge pages. + return "" + } + switch { + case strings.Contains(prefix, "<title>just a moment") || // CF JS challenge + strings.Contains(prefix, "challenges.cloudflare.com") || // CF Turnstile + (strings.Contains(prefix, "attention required") && strings.Contains(prefix, "cloudflare")): + return "Cloudflare" + case strings.Contains(prefix, "akamai") && (strings.Contains(prefix, "request unsuccessful") || strings.Contains(prefix, "access denied")): + return "Akamai" + case strings.Contains(prefix, "x-vercel-mitigated") || strings.Contains(prefix, "x-vercel-challenge-token") || + (strings.Contains(prefix, "vercel") && strings.Contains(prefix, "challenge")): + return "Vercel" + case strings.Contains(prefix, "request blocked") && strings.Contains(prefix, "aws waf"): + return "AWS WAF" + case strings.Contains(prefix, "datadome") && (strings.Contains(prefix, "blocked") || strings.Contains(prefix, "captcha") || strings.Contains(prefix, "challenge")): + return "DataDome" + case strings.Contains(prefix, "perimeterx") || strings.Contains(prefix, "px-captcha"): + return "PerimeterX" + } + return "" +} + +// suggestReadCommand walks the Cobra tree to find an endpoint-mirror command +// an operator can run to confirm credentials work end-to-end. Picks the +// first leaf that (a) carries the `pp:endpoint` annotation, so it actually +// dials the API rather than reading a local file like `feedback list` or +// `profile list`; (b) has a list/get verb; and (c) takes no positional +// arguments, so the suggestion is copy-paste runnable. Returns the dotted +// command path (e.g. "issues list") or "" when no such command exists — +// common in mutation-only CLIs and in CLIs where every read command has +// required positional arguments. +func suggestReadCommand(root *cobra.Command) string { + if root == nil { + return "" + } + var found string + var walk func(*cobra.Command, []string) + walk = func(cmd *cobra.Command, path []string) { + if found != "" { + return + } + for _, child := range cmd.Commands() { + childPath := append(append([]string{}, path...), child.Name()) + if isSuggestableReadLeaf(child) { + found = strings.Join(childPath, " ") + return + } + // Recurse even into Hidden parents: printed CLIs mark raw + // resource parents Hidden to keep --help curated, but their + // endpoint leaves remain runnable (`<cli> projects list` + // works). Skipping hidden subtrees would make this return "" + // in nearly every CLI. isSuggestableReadLeaf still rejects a + // leaf that is itself Hidden. + walk(child, childPath) + if found != "" { + return + } + } + } + walk(root, nil) + return found +} + +func isSuggestableReadLeaf(cmd *cobra.Command) bool { + if cmd == nil || cmd.Hidden || cmd.HasSubCommands() || !cmd.Runnable() { + return false + } + // Only endpoint-mirror commands count; framework commands like + // `feedback list` and `profile list` read local files and would + // recreate the false-confidence failure mode the suggestion is + // supposed to avoid. + if cmd.Annotations["pp:endpoint"] == "" { + return false + } + verb := strings.ToLower(strings.SplitN(cmd.Use, " ", 2)[0]) + if verb != "list" && verb != "get" { + return false + } + // Endpoint commands with positional path params advertise them in + // Use as `<id>` (required) or `[id]` (optional). The runtime body + // rejects empty args by printing help, so suggesting one would not + // actually exercise the token — reject before the Args probe below. + if strings.ContainsAny(cmd.Use, "<[") { + return false + } + // Probe the Args validator with an empty positional-arg list. A nil + // validator accepts anything (including zero args); a non-nil validator + // that returns nil for [] accepts zero args. Either qualifies — the + // suggestion `<cli> list` is then a complete command. + if cmd.Args == nil { + return true + } + return cmd.Args(cmd, []string{}) == nil +} + +func newDoctorCmd(flags *rootFlags) *cobra.Command { + var failOn string + cmd := &cobra.Command{ + Use: "doctor", + Short: "Check CLI health", + Example: ` devonthink-pp-cli doctor + devonthink-pp-cli doctor --json + devonthink-pp-cli doctor --fail-on warn`, + RunE: func(cmd *cobra.Command, args []string) error { + report := map[string]any{} + + // Check config + cfg, err := config.Load(flags.configPath) + if err != nil { + report["config"] = fmt.Sprintf("error: %s", err) + } else { + report["config"] = "ok" + report["config_path"] = cfg.Path + report["base_url"] = cfg.BaseURL + } + + // Check auth + report["auth"] = "not required" + + // Check auth environment variables + + // Check API connectivity and validate credentials. + // + // The doctor uses the same client every other command uses -- + // flags.newClient() returns a *client.Client wrapping whatever + // transport the spec declared (Surf for browser-chrome, stdlib + // for standard). A separate stdlib http.Client would silently + // bypass that choice and report false negatives against + // Cloudflare-fronted, Akamai-fronted, or otherwise bot-detected + // sites. By going through flags.newClient(), the doctor's + // reachability verdict matches what real commands experience. + if cfg != nil && cfg.BaseURL != "" { + c, clientErr := flags.newClient() + if clientErr != nil { + report["api"] = fmt.Sprintf("client init error: %s", clientErr) + } else { + // Step 1: Basic reachability via the configured transport. + reachBody, reachErr := c.Get(cmd.Context(), "/", nil) + var reachAPIErr *client.APIError + switch { + case reachErr == nil: + // 2xx response — clearly reachable. Still inspect the + // body for a known interstitial; some bot walls return + // 200 with a JS challenge page. + if vendor := looksLikeDoctorInterstitial(reachBody); vendor != "" { + report["api"] = fmt.Sprintf("blocked by %s interstitial — the configured transport reached the wall. Try a different network, wait for the IP-level rate limit to clear, or check that the browser-chrome transport is bound correctly.", vendor) + } else { + report["api"] = "reachable" + } + case errors.As(reachErr, &reachAPIErr): + // Non-2xx from the server. The network reached, the + // server responded — that's "reachable" for our + // purposes. Inspect the response body for a known + // interstitial first; otherwise note the status. + status := reachAPIErr.StatusCode + if vendor := looksLikeDoctorInterstitial([]byte(reachAPIErr.Body)); vendor != "" { + report["api"] = fmt.Sprintf("blocked by %s interstitial (HTTP %d) — the configured transport reached the wall.", vendor, status) + } else { + report["api"] = fmt.Sprintf("reachable (HTTP %d at /)", status) + } + default: + // Network-level failure: DNS, connection refused, TLS, + // transport init, etc. The transport itself didn't + // connect. + report["api"] = fmt.Sprintf("unreachable: %s", reachErr) + } + + // Step 2: Validate credentials with an authenticated probe. + authHeader := cfg.AuthHeader() + if authHeader == "" { + // No auth configured — skip credential validation + } else if reachErr != nil && !errors.As(reachErr, &reachAPIErr) { + report["credentials"] = "skipped (API unreachable)" + } else { + suggestion := suggestReadCommand(cmd.Root()) + if suggestion != "" { + report["credentials"] = fmt.Sprintf("present, not verified. Run `%s %s` to confirm the token works end-to-end.", "devonthink-pp-cli", suggestion) + } else { + report["credentials"] = "present, not verified. Run any read command to confirm the token works end-to-end." + } + } + } + } else if cfg != nil && cfg.BaseURL == "" { + report["api"] = "not configured (set base_url in config file)" + } + // Cache health: only reported when this CLI has a local store. + // Surfaces rows + last_synced_at per resource, schema version, + // and a fresh/stale/unknown verdict so agents can introspect + // whether to trust the cached data before issuing queries. + report["cache"] = collectCacheReport(cmd.Context(), "") + + // Verify mode state. Surfaced so an operator who unintentionally + // inherits PRINTING_PRESS_VERIFY=1 (parent shell, CI runner, container + // image) detects the foot-gun without inspecting a response body. + // Pairs with the synthetic envelope's verify_noop / reason literals + // as a second diagnosis anchor. + if cliutil.IsVerifyEnv() { + if cliutil.IsVerifyLiveHTTPEnv() { + report["verify_mode"] = "INFO ACTIVE — live HTTP opt-in (mutating verbs dial out)" + } else { + report["verify_mode"] = "INFO ACTIVE — mutating HTTP verbs short-circuit (PRINTING_PRESS_VERIFY=1; no network calls for DELETE/POST/PUT/PATCH)" + } + } else { + report["verify_mode"] = "normal operation" + } + + report["version"] = version + + if flags.asJSON { + if err := printJSONFiltered(cmd.OutOrStdout(), report, flags); err != nil { + return err + } + return doctorExitForFailOn(failOn, report) + } + + // Human-readable output with color + w := cmd.OutOrStdout() + checkKeys := []struct{ key, label string }{ + {"config", "Config"}, + {"auth", "Auth"}, + {"env_vars", "Env Vars"}, + {"verify_mode", "Verify Mode"}, + {"api", "API"}, + {"credentials", "Credentials"}, + } + for _, ck := range checkKeys { + v, ok := report[ck.key] + if !ok { + continue + } + s := fmt.Sprintf("%v", v) + indicator := green("OK") + switch { + case strings.HasPrefix(s, "INFO"): + indicator = yellow("INFO") + case strings.HasPrefix(s, "ERROR"): + indicator = red("FAIL") + case strings.HasPrefix(s, "optional"): + // Optional-auth CLI with no key set — informational, not a failure. + indicator = yellow("INFO") + case strings.Contains(s, "scope-limited"): + indicator = yellow("WARN") + case strings.Contains(s, "not verified"): + // "present, not verified" — credentials are loaded but no + // probe ran. Informational, not a warning; a clean config + // shouldn't render yellow WARN in CI dashboards. + indicator = yellow("INFO") + case strings.Contains(s, "error") || strings.Contains(s, "not configured") || strings.Contains(s, "unreachable") || strings.Contains(s, "invalid") || strings.Contains(s, "missing"): + indicator = red("FAIL") + case s == "not required": + // Public APIs: no auth needed is a healthy state, not a warning. + indicator = green("OK") + case strings.Contains(s, "not ") || strings.Contains(s, "skipped") || strings.Contains(s, "inferred"): + indicator = yellow("WARN") + } + fmt.Fprintf(w, " %s %s: %s\n", indicator, ck.label, s) + } + // Print info keys without status indicator + for _, key := range []string{"config_path", "base_url", "auth_source", "version"} { + if v, ok := report[key]; ok { + fmt.Fprintf(w, " %s: %v\n", key, v) + } + } + // Print auth setup hints (indented under Auth line) + // Cache section: render after the primary health block so it + // sits next to version info, mirroring the JSON report layout. + if cacheAny, ok := report["cache"]; ok { + if cacheRep, ok := cacheAny.(map[string]any); ok { + renderCacheReport(w, cacheRep) + } + } + return doctorExitForFailOn(failOn, report) + }, + } + cmd.Flags().StringVar(&failOn, "fail-on", "", "Exit non-zero when a health level is reached: stale, error. Default is never.") + return cmd +} + +// doctorExitForFailOn returns a non-nil error when the report's worst +// status meets or exceeds the --fail-on threshold. "error" always trips +// when any section reports an error; "stale" also trips when the cache +// section is stale. The default empty string means never fail on status. +func doctorExitForFailOn(failOn string, report map[string]any) error { + if failOn == "" { + return nil + } + worstError := false + worstStale := false + for _, v := range report { + s, ok := v.(string) + if ok { + if strings.Contains(s, "error") || strings.Contains(s, "unreachable") || strings.Contains(s, "invalid") || strings.Contains(s, "missing") { + worstError = true + } + } + if m, ok := v.(map[string]any); ok { + if st, _ := m["status"].(string); st == "error" { + worstError = true + } else if st == "stale" { + worstStale = true + } + } + } + switch failOn { + case "error": + if worstError { + return fmt.Errorf("doctor: --fail-on=error triggered") + } + case "stale": + if worstError || worstStale { + return fmt.Errorf("doctor: --fail-on=stale triggered") + } + default: + return fmt.Errorf("doctor: unknown --fail-on value %q (valid: stale, error)", failOn) + } + return nil +} + +// collectCacheReport opens the local store, reads per-resource sync state, +// and returns a map summarising cache health. Never panics on missing DB +// or open failure; returns a map with status=unknown or status=error so the +// caller can render and agents can interpret. +// +// staleAfterSpec is the CLI's configured threshold (e.g. "6h"); empty means +// use the runtime default. The default is deliberately conservative (6h) +// because the alternative is no freshness story at all. +func collectCacheReport(ctx context.Context, staleAfterSpec string) map[string]any { + report := map[string]any{} + dbPath := defaultDBPath("devonthink-pp-cli") + report["db_path"] = dbPath + + fi, err := os.Stat(dbPath) + if err != nil { + if os.IsNotExist(err) { + report["status"] = "unknown" + report["hint"] = "Database not created yet; run 'devonthink-pp-cli sync' to hydrate." + return report + } + report["status"] = "error" + report["error"] = err.Error() + return report + } + report["db_bytes"] = fi.Size() + + s, err := store.OpenWithContext(ctx, dbPath) + if err != nil { + report["status"] = "error" + report["error"] = err.Error() + return report + } + defer s.Close() + + if v, verr := s.SchemaVersion(); verr == nil { + report["schema_version"] = v + } + + staleAfter := 6 * time.Hour + if staleAfterSpec != "" { + if d, derr := time.ParseDuration(staleAfterSpec); derr == nil { + staleAfter = d + } + } + + rows, qerr := s.DB().Query(`SELECT resource_type, COALESCE(total_count, 0), last_synced_at FROM sync_state ORDER BY resource_type`) + if qerr != nil { + // sync_state may not exist on a fresh DB that has migrated but not + // yet had any sync runs — treat as unknown rather than error. + report["status"] = "unknown" + report["hint"] = "No sync state recorded; run 'devonthink-pp-cli sync' to populate." + return report + } + defer rows.Close() + + var resources []map[string]any + fresh := true + haveAny := false + oldest := time.Duration(0) + for rows.Next() { + var rtype string + var count int64 + var lastSynced sql.NullTime + if err := rows.Scan(&rtype, &count, &lastSynced); err != nil { + continue + } + r := map[string]any{"type": rtype, "rows": count} + if lastSynced.Valid { + haveAny = true + r["last_synced_at"] = lastSynced.Time.UTC().Format(time.RFC3339) + age := time.Since(lastSynced.Time) + r["staleness"] = age.Round(time.Minute).String() + if age > staleAfter { + fresh = false + } + if age > oldest { + oldest = age + } + } else { + r["staleness"] = "never" + fresh = false + } + resources = append(resources, r) + } + report["resources"] = resources + report["stale_after"] = staleAfter.String() + + switch { + case !haveAny && len(resources) == 0: + report["status"] = "unknown" + report["hint"] = "sync_state is empty; run 'devonthink-pp-cli sync' to hydrate." + case fresh: + report["status"] = "fresh" + default: + report["status"] = "stale" + report["oldest_age"] = oldest.Round(time.Minute).String() + report["hint"] = "Some resources are older than stale_after; run 'devonthink-pp-cli sync' to refresh." + } + return report +} + +func renderCacheReport(w io.Writer, rep map[string]any) { + status, _ := rep["status"].(string) + indicator := green("OK") + switch status { + case "stale": + indicator = yellow("WARN") + case "error": + indicator = red("FAIL") + case "unknown": + indicator = yellow("INFO") + } + fmt.Fprintf(w, " %s Cache: %s\n", indicator, status) + if v, ok := rep["db_path"]; ok { + fmt.Fprintf(w, " db_path: %v\n", v) + } + if v, ok := rep["schema_version"]; ok { + fmt.Fprintf(w, " schema_version: %v\n", v) + } + if v, ok := rep["db_bytes"]; ok { + fmt.Fprintf(w, " db_bytes: %v\n", v) + } + if v, ok := rep["stale_after"]; ok { + fmt.Fprintf(w, " stale_after: %v\n", v) + } + if v, ok := rep["oldest_age"]; ok { + fmt.Fprintf(w, " oldest_age: %v\n", v) + } + if resourcesAny, ok := rep["resources"]; ok { + if resources, ok := resourcesAny.([]map[string]any); ok && len(resources) > 0 { + fmt.Fprintf(w, " resources:\n") + for _, r := range resources { + rtype, _ := r["type"].(string) + rows := r["rows"] + staleness, _ := r["staleness"].(string) + fmt.Fprintf(w, " - %s: %v rows, %s\n", rtype, rows, staleness) + } + } + } + if hint, ok := rep["hint"]; ok { + fmt.Fprintf(w, " hint: %v\n", hint) + } +} diff --git a/library/productivity/devonthink/internal/cli/export.go b/library/productivity/devonthink/internal/cli/export.go new file mode 100644 index 0000000000..aecaf6647c --- /dev/null +++ b/library/productivity/devonthink/internal/cli/export.go @@ -0,0 +1,122 @@ +// Copyright 2026 rowdy and contributors. Licensed under Apache-2.0. See LICENSE. +// Generated by CLI Printing Press (https://github.com/mvanhorn/cli-printing-press). DO NOT EDIT. + +package cli + +import ( + "bufio" + "encoding/json" + "fmt" + "os" + "strings" + + "github.com/spf13/cobra" +) + +func newExportCmd(flags *rootFlags) *cobra.Command { + var format string + var outputFile string + var limit int + var noCache bool + + cmd := &cobra.Command{ + Use: "export <resource> [id]", + Short: "Export data to JSONL or JSON for backup, migration, or analysis", + Long: `Export paginated API data to a local file. Supports JSONL (one JSON object +per line, streaming-friendly) and JSON (array). JSONL is recommended for +large datasets as it has no memory pressure.`, + Example: ` # Export all items as JSONL (streaming, recommended for large datasets) + devonthink-pp-cli export <resource> --format jsonl --output data.jsonl + + # Export with limit + devonthink-pp-cli export <resource> --format jsonl --limit 1000 + + # Pipe to another tool + devonthink-pp-cli export <resource> --format jsonl | jq '.id'`, + Args: cobra.MinimumNArgs(1), + RunE: func(cmd *cobra.Command, args []string) error { + validResources := map[string]bool{ + "databases": true, + "ledger": true, + "selection": true, + } + validResourceList := []string{ + "databases", + "ledger", + "selection", + } + resource := args[0] + if !validResources[resource] { + return usageErr(fmt.Errorf("unknown resource %q; valid: %s", resource, strings.Join(validResourceList, ", "))) + } + + c, err := flags.newClient() + if err != nil { + return err + } + if noCache { + c.NoCache = true + } + + path := "/" + resource + if len(args) > 1 { + path += "/" + args[1] + } + + var writer *bufio.Writer + if outputFile != "" { + f, err := os.Create(outputFile) // #nosec G304 -- export intentionally writes to a user-selected local path. + if err != nil { + return fmt.Errorf("creating output file: %w", err) + } + defer f.Close() + writer = bufio.NewWriter(f) + defer writer.Flush() + } else { + writer = bufio.NewWriter(os.Stdout) + defer writer.Flush() + } + + data, err := c.Get(cmd.Context(), path, nil) + if err != nil { + return classifyAPIError(err, flags) + } + + switch format { + case "jsonl": + var items []json.RawMessage + if err := json.Unmarshal(data, &items); err != nil { + fmt.Fprintln(writer, string(data)) + return nil + } + count := 0 + for _, item := range items { + if limit > 0 && count >= limit { + break + } + fmt.Fprintln(writer, string(item)) + count++ + } + if outputFile != "" { + fmt.Fprintf(os.Stderr, "Exported %d records to %s\n", count, outputFile) + } + default: + enc := json.NewEncoder(writer) + enc.SetIndent("", " ") + var parsed any + if err := json.Unmarshal(data, &parsed); err != nil { + return err + } + return enc.Encode(parsed) + } + return nil + }, + } + + cmd.Flags().StringVar(&format, "format", "jsonl", "Output format: jsonl or json") + cmd.Flags().StringVarP(&outputFile, "output", "o", "", "Output file path (default: stdout)") + cmd.Flags().IntVar(&limit, "limit", 0, "Maximum records to export (0 = unlimited)") + cmd.Flags().BoolVar(&noCache, "no-cache", false, "Bypass response cache for fresh data") + + return cmd +} diff --git a/library/productivity/devonthink/internal/cli/feedback.go b/library/productivity/devonthink/internal/cli/feedback.go new file mode 100644 index 0000000000..642c17d341 --- /dev/null +++ b/library/productivity/devonthink/internal/cli/feedback.go @@ -0,0 +1,228 @@ +// Copyright 2026 rowdy and contributors. Licensed under Apache-2.0. See LICENSE. +// Generated by CLI Printing Press (https://github.com/mvanhorn/cli-printing-press). DO NOT EDIT. + +package cli + +import ( + "bytes" + "encoding/json" + "fmt" + "io" + "net/http" + "os" + "path/filepath" + "strings" + "time" + + "github.com/spf13/cobra" +) + +// FeedbackEntry is one line in the local feedback ledger. Every run of +// the feedback command appends one entry; upstream POST is a separate, +// optional step. +type FeedbackEntry struct { + Text string `json:"text"` + CLI string `json:"cli"` + Version string `json:"version"` + AgentID string `json:"agent_id,omitempty"` + Timestamp time.Time `json:"timestamp"` +} + +const feedbackMaxTextLen = 4096 + +func feedbackFilePath() (string, error) { + home, err := os.UserHomeDir() + if err != nil { + return "", fmt.Errorf("resolving home dir: %w", err) + } + dir := filepath.Join(home, ".local", "share", "devonthink-pp-cli") + if err := os.MkdirAll(dir, 0o700); err != nil { + return "", fmt.Errorf("creating state dir: %w", err) + } + return filepath.Join(dir, "feedback.jsonl"), nil +} + +// FeedbackEndpointConfigured reports whether an upstream feedback URL +// is available. Surfaced via agent-context so introspecting agents know +// whether their feedback will ship upstream. +func FeedbackEndpointConfigured() bool { + return os.Getenv("DEVONTHINK_FEEDBACK_ENDPOINT") != "" +} + +func feedbackEndpoint() string { + return os.Getenv("DEVONTHINK_FEEDBACK_ENDPOINT") +} + +func feedbackAutoSend() bool { + v := strings.ToLower(strings.TrimSpace(os.Getenv("DEVONTHINK_FEEDBACK_AUTO_SEND"))) + return v == "1" || v == "true" || v == "yes" +} + +func appendFeedback(entry FeedbackEntry) error { + p, err := feedbackFilePath() + if err != nil { + return err + } + f, err := os.OpenFile(p, os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0o600) // #nosec G304 -- feedback ledger path is resolved under the CLI state directory. + if err != nil { + return fmt.Errorf("opening feedback ledger: %w", err) + } + defer f.Close() + return json.NewEncoder(f).Encode(entry) +} + +func postFeedback(url string, entry FeedbackEntry) error { + body, err := json.Marshal(entry) + if err != nil { + return err + } + req, err := http.NewRequest(http.MethodPost, url, bytes.NewReader(body)) + if err != nil { + return fmt.Errorf("building feedback request: %w", err) + } + req.Header.Set("Content-Type", "application/json") + req.Header.Set("User-Agent", "devonthink-pp-cli/feedback") + client := &http.Client{Timeout: 15 * time.Second} + resp, err := client.Do(req) + if err != nil { + return fmt.Errorf("posting feedback: %w", err) + } + defer resp.Body.Close() + if resp.StatusCode >= 400 { + return fmt.Errorf("feedback endpoint returned %s", resp.Status) + } + return nil +} + +func newFeedbackCmd(flags *rootFlags) *cobra.Command { + var useStdin bool + var send bool + cmd := &cobra.Command{ + Use: "feedback [text]", + Short: "Record feedback about this CLI (local by default; upstream opt-in)", + Long: `Feedback is captured locally first at ~/.local/share/devonthink-pp-cli/feedback.jsonl. +When ` + "`DEVONTHINK_FEEDBACK_ENDPOINT`" + ` is set and either --send is +passed or ` + "`DEVONTHINK_FEEDBACK_AUTO_SEND=true`" + `, the entry is +POSTed as JSON after the local write. + +Write what surprised you or tripped you up, not a bug report. The +loop is: agent notices friction -> one invocation -> captured -> the +maintainer sees it.`, + RunE: func(cmd *cobra.Command, args []string) error { + var text string + if useStdin { + data, err := io.ReadAll(cmd.InOrStdin()) + if err != nil { + return fmt.Errorf("reading stdin: %w", err) + } + text = strings.TrimSpace(string(data)) + } else if len(args) > 0 { + text = strings.Join(args, " ") + } + text = strings.TrimSpace(text) + if text == "" { + return fmt.Errorf("feedback text is empty (pass arguments or --stdin)") + } + truncated := false + if len(text) > feedbackMaxTextLen { + text = text[:feedbackMaxTextLen] + truncated = true + } + + entry := FeedbackEntry{ + Text: text, + CLI: "devonthink-pp-cli", + Version: version, + AgentID: os.Getenv("AGENT_ID"), + Timestamp: time.Now().UTC(), + } + if err := appendFeedback(entry); err != nil { + return err + } + + upstreamResult := map[string]any{"sent": false} + if endpoint := feedbackEndpoint(); endpoint != "" && (send || feedbackAutoSend()) { + if err := postFeedback(endpoint, entry); err != nil { + fmt.Fprintf(cmd.ErrOrStderr(), "warning: feedback upstream POST failed: %v\n", err) + upstreamResult["sent"] = false + upstreamResult["error"] = err.Error() + } else { + upstreamResult["sent"] = true + upstreamResult["endpoint"] = endpoint + } + } + + if flags.asJSON { + return printJSONFiltered(cmd.OutOrStdout(), map[string]any{ + "recorded": true, + "truncated": truncated, + "upstream": upstreamResult, + "entry": entry, + }, flags) + } + fmt.Fprintf(cmd.OutOrStdout(), "feedback recorded locally (%d chars%s)\n", len(text), func() string { + if truncated { + return ", truncated" + } + return "" + }()) + if sent, _ := upstreamResult["sent"].(bool); sent { + fmt.Fprintf(cmd.OutOrStdout(), "upstream POST: %v\n", upstreamResult["endpoint"]) + } + return nil + }, + } + cmd.Flags().BoolVar(&useStdin, "stdin", false, "Read feedback body from stdin rather than arguments") + cmd.Flags().BoolVar(&send, "send", false, "POST to the configured feedback endpoint in addition to local write") + + cmd.AddCommand(newFeedbackListCmd(flags)) + return cmd +} + +func newFeedbackListCmd(flags *rootFlags) *cobra.Command { + var limit int + cmd := &cobra.Command{ + Use: "list", + Short: "List recent feedback entries", + Annotations: map[string]string{ + "mcp:read-only": "true", + }, + Example: ` devonthink-pp-cli feedback list + devonthink-pp-cli feedback list --limit 5 + devonthink-pp-cli feedback list --json`, + RunE: func(cmd *cobra.Command, _ []string) error { + p, err := feedbackFilePath() + if err != nil { + return err + } + data, err := os.ReadFile(p) // #nosec G304 -- feedback ledger path is resolved under the CLI state directory. + if err != nil { + if os.IsNotExist(err) { + if flags.asJSON { + return printJSONFiltered(cmd.OutOrStdout(), []FeedbackEntry{}, flags) + } + return nil + } + return err + } + var entries []FeedbackEntry + for _, line := range strings.Split(string(data), "\n") { + line = strings.TrimSpace(line) + if line == "" { + continue + } + var e FeedbackEntry + if err := json.Unmarshal([]byte(line), &e); err != nil { + continue + } + entries = append(entries, e) + } + if limit > 0 && limit < len(entries) { + entries = entries[len(entries)-limit:] + } + return printJSONFiltered(cmd.OutOrStdout(), entries, flags) + }, + } + cmd.Flags().IntVar(&limit, "limit", 20, "Maximum number of recent entries to return") + return cmd +} diff --git a/library/productivity/devonthink/internal/cli/graph.go b/library/productivity/devonthink/internal/cli/graph.go new file mode 100644 index 0000000000..958c0cec24 --- /dev/null +++ b/library/productivity/devonthink/internal/cli/graph.go @@ -0,0 +1,22 @@ +// Copyright 2026 rowdy and contributors. Licensed under Apache-2.0. See LICENSE. +// Generated by CLI Printing Press (https://github.com/mvanhorn/cli-printing-press). DO NOT EDIT. + +package cli + +import ( + "github.com/spf13/cobra" +) + +func newGraphCmd(flags *rootFlags) *cobra.Command { + cmd := &cobra.Command{ + Use: "graph", + Short: "Links, mentions, and knowledge graph health", + Hidden: true, + Annotations: map[string]string{"mcp:read-only": "true"}, + RunE: parentNoSubcommandRunE(flags), + } + + cmd.AddCommand(newGraphAuditCmd(flags)) + cmd.AddCommand(newGraphLinksCmd(flags)) + return cmd +} diff --git a/library/productivity/devonthink/internal/cli/graph_audit.go b/library/productivity/devonthink/internal/cli/graph_audit.go new file mode 100644 index 0000000000..85e7e62078 --- /dev/null +++ b/library/productivity/devonthink/internal/cli/graph_audit.go @@ -0,0 +1,91 @@ +// Copyright 2026 rowdy and contributors. Licensed under Apache-2.0. See LICENSE. +// Generated by CLI Printing Press (https://github.com/mvanhorn/cli-printing-press). DO NOT EDIT. + +package cli + +import ( + "encoding/json" + "fmt" + "os" + + "github.com/spf13/cobra" +) + +func newGraphAuditCmd(flags *rootFlags) *cobra.Command { + var flagDatabase string + var flagLimit int + + cmd := &cobra.Command{ + Use: "audit", + Short: "Detect orphans, unresolved wiki links, weak hubs, and tag-only clusters", + Example: " devonthink-pp-cli graph audit --database Research --agent --select issues,type,count,samples", + Annotations: map[string]string{"pp:endpoint": "graph.audit", "pp:method": "GET", "pp:path": "/graph/audit", "mcp:read-only": "true"}, + RunE: func(cmd *cobra.Command, args []string) error { + c, err := flags.newClient() + if err != nil { + return err + } + + path := "/graph/audit" + params := map[string]string{} + if flagDatabase != "" { + params["database"] = formatCLIParamValue(flagDatabase) + } + if flagLimit != 0 { + params["limit"] = formatCLIParamValue(flagLimit) + } + data, prov, err := resolveReadWithStrategy(cmd.Context(), c, flags, "auto", "graph", false, path, params, nil, cmd.ErrOrStderr()) + if err != nil { + return classifyAPIError(err, flags) + } + // Honor --limit when the API accepts but ignores ?limit=N. + data = truncateJSONArray(data, flagLimit) + // Print provenance to stderr for human-facing output only. + // Machine-format flags (--json, --csv, --compact, --quiet, --plain, + // --select) and piped stdout suppress this line; the JSON envelope + // already carries meta.source for those consumers. + // SYNC: keep this gate aligned with command_promoted.go.tmpl. + if wantsHumanTable(cmd.OutOrStdout(), flags) { + var countItems []json.RawMessage + _ = json.Unmarshal(data, &countItems) + printProvenance(cmd, len(countItems), prov) + } + // For JSON output, wrap with provenance envelope before passing through flags. + // --select wins over --compact when both are set; --compact only runs when + // no explicit fields were requested. Explicit format flags (--csv, --quiet, + // --plain) opt out of the auto-JSON path so piped consumers that asked for + // a non-JSON format reach the standard pipeline below. + if flags.asJSON || (!isTerminal(cmd.OutOrStdout()) && !flags.csv && !flags.quiet && !flags.plain) { + filtered := data + if flags.selectFields != "" { + filtered = filterFields(filtered, flags.selectFields) + } else if flags.compact { + filtered = compactFields(filtered) + } + wrapped, wrapErr := wrapWithProvenance(filtered, prov) + if wrapErr != nil { + return wrapErr + } + return printOutput(cmd.OutOrStdout(), wrapped, true) + } + // For all other output modes (table, csv, plain, quiet), use the standard pipeline + if wantsHumanTable(cmd.OutOrStdout(), flags) { + var items []map[string]any + if json.Unmarshal(data, &items) == nil && len(items) > 0 { + if err := printAutoTable(cmd.OutOrStdout(), items); err != nil { + return err + } + if len(items) >= 25 { + fmt.Fprintf(os.Stderr, "\nShowing %d results. To narrow: add --limit, --json --select, or filter flags.\n", len(items)) + } + return nil + } + } + return printOutputWithFlags(cmd.OutOrStdout(), data, flags) + }, + } + cmd.Flags().StringVar(&flagDatabase, "database", "", "Database name or UUID") + cmd.Flags().IntVar(&flagLimit, "limit", 50, "Maximum sample records per issue") + + return cmd +} diff --git a/library/productivity/devonthink/internal/cli/graph_links.go b/library/productivity/devonthink/internal/cli/graph_links.go new file mode 100644 index 0000000000..e17378d73a --- /dev/null +++ b/library/productivity/devonthink/internal/cli/graph_links.go @@ -0,0 +1,94 @@ +// Copyright 2026 rowdy and contributors. Licensed under Apache-2.0. See LICENSE. +// Generated by CLI Printing Press (https://github.com/mvanhorn/cli-printing-press). DO NOT EDIT. + +package cli + +import ( + "encoding/json" + "fmt" + "os" + + "github.com/spf13/cobra" +) + +func newGraphLinksCmd(flags *rootFlags) *cobra.Command { + var flagUuid string + var flagDatabase string + var flagDirection string + + cmd := &cobra.Command{ + Use: "links", + Short: "List item links, wiki links, mentions, and unresolved wiki names", + Example: " devonthink-pp-cli graph links", + Annotations: map[string]string{"pp:endpoint": "graph.links", "pp:method": "GET", "pp:path": "/graph/links", "mcp:read-only": "true"}, + RunE: func(cmd *cobra.Command, args []string) error { + c, err := flags.newClient() + if err != nil { + return err + } + + path := "/graph/links" + params := map[string]string{} + if flagUuid != "" { + params["uuid"] = formatCLIParamValue(flagUuid) + } + if flagDatabase != "" { + params["database"] = formatCLIParamValue(flagDatabase) + } + if flagDirection != "" { + params["direction"] = formatCLIParamValue(flagDirection) + } + data, prov, err := resolveReadWithStrategy(cmd.Context(), c, flags, "auto", "graph", false, path, params, nil, cmd.ErrOrStderr()) + if err != nil { + return classifyAPIError(err, flags) + } + // Print provenance to stderr for human-facing output only. + // Machine-format flags (--json, --csv, --compact, --quiet, --plain, + // --select) and piped stdout suppress this line; the JSON envelope + // already carries meta.source for those consumers. + // SYNC: keep this gate aligned with command_promoted.go.tmpl. + if wantsHumanTable(cmd.OutOrStdout(), flags) { + var countItems []json.RawMessage + _ = json.Unmarshal(data, &countItems) + printProvenance(cmd, len(countItems), prov) + } + // For JSON output, wrap with provenance envelope before passing through flags. + // --select wins over --compact when both are set; --compact only runs when + // no explicit fields were requested. Explicit format flags (--csv, --quiet, + // --plain) opt out of the auto-JSON path so piped consumers that asked for + // a non-JSON format reach the standard pipeline below. + if flags.asJSON || (!isTerminal(cmd.OutOrStdout()) && !flags.csv && !flags.quiet && !flags.plain) { + filtered := data + if flags.selectFields != "" { + filtered = filterFields(filtered, flags.selectFields) + } else if flags.compact { + filtered = compactFields(filtered) + } + wrapped, wrapErr := wrapWithProvenance(filtered, prov) + if wrapErr != nil { + return wrapErr + } + return printOutput(cmd.OutOrStdout(), wrapped, true) + } + // For all other output modes (table, csv, plain, quiet), use the standard pipeline + if wantsHumanTable(cmd.OutOrStdout(), flags) { + var items []map[string]any + if json.Unmarshal(data, &items) == nil && len(items) > 0 { + if err := printAutoTable(cmd.OutOrStdout(), items); err != nil { + return err + } + if len(items) >= 25 { + fmt.Fprintf(os.Stderr, "\nShowing %d results. To narrow: add --limit, --json --select, or filter flags.\n", len(items)) + } + return nil + } + } + return printOutputWithFlags(cmd.OutOrStdout(), data, flags) + }, + } + cmd.Flags().StringVar(&flagUuid, "uuid", "", "Record UUID or item link") + cmd.Flags().StringVar(&flagDatabase, "database", "", "Database name or UUID") + cmd.Flags().StringVar(&flagDirection, "direction", "both", "incoming, outgoing, or both") + + return cmd +} diff --git a/library/productivity/devonthink/internal/cli/helpers.go b/library/productivity/devonthink/internal/cli/helpers.go new file mode 100644 index 0000000000..688d3f859b --- /dev/null +++ b/library/productivity/devonthink/internal/cli/helpers.go @@ -0,0 +1,1835 @@ +// Copyright 2026 rowdy and contributors. Licensed under Apache-2.0. See LICENSE. +// Generated by CLI Printing Press (https://github.com/mvanhorn/cli-printing-press). DO NOT EDIT. + +package cli + +import ( + "bytes" + "context" + "encoding/json" + "errors" + "fmt" + "github.com/mvanhorn/printing-press-library/library/productivity/devonthink/internal/client" + "github.com/spf13/cobra" + "github.com/spf13/pflag" + "io" + "net/url" + "os" + "path/filepath" + "sort" + "strconv" + "strings" + "text/tabwriter" + "time" + "unicode" +) + +var As = errors.As + +const paginatedGetMaxPages = 100 + +func formatCLIParamValue(v any) string { + if f, ok := v.(float64); ok { + return strconv.FormatFloat(f, 'f', -1, 64) + } + return fmt.Sprintf("%v", v) +} + +// noColor is set by the --no-color flag +var noColor bool + +// humanFriendly is set by the --human-friendly flag; colors are off by default (agent-safe) +var humanFriendly bool + +func colorEnabled() bool { + if noColor { + return false + } + if !humanFriendly { + return false + } + if os.Getenv("NO_COLOR") != "" { + return false + } + if os.Getenv("TERM") == "dumb" { + return false + } + return isTerminal(os.Stdout) +} + +func isTerminal(w io.Writer) bool { + if f, ok := w.(*os.File); ok { + fi, err := f.Stat() + if err != nil { + return true + } + return (fi.Mode() & os.ModeCharDevice) != 0 + } + return false +} + +func bold(s string) string { + if !colorEnabled() { + return s + } + return "\033[1m" + s + "\033[0m" +} + +func green(s string) string { + if !colorEnabled() { + return s + } + return "\033[32m" + s + "\033[0m" +} + +func red(s string) string { + if !colorEnabled() { + return s + } + return "\033[31m" + s + "\033[0m" +} + +func yellow(s string) string { + if !colorEnabled() { + return s + } + return "\033[33m" + s + "\033[0m" +} + +type cliError struct { + code int + err error +} + +func (e *cliError) Error() string { return e.err.Error() } +func (e *cliError) Unwrap() error { return e.err } + +func usageErr(err error) error { return &cliError{code: 2, err: err} } +func notFoundErr(err error) error { return &cliError{code: 3, err: err} } +func authErr(err error) error { return &cliError{code: 4, err: err} } +func apiErr(err error) error { return &cliError{code: 5, err: err} } +func configErr(err error) error { return &cliError{code: 10, err: err} } +func rateLimitErr(err error) error { return &cliError{code: 7, err: err} } + +// partialFailureErr signals that the upstream API returned a 2xx with a +// body shape indicating some operations in a batch failed (e.g. Google +// Ads `partialFailureError`, similar shapes from Drive batch, Sheets +// batchUpdate, Cloud Resource Manager). Distinct from apiErr (HTTP-level +// failure) so callers can distinguish "request rejected" from "request +// accepted but some ops failed". +func partialFailureErr(err error) error { return &cliError{code: 6, err: err} } + +// partialFailureReport describes the structured detection result for a +// mutate-style response body. Emitted in the envelope under +// "partial_failure" so machine-readable callers can route per-operation +// remediation. +type partialFailureReport struct { + Field string `json:"field"` + Message string `json:"message,omitempty"` + Code int `json:"code,omitempty"` + Details any `json:"details,omitempty"` + ResourceNames []string `json:"resource_names,omitempty"` +} + +// detectPartialFailure inspects a mutate-style JSON response for a +// partial-failure-shaped field. Returns nil when no partial failure is +// detected. The detector is intentionally generic across APIs that emit +// 2xx-with-batch-errors. New partial-failure-shaped fields are added to +// partialFailureFields, not at call sites. When `results[]` is present +// (Google Ads convention) it extracts per-op `resourceName` so callers +// can see which operations did succeed. +func detectPartialFailure(data []byte) *partialFailureReport { + if len(data) == 0 { + return nil + } + var top map[string]any + if err := json.Unmarshal(data, &top); err != nil { + return nil + } + partialFailureFields := []string{"partialFailureError"} + for _, field := range partialFailureFields { + raw, ok := top[field] + if !ok || raw == nil { + continue + } + obj, ok := raw.(map[string]any) + if !ok { + continue + } + message, _ := obj["message"].(string) + var code int + if n, ok := obj["code"].(float64); ok { + code = int(n) + } + // Empty object means partial-failure mode was off or no ops + // failed; do not flag. + if code == 0 && strings.TrimSpace(message) == "" { + continue + } + report := &partialFailureReport{ + Field: field, + Message: message, + Code: code, + Details: obj["details"], + } + if results, ok := top["results"].([]any); ok { + for _, r := range results { + if rm, ok := r.(map[string]any); ok { + if name, ok := rm["resourceName"].(string); ok && name != "" { + report.ResourceNames = append(report.ResourceNames, name) + } + } + } + } + return report + } + return nil +} + +// dryRunOK reports whether the command should short-circuit without doing any +// real work because --dry-run was set. The verify pipeline probes hand-written +// commands with --dry-run; commands that put validation in cobra's `Args:` or +// `MarkFlagRequired` cannot reach a dry-run guard inside RunE because cobra +// runs those checks before RunE. The verify-friendly pattern for hand-written +// commands is: +// +// RunE: func(cmd *cobra.Command, args []string) error { +// if len(args) == 0 { +// return cmd.Help() +// } +// if dryRunOK(flags) { +// return nil +// } +// // ... real work ... +// } +// +// See SKILL.md "Phase 3: Build The GOAT" for the full pattern. +func dryRunOK(flags *rootFlags) bool { + return flags != nil && flags.dryRun +} + +// boundCtx applies the root --timeout flag to hand-written command work that +// does not go through the generated internal/client.Client. Generated endpoint +// commands already pass flags.timeout into client.New; sibling typed clients +// used by novel commands need an explicit command-level context boundary. +func boundCtx(parent context.Context, flags *rootFlags) (context.Context, context.CancelFunc) { + if flags == nil || flags.timeout <= 0 { + return parent, func() {} + } + return context.WithTimeout(parent, flags.timeout) +} + +// parentNoSubcommandRunE returns a RunE that handles parents invoked without a +// subcommand. In machine output (--json/--agent) the parent emits a structured +// error to stdout listing valid subcommands and exits 2; otherwise cobra's +// default help text is printed. Without this, agents driving the CLI in +// --agent mode received only human-readable help on stdout and exit 0, with no +// signal that the invocation was incomplete. +func parentNoSubcommandRunE(flags *rootFlags) func(*cobra.Command, []string) error { + return func(cmd *cobra.Command, args []string) error { + if flags != nil && flags.asJSON { + subs := make([]string, 0, len(cmd.Commands())) + for _, c := range cmd.Commands() { + if c.IsAvailableCommand() && c.Name() != "help" { + subs = append(subs, c.Name()) + } + } + sort.Strings(subs) + _ = json.NewEncoder(cmd.OutOrStdout()).Encode(map[string]any{ + "error": "subcommand required", + "valid_subcommands": subs, + }) + return usageErr(fmt.Errorf("subcommand required for %q", cmd.CommandPath())) + } + return cmd.Help() + } +} + +// accessWarning describes an API access-denial that sync converts into a +// non-fatal warning. It carries enough structured data for the sync_warning +// JSON event without parsing free-form error strings downstream. +type accessWarning struct { + Status int // HTTP status when applicable; 0 for GraphQL field-level denials. + Reason string // "forbidden" | "insufficient_access" | "unauthenticated" + Message string // human-readable detail (the API's body or GraphQL error message) +} + +// syncErrorJSON returns a one-line JSON sync_error event. When err wraps a +// *client.APIError, the structured status/method/path/body fields let +// operators see the API's response body without parsing a wrapped error +// string — required for diagnosing 4xx responses on endpoints whose OpenAPI +// spec marks filter params optional but the API treats them as required. +// +// parent is non-empty only on the dependent-resource path, where it +// identifies which parent ID's child request failed. +func syncErrorJSON(resource, parent string, err error) string { + payload := struct { + Event string `json:"event"` + Resource string `json:"resource"` + Parent string `json:"parent,omitempty"` + Status int `json:"status,omitempty"` + Method string `json:"method,omitempty"` + Path string `json:"path,omitempty"` + Body string `json:"body,omitempty"` + Error string `json:"error"` + }{ + Event: "sync_error", + Resource: resource, + Parent: parent, + Error: err.Error(), + } + var apiErr *client.APIError + if errors.As(err, &apiErr) { + payload.Status = apiErr.StatusCode + payload.Method = apiErr.Method + payload.Path = apiErr.Path + payload.Body = apiErr.Body + } + out, _ := json.Marshal(payload) + return string(out) +} + +// syncWarningJSON renders a sync_warning event as a single valid JSON line. +// Marshaling (rather than fmt.Fprintf string interpolation) escapes the +// message field, whose value is an upstream error body that may be +// pretty-printed multi-line JSON — embedding it raw broke the NDJSON stream. +func syncWarningJSON(resource, parent string, status int, reason, message string) string { + payload := struct { + Event string `json:"event"` + Resource string `json:"resource"` + Parent string `json:"parent,omitempty"` + // status/reason/message are always present (matching the prior raw + // fmt.Fprintf shape); only parent was conditional. No omitempty so + // consumers parsing the event don't see fields disappear on zero values. + Status int `json:"status"` + Reason string `json:"reason"` + Message string `json:"message"` + }{ + Event: "sync_warning", + Resource: resource, + Parent: parent, + Status: status, + Reason: reason, + Message: message, + } + out, _ := json.Marshal(payload) + return string(out) +} + +// syncUserParams carries user-supplied query parameters injected into sync +// HTTP requests. flatGlobal entries come from --param and inject into +// flat-list requests only; trueGlobal entries come from --global-param and +// inject into every request including dependent path-scoped calls. +// perResource entries win over both on key conflict. +// +// The flat/dependent split avoids a real failure mode: a top-level scope +// like workspace=<gid> belongs on flat-list requests (/projects, /tags) but +// re-injecting it onto a path-scoped dependent request +// (/projects/<gid>/tasks?workspace=<gid>) makes APIs like Asana reject the +// call ("Must specify exactly one of project, tag, ..."). Operators who +// need the old "apply everywhere" semantic opt back in with --global-param. +type syncUserParams struct { + flatGlobal map[string]string + trueGlobal map[string]string + perResource map[string]map[string]string +} + +// parseSyncUserParams parses the repeatable --param key=value, +// --global-param key=value, and --resource-param resource:key=value flags. +// Returns usage errors keyed on the specific invalid token so the user +// sees which entry was rejected. +func parseSyncUserParams(flatGlobalFlags, resourceParamFlags, trueGlobalFlags []string) (*syncUserParams, error) { + flatGlobal, err := parseSyncKVFlags(flatGlobalFlags, "--param") + if err != nil { + return nil, err + } + trueGlobal, err := parseSyncKVFlags(trueGlobalFlags, "--global-param") + if err != nil { + return nil, err + } + p := &syncUserParams{ + flatGlobal: flatGlobal, + trueGlobal: trueGlobal, + perResource: map[string]map[string]string{}, + } + for _, spec := range resourceParamFlags { + resource, kv, ok := strings.Cut(spec, ":") + if !ok || resource == "" { + return nil, fmt.Errorf("invalid --resource-param %q: expected resource:key=value", spec) + } + k, v, ok := strings.Cut(kv, "=") + if !ok || k == "" { + return nil, fmt.Errorf("invalid --resource-param %q: expected resource:key=value", spec) + } + if p.perResource[resource] == nil { + p.perResource[resource] = map[string]string{} + } + p.perResource[resource][k] = v + } + return p, nil +} + +// parseSyncKVFlags parses a slice of "key=value" tokens into a map. The +// flagName label flows into the usage error so a malformed entry tells +// the user which flag was at fault. +func parseSyncKVFlags(flags []string, flagName string) (map[string]string, error) { + out := map[string]string{} + for _, kv := range flags { + k, v, ok := strings.Cut(kv, "=") + if !ok || k == "" { + return nil, fmt.Errorf("invalid %s %q: expected key=value", flagName, kv) + } + out[k] = v + } + return out, nil +} + +// applyTo merges user params into the request map. Called after +// spec-derived params (cursor, since, page-size, dates) so user flags can +// override them. isDependent=true skips flatGlobal (--param), which +// targets flat-list endpoints; trueGlobal (--global-param) and perResource +// always apply. +func (p *syncUserParams) applyTo(resource string, params map[string]string, isDependent bool) { + if p == nil { + return + } + if !isDependent { + for k, v := range p.flatGlobal { + params[k] = v + } + } + for k, v := range p.trueGlobal { + params[k] = v + } + for k, v := range p.perResource[resource] { + params[k] = v + } +} + +// validateResourceNames returns a usage-shaped error when any --resource-param +// key targets a resource not in known. Without this check a typo +// (e.g. --resource-param chanels:mine=true) is a silent no-op: the param +// never matches a real resource and sync proceeds with missing rows. +func (p *syncUserParams) validateResourceNames(known []string) error { + if p == nil || len(p.perResource) == 0 { + return nil + } + set := make(map[string]struct{}, len(known)) + for _, r := range known { + set[r] = struct{}{} + } + var unknown []string + for r := range p.perResource { + if _, ok := set[r]; !ok { + unknown = append(unknown, r) + } + } + if len(unknown) == 0 { + return nil + } + sort.Strings(unknown) + return fmt.Errorf("--resource-param references unknown resource(s): %s (known: %s)", + strings.Join(unknown, ", "), strings.Join(known, ", ")) +} + +// isSyncAccessWarning is a stub: this CLI has no auth, so the API cannot +// reject sync requests on access-policy grounds. Every error stays a hard +// failure. Defining the function unconditionally keeps sync.go agnostic to +// auth presence. +func isSyncAccessWarning(err error) (*accessWarning, bool) { return nil, false } + +type noopResult struct { + Status string `json:"status"` + Reason string `json:"reason"` +} + +func writeNoop(flags *rootFlags, reason, prose string) error { + if flags != nil && flags.asJSON { + return json.NewEncoder(os.Stdout).Encode(noopResult{Status: "noop", Reason: reason}) + } + fmt.Fprintln(os.Stderr, prose) + return nil +} + +func writeAPIErrorEnvelope(flags *rootFlags, err error, code int) { + if flags == nil || !flags.asJSON { + return + } + _ = json.NewEncoder(os.Stdout).Encode(map[string]any{ + "error": err.Error(), + "code": code, + }) +} + +// classifyAPIError maps API errors to structured exit codes with actionable hints. +func classifyAPIError(err error, flags *rootFlags) error { + var typed *cliError + if errors.As(err, &typed) { + return err + } + + msg := err.Error() + switch { + case strings.Contains(msg, "HTTP 409"): + if flags != nil && flags.idempotent { + return writeNoop(flags, "already_exists", "already exists (no-op)") + } + classified := apiErr(err) + writeAPIErrorEnvelope(flags, classified, ExitCode(classified)) + return classified + case strings.Contains(msg, "HTTP 401"): + return authErr(fmt.Errorf("%w\nhint: check your API credentials."+ + "\n Run 'devonthink-pp-cli doctor' to check auth status.", err)) + case strings.Contains(msg, "HTTP 403"): + return authErr(fmt.Errorf("%w\nhint: permission denied. This API is configured without credentials, so the service may be blocking the request by rate limit, geography, bot protection, or endpoint policy."+ + "\n Run 'devonthink-pp-cli doctor' to check connectivity.", err)) + case strings.Contains(msg, "HTTP 404"): + return notFoundErr(fmt.Errorf("%w\nhint: resource not found. Run the 'list' command to see available items", err)) + case strings.Contains(msg, "HTTP 429"): + return rateLimitErr(err) + default: + return apiErr(err) + } +} + +func truncate(s string, max int) string { + if len(s) <= max { + return s + } + if max <= 3 { + return s[:max] + } + return s[:max-3] + "..." +} + +func newTabWriter(w io.Writer) *tabwriter.Writer { + return tabwriter.NewWriter(w, 2, 4, 2, ' ', 0) +} + +// replacePathParam percent-encodes value so path-reserved characters in +// user input do not collapse into extra path segments. +func replacePathParam(path, name, value string) string { + return strings.ReplaceAll(path, "{"+name+"}", url.PathEscape(value)) +} + +// paginatedGet fetches pages and concatenates array results. The headers +// argument carries per-endpoint required headers (e.g. cal-api-version) that +// must be sent on every page request, including the first; pass nil when the +// endpoint has no per-endpoint header overrides. +func paginatedGet(ctx context.Context, c interface { + GetWithHeaders(ctx context.Context, path string, params map[string]string, headers map[string]string) (json.RawMessage, error) +}, path string, params map[string]string, headers map[string]string, fetchAll bool, cursorParam, paginationType, limitParam, nextCursorPath, hasMoreField string) (json.RawMessage, error) { + // Cursor params are exempt from the "0"/"false" strip: offset-paginated + // APIs send offset=0 on the first page. + clean := map[string]string{} + for k, v := range params { + if v == "" { + continue + } + if k == cursorParam || (v != "0" && v != "false") { + clean[k] = v + } + } + + if !fetchAll { + data, err := c.GetWithHeaders(ctx, path, clean, headers) + if err != nil { + return nil, err + } + emitTruncationWarning(data, nextCursorPath, hasMoreField, paginationType) + return data, nil + } + + // Fetch all pages + allItems := make([]json.RawMessage, 0) + page := 0 + for { + page++ + if humanFriendly { + fmt.Fprintf(os.Stderr, "fetching page %d...\n", page) + } else { + fmt.Fprintf(os.Stderr, `{"event":"page_fetch","page":%d}`+"\n", page) + } + + data, err := c.GetWithHeaders(ctx, path, clean, headers) + if err != nil { + return nil, err + } + + // Try to extract items array + var items []json.RawMessage + if json.Unmarshal(data, &items) == nil { + allItems = append(allItems, items...) + if next, ok := nextFullPageOffsetCursor(clean, cursorParam, paginationType, limitParam, len(items)); ok { + if page >= paginatedGetMaxPages { + emitPaginatedGetMaxPagesWarning() + break + } + clean[cursorParam] = next + continue + } + } else { + // Response is an object - look for array inside + var obj map[string]json.RawMessage + if json.Unmarshal(data, &obj) == nil { + itemCount := 0 + if nested, ok := extractPaginatedItems(obj); ok { + allItems = append(allItems, nested...) + itemCount = len(nested) + } + + // Check for next cursor + if nextCursorPath != "" { + if tokenRaw, ok := rawAtPath(obj, nextCursorPath); ok { + if token := paginationCursorToken(tokenRaw); token != "" { + if page >= paginatedGetMaxPages { + emitPaginatedGetMaxPagesWarning() + break + } + clean[cursorParam] = token + continue + } + } + } + + // Check has_more. Page and offset paginators can advance + // client-side; cursor-based APIs still need a body cursor. + hasExplicitNoMore := false + if hasMoreField != "" { + if moreRaw, ok := rawAtPath(obj, hasMoreField); ok { + var more bool + if json.Unmarshal(moreRaw, &more) == nil { + if more { + if next, ok := nextClientSidePaginationCursor(clean, cursorParam, paginationType, limitParam); ok { + if page >= paginatedGetMaxPages { + emitPaginatedGetMaxPagesWarning() + break + } + clean[cursorParam] = next + continue + } + emitMissingPaginationCursorWarning(nextCursorPath) + break + } + hasExplicitNoMore = true + } + } + } + if !hasExplicitNoMore { + if next, ok := nextFullPageOffsetCursor(clean, cursorParam, paginationType, limitParam, itemCount); ok { + if page >= paginatedGetMaxPages { + emitPaginatedGetMaxPagesWarning() + break + } + clean[cursorParam] = next + continue + } + } + } + // No more pages + break + } + + // For direct arrays, can't paginate without cursor + break + } + + if fetchAll && page == 1 && nextCursorPath == "" && hasMoreField == "" { + emitMissingPaginationSignalWarning() + } + if humanFriendly { + fmt.Fprintf(os.Stderr, "fetched %d items across %d pages\n", len(allItems), page) + } else { + fmt.Fprintf(os.Stderr, `{"event":"complete","total":%d,"pages":%d}`+"\n", len(allItems), page) + } + result, _ := json.Marshal(allItems) + return json.RawMessage(result), nil +} + +func nextFullPageOffsetCursor(params map[string]string, cursorParam, paginationType, limitParam string, itemCount int) (string, bool) { + if paginationType != "offset" || itemCount == 0 { + return "", false + } + limit, err := strconv.Atoi(params[limitParam]) + if err != nil || limit <= 0 || itemCount < limit { + return "", false + } + return nextClientSidePaginationCursor(params, cursorParam, paginationType, limitParam) +} + +func nextClientSidePaginationCursor(params map[string]string, cursorParam, paginationType, limitParam string) (string, bool) { + if cursorParam == "" { + return "", false + } + switch paginationType { + case "page": + current := params[cursorParam] + if current == "" { + current = "1" + } + n, err := strconv.Atoi(current) + if err != nil { + return "", false + } + return strconv.Itoa(n + 1), true + case "offset": + current := params[cursorParam] + if current == "" { + current = "0" + } + n, err := strconv.Atoi(current) + if err != nil { + return "", false + } + limit, err := strconv.Atoi(params[limitParam]) + if err != nil || limit <= 0 { + return "", false + } + return strconv.Itoa(n + limit), true + default: + return "", false + } +} + +// Silent page-1 truncation is the worst-possible mode for agents, +// who otherwise compute totals against an incomplete set without +// passing --all. +func emitTruncationWarning(data json.RawMessage, nextCursorPath, hasMoreField, paginationType string) { + if nextCursorPath == "" && hasMoreField == "" { + return + } + var obj map[string]json.RawMessage + if err := json.Unmarshal(data, &obj); err != nil { + return + } + var nextCursor string + if nextCursorPath != "" { + if tokenRaw, ok := rawAtPath(obj, nextCursorPath); ok { + nextCursor = paginationCursorToken(tokenRaw) + } + } + var hasMore bool + if hasMoreField != "" { + if moreRaw, ok := rawAtPath(obj, hasMoreField); ok { + _ = json.Unmarshal(moreRaw, &hasMore) + } + } + if nextCursor == "" && !hasMore { + return + } + // --all advances when a next-cursor is configured, or when the endpoint + // uses client-side numeric page/offset advancement. Opaque cursor APIs + // still need a returned cursor to advance safely. + if nextCursor != "" || ((paginationType == "page" || paginationType == "offset") && hasMore) { + if humanFriendly { + fmt.Fprintf(os.Stderr, "warning: results truncated; more pages available. Re-run with --all to fetch every page.\n") + } else { + fmt.Fprintf(os.Stderr, `{"event":"truncated","hint":"pass --all to fetch every page"}`+"\n") + } + return + } + if humanFriendly { + fmt.Fprintf(os.Stderr, "warning: results truncated; more pages available.\n") + } else { + fmt.Fprintf(os.Stderr, `{"event":"truncated"}`+"\n") + } +} + +func emitMissingPaginationSignalWarning() { + if humanFriendly { + fmt.Fprintf(os.Stderr, "warning: --all requested, but this endpoint does not declare a next cursor or has-more field; returning page 1 only.\n") + } else { + fmt.Fprintf(os.Stderr, `{"event":"truncated","reason":"pagination_signal_missing","message":"--all requested but this endpoint does not declare a next cursor or has-more field; returning page 1 only"}`+"\n") + } +} + +func emitPaginatedGetMaxPagesWarning() { + if humanFriendly { + fmt.Fprintf(os.Stderr, "warning: --all reached the %d-page safety limit; returning fetched pages only.\n", paginatedGetMaxPages) + } else { + fmt.Fprintf(os.Stderr, `{"event":"truncated","reason":"max_pages_cap_hit","message":"--all reached the %d-page safety limit; returning fetched pages only"}`+"\n", paginatedGetMaxPages) + } +} + +func emitMissingPaginationCursorWarning(nextCursorPath string) { + if humanFriendly { + fmt.Fprintf(os.Stderr, "warning: --all requested, but the response indicated more pages without a usable next cursor; returning fetched pages only.\n") + } else if nextCursorPath != "" { + fmt.Fprintf(os.Stderr, `{"event":"truncated","reason":"pagination_cursor_missing","next_cursor_path":%q,"message":"--all requested but the response indicated more pages without a usable next cursor; returning fetched pages only"}`+"\n", nextCursorPath) + } else { + fmt.Fprintf(os.Stderr, `{"event":"truncated","reason":"pagination_cursor_missing","message":"--all requested but the response indicated more pages without a usable next cursor; returning fetched pages only"}`+"\n") + } +} + +func paginationCursorToken(raw json.RawMessage) string { + var token string + if json.Unmarshal(raw, &token) == nil && token != "" { + return token + } + var number json.Number + if json.Unmarshal(raw, &number) == nil { + if n, err := number.Int64(); err == nil && n > 0 { + return number.String() + } + } + return "" +} + +func extractPaginatedItems(obj map[string]json.RawMessage) ([]json.RawMessage, bool) { + for _, field := range []string{"data", "items", "results", "messages", "members", "values"} { + if arr, ok := obj[field]; ok { + var nested []json.RawMessage + if json.Unmarshal(arr, &nested) == nil { + return nested, true + } + } + } + + var onlyArray []json.RawMessage + arrayCount := 0 + for _, raw := range obj { + var candidate []json.RawMessage + if json.Unmarshal(raw, &candidate) == nil { + onlyArray = candidate + arrayCount++ + } + } + if arrayCount == 1 { + return onlyArray, true + } + return nil, false +} + +func rawAtPath(obj map[string]json.RawMessage, path string) (json.RawMessage, bool) { + if raw, ok := obj[path]; ok { + return raw, true + } + + current := obj + parts := strings.Split(path, ".") + for i, part := range parts { + raw, ok := current[part] + if !ok { + return nil, false + } + if i == len(parts)-1 { + return raw, true + } + if err := json.Unmarshal(raw, ¤t); err != nil { + return nil, false + } + } + return nil, false +} + +// printJSONFiltered marshals a Go-typed value through the same output +// pipeline endpoint-mirror commands use. Hand-written novel commands that +// build a typed slice/struct call this so --select, --compact, --csv, and +// --quiet all behave the same way as on generator-emitted commands. +func printJSONFiltered(w io.Writer, v any, flags *rootFlags) error { + raw, err := json.Marshal(v) + if err != nil { + return err + } + return printOutputWithFlags(w, json.RawMessage(raw), flags) +} + +// filterFields keeps only the specified fields (comma-separated) from JSON objects/arrays. +// Supports dotted paths like "events.shortName" to descend into nested structures. +// Arrays are traversed element-wise: "events.shortName" keeps shortName on each event. +func filterFields(data json.RawMessage, fields string) json.RawMessage { + var paths [][]string + for _, f := range strings.Split(fields, ",") { + f = strings.TrimSpace(f) + if f == "" { + continue + } + parts := strings.Split(f, ".") + for i := range parts { + parts[i] = strings.ToLower(parts[i]) + } + paths = append(paths, parts) + } + if len(paths) == 0 { + return data + } + return filterFieldsRec(data, paths) +} + +// filterFieldsRec applies path filters to a JSON value. Each path is a list of +// lowercase segments; arrays descend element-wise. +func filterFieldsRec(data json.RawMessage, paths [][]string) json.RawMessage { + var arr []json.RawMessage + if err := json.Unmarshal(data, &arr); err == nil { + out := make([]json.RawMessage, len(arr)) + for i, el := range arr { + out[i] = filterFieldsRec(el, paths) + } + result, _ := json.Marshal(out) + return result + } + + var obj map[string]json.RawMessage + if err := json.Unmarshal(data, &obj); err == nil { + keepWhole := map[string]bool{} + subPaths := map[string][][]string{} + for _, p := range paths { + if len(p) == 0 { + continue + } + head := p[0] + if len(p) == 1 { + keepWhole[head] = true + } else { + subPaths[head] = append(subPaths[head], p[1:]) + } + } + filtered := map[string]json.RawMessage{} + matchedAny := false + for k, v := range obj { + matched := matchSelectSegment(k, keepWhole, subPaths) + if matched == "" { + continue + } + matchedAny = true + if keepWhole[matched] { + filtered[k] = v + continue + } + if subs := subPaths[matched]; subs != nil { + filtered[k] = filterFieldsRec(v, subs) + } + } + // Envelope fallback: when no top-level keys matched but at least one + // sibling is a non-null array, treat the object as a list envelope + // (`{"items":[...]}`, `{"data":[...]}`, `{"total_count":N,"items":[...]}`) + // and apply the selector inside the array(s). Non-array siblings pass + // through verbatim so envelope metadata (counts, null pagination + // cursors) stays visible. The foundArray guard preserves the prior + // empty-object result for flat objects where no key matches and no + // array exists. The `arr != nil` check rejects JSON null, which + // json.Unmarshal otherwise accepts into a []json.RawMessage as a + // nil slice and would coerce to `[]`. + if !matchedAny { + pending := map[string]json.RawMessage{} + foundArray := false + for k, v := range obj { + var arr []json.RawMessage + if json.Unmarshal(v, &arr) == nil && arr != nil { + foundArray = true + pending[k] = filterFieldsRec(v, paths) + } else { + pending[k] = v + } + } + if foundArray { + for k, v := range pending { + filtered[k] = v + } + } + } + result, _ := json.Marshal(filtered) + return result + } + + return data +} + +// matchSelectSegment returns the matching lowercase segment, or "" if no match. +// Supports direct case-insensitive match and camelCase→kebab-case conversion. +func matchSelectSegment(fieldName string, keepWhole map[string]bool, subPaths map[string][][]string) string { + lower := strings.ToLower(fieldName) + if keepWhole[lower] || subPaths[lower] != nil { + return lower + } + kebab := camelToKebab(fieldName) + if kebab != lower && (keepWhole[kebab] || subPaths[kebab] != nil) { + return kebab + } + return "" +} + +// camelToKebab converts "orderDate" or "orderdate" to "order-date" by splitting on +// uppercase boundaries. For already-lowercase input, splits on known word boundaries. +func camelToKebab(s string) string { + var b strings.Builder + runes := []rune(s) + for i, r := range runes { + if i > 0 && unicode.IsUpper(r) && unicode.IsLower(runes[i-1]) { + b.WriteByte('-') + } + b.WriteRune(unicode.ToLower(r)) + } + return b.String() +} + +// printOutputWithFlags routes output through the right format based on flags. +func printOutputWithFlags(w io.Writer, data json.RawMessage, flags *rootFlags) error { + // --select wins over --compact when both are set: an explicit field list + // is the user's authoritative request, so the high-gravity allow-list + // must not strip those fields out before --select can pick them. When + // only --compact is set (e.g., --agent without --select), the allow-list + // still runs. + if flags.selectFields != "" { + data = filterFields(data, flags.selectFields) + } else if flags.compact { + data = compactFields(data) + } + // --quiet: suppress all output, exit code communicates result + if flags.quiet { + return nil + } + // --csv: render as CSV + if flags.csv { + return printCSV(w, data) + } + return printOutput(w, data, flags.asJSON) +} + +// compactVerboseListFields are prose-shaped fields stripped from list-item +// projections. On lists, "body"/"content"/"html"/"markdown" are verbose +// noise and the row's identity is carried by id/name/title/etc. +var compactVerboseListFields = map[string]bool{ + "description": true, "body": true, "content": true, + "comments": true, "attachments": true, "html": true, "markdown": true, +} + +// compactVerboseObjectFields are metadata fields stripped from single-object +// responses. "body"/"content"/"html"/"markdown" are intentionally absent: +// for a `get` command those fields are the primary payload, and stripping +// them under `--agent`/`--compact` silently emits a useless envelope. +// Use `--select` to drop them explicitly. +var compactVerboseObjectFields = map[string]bool{ + "description": true, + "comments": true, + "attachments": true, +} + +// compactFields keeps only the most important fields for agent consumption. +// For arrays: allowlist of high-gravity fields (no descriptions). +// For single objects: blocklist that strips known-verbose fields (descriptions, comments, etc.). +func compactFields(data json.RawMessage) json.RawMessage { + // Try array first + var items []map[string]any + if err := json.Unmarshal(data, &items); err == nil { + return compactListFields(items) + } + + // Single object — use blocklist + var obj map[string]any + if err := json.Unmarshal(data, &obj); err == nil { + return compactObjectFields(obj) + } + + return data +} + +// compactListFields keeps only high-gravity fields for array responses. +// +// Two-layer keep rule: +// +// 1. A static allow-list covers canonical scalars (id/name/price/status/...). +// 2. A data-driven extension also keeps any key present in at least 80% of +// input rows. This catches hand-written novel commands whose payload keys +// (object_name, match_key, snippet, series, metrics) aren't on the +// canonical allow-list, without forcing every printed CLI to expand the +// list. +// +// Verbose fields (description, body, content, etc.) are excluded from the +// data-driven extension regardless of frequency, so the compact intent +// (short identifying values for agent consumption, not full prose) is +// preserved. +// +// When an item still carries none of the keep keys, the original is +// preserved so `--agent` does not silently emit {} for shapes whose key +// names are entirely off-canonical. +func compactListFields(items []map[string]any) json.RawMessage { + keepFields := map[string]bool{ + // Identity + "id": true, "name": true, "title": true, "identifier": true, + "code": true, "slug": true, "key": true, + // Categorization + "status": true, "state": true, "type": true, "kind": true, "priority": true, + // Communication + "url": true, "email": true, + // Monetary + "price": true, "amount": true, "cost": true, "fare": true, + "rate": true, "currency": true, + // Metrics + "rating": true, "score": true, "count": true, + // Locale / geo + "language": true, "locale": true, "country": true, "region": true, + "city": true, "domain": true, + // Temporal + "created_at": true, "updated_at": true, "createdAt": true, "updatedAt": true, + "date": true, + // Versioning + "version": true, + } + if len(items) > 0 { + keyCounts := map[string]int{} + for _, item := range items { + for k := range item { + if compactVerboseListFields[k] { + continue + } + keyCounts[k]++ + } + } + // ceil(len(items) * 0.8) without importing math. Capped at len-1 for + // len >= 2 so a single missing row cannot veto a key on small lists + // (without the cap, ceil(0.8*n) == n for n in {2,3,4}, which silently + // reintroduces the partial-strip bug whenever a heterogeneous 2-4 row + // response mixes one allow-list key with novel keys). + threshold := (len(items)*4 + 4) / 5 + if len(items) >= 2 && threshold > len(items)-1 { + threshold = len(items) - 1 + } + for k, count := range keyCounts { + if count >= threshold { + keepFields[k] = true + } + } + } + + filtered := make([]map[string]any, 0, len(items)) + for _, item := range items { + compact := map[string]any{} + for k, v := range item { + if keepFields[k] { + compact[k] = v + } + } + if len(compact) == 0 { + compact = item + } + filtered = append(filtered, compact) + } + result, _ := json.Marshal(filtered) + return result +} + +// isCompactScalar reports whether v is a small primitive (string, number, +// bool, null) suitable for compact table decisions. Compact list projection +// may still retain frequent nested payload fields; this helper is about +// display density, not whether a field carries agent-useful payload. +func isCompactScalar(v any) bool { + switch v.(type) { + case nil, bool, float64, string: + return true + default: + return false + } +} + +// compactObjectFields strips known-verbose metadata fields from single-object +// responses. The blocklist deliberately excludes "body"/"content"/"html"/ +// "markdown" — those fields are payload on `get` commands and stripping them +// under `--agent`/`--compact` is a silent loss; agents who want to omit them +// can pass `--select` to specify only the fields they need. +func compactObjectFields(obj map[string]any) json.RawMessage { + compact := map[string]any{} + for k, v := range obj { + if !compactVerboseObjectFields[k] { + compact[k] = v + } + } + result, _ := json.Marshal(compact) + return result +} + +// printCSV renders JSON arrays as CSV with header row. +func printCSV(w io.Writer, data json.RawMessage) error { + var items []map[string]any + if err := json.Unmarshal(data, &items); err != nil || len(items) == 0 { + // Single object or empty - just print as JSON + fmt.Fprintln(w, string(data)) + return nil + } + // Collect all keys for header + keySet := map[string]bool{} + for _, item := range items { + for k := range item { + keySet[k] = true + } + } + var keys []string + for k := range keySet { + keys = append(keys, k) + } + sort.Strings(keys) + // Header + fmt.Fprintln(w, strings.Join(keys, ",")) + // Rows + for _, item := range items { + var vals []string + for _, k := range keys { + v := item[k] + if v == nil { + vals = append(vals, "") + } else { + var s string + if f, ok := v.(float64); ok { + s = strconv.FormatFloat(f, 'f', -1, 64) + } else { + s = fmt.Sprintf("%v", v) + } + if strings.ContainsAny(s, ",\"\n") { + s = `"` + strings.ReplaceAll(s, `"`, `""`) + `"` + } + vals = append(vals, s) + } + } + fmt.Fprintln(w, strings.Join(vals, ",")) + } + return nil +} + +// printOutput auto-detects arrays and renders as tables, or prints raw JSON for objects. +func printOutput(w io.Writer, data json.RawMessage, asJSON bool) error { + if !asJSON && !isTerminal(w) { + asJSON = true + } + + if asJSON { + enc := json.NewEncoder(w) + enc.SetIndent("", " ") + return enc.Encode(data) + } + + // Try to detect if response is an array + var items []map[string]any + if err := json.Unmarshal(data, &items); err == nil && len(items) > 0 { + if err := printAutoTable(w, items); err != nil { + return err + } + // Agent-friendly: show count and suggest narrowing when results are large + if len(items) >= 25 { + fmt.Fprintf(os.Stderr, "\nShowing %d results. To narrow: add --limit, --json --select, or filter flags.\n", len(items)) + } + return nil + } + + // Single object - pretty print + var obj map[string]any + if err := json.Unmarshal(data, &obj); err == nil { + enc := json.NewEncoder(w) + enc.SetIndent("", " ") + return enc.Encode(obj) + } + + // Fallback: print raw + fmt.Fprintln(w, string(data)) + return nil +} + +// levenshteinDistance computes the edit distance between two strings using a two-row DP approach. +func levenshteinDistance(a, b string) int { + if len(a) == 0 { + return len(b) + } + if len(b) == 0 { + return len(a) + } + if len(a) < len(b) { + a, b = b, a + } + prev := make([]int, len(b)+1) + curr := make([]int, len(b)+1) + for j := range prev { + prev[j] = j + } + for i := 1; i <= len(a); i++ { + curr[0] = i + for j := 1; j <= len(b); j++ { + cost := 1 + if a[i-1] == b[j-1] { + cost = 0 + } + ins := curr[j-1] + 1 + del := prev[j] + 1 + sub := prev[j-1] + cost + min := ins + if del < min { + min = del + } + if sub < min { + min = sub + } + curr[j] = min + } + prev, curr = curr, prev + } + return prev[len(b)] +} + +// suggestFlag returns the closest known flag name to the unknown string, or "" if none is close enough. +func suggestFlag(unknown string, cmd *cobra.Command) string { + unknown = strings.TrimLeft(unknown, "-") + best := "" + bestDist := 4 // only consider distance <= 3 + check := func(name string) { + d := levenshteinDistance(unknown, name) + if d < bestDist && d*5 <= len(unknown)*2 { + bestDist = d + best = name + } + } + cmd.Flags().VisitAll(func(f *pflag.Flag) { + check(f.Name) + }) + cmd.InheritedFlags().VisitAll(func(f *pflag.Flag) { + check(f.Name) + }) + return best +} + +// wantsHumanTable returns true when output should be a human-friendly table. +// Smart default: terminal=table, pipe=JSON. +// - Human in terminal: isTerminal()=true → table +// - Claude Code/Codex bash tool: stdout piped → JSON +// - --json/--csv/--compact/--agent: machine format → JSON +func wantsHumanTable(w io.Writer, flags *rootFlags) bool { + if flags.asJSON || flags.csv || flags.compact || flags.quiet || flags.plain { + return false + } + if flags.selectFields != "" { + return false + } + return isTerminal(w) +} + +func printAutoTable(w io.Writer, items []map[string]any) error { + if len(items) == 0 { + return nil + } + + // Count scalar vs complex fields to decide format + scalarCount := 0 + for _, v := range items[0] { + if isCompactScalar(v) { + scalarCount++ + } + } + + // Use sectional/card layout for complex items (many fields or nested data) + if len(items[0]) > 8 || scalarCount < len(items[0])-2 { + return printAutoCards(w, items) + } + + headers := prioritizeHeaders(items[0]) + + // Limit to 6 columns max for readability + if len(headers) > 6 { + headers = headers[:6] + } + + // Build rows + rows := make([][]string, 0, len(items)) + for _, item := range items { + row := make([]string, len(headers)) + for i, h := range headers { + row[i] = formatCellValue(item[h]) + } + rows = append(rows, row) + } + + // Print with tab alignment using tabwriter + tw := newTabWriter(w) + upperHeaders := make([]string, len(headers)) + for i, h := range headers { + upperHeaders[i] = bold(strings.ToUpper(h)) + } + + fmt.Fprintln(tw, strings.Join(upperHeaders, "\t")) + for _, row := range rows { + fmt.Fprintln(tw, strings.Join(row, "\t")) + } + return tw.Flush() +} + +// prioritizeHeaders orders scalar fields by importance for table display. +func prioritizeHeaders(item map[string]any) []string { + return prioritizeFields(item, false) +} + +// prioritizeAllHeaders orders all fields (including arrays) by importance for card display. +func prioritizeAllHeaders(item map[string]any) []string { + return prioritizeFields(item, true) +} + +// prioritizeFields orders fields by importance: identity → temporal → status → other. +// When includeComplex is true, arrays and objects are included (for card layout). +// +// Uses exact-or-suffix matching to avoid false positives: "name" matches "Name" and +// "UserName" but not "BuildingName" (because "Building" is not a known prefix that +// indicates identity). The field is split on camelCase/snake_case boundaries and the +// LAST segment is matched against patterns. +func prioritizeFields(item map[string]any, includeComplex bool) []string { + // Priority tiers — matched against the last segment of the field name. + // "OrderDate" → last segment "date" → tier 1 (temporal). + // "BuildingName" → last segment "name" → tier 0... but we want to avoid this. + // Solution: exact match on the full lowered name OR suffix segment match, + // with a penalty for compound names that have a non-identity prefix. + type pattern struct { + word string + tier int + } + // Exact matches (full field name, case-insensitive) — highest confidence + exactMatches := map[string]int{ + "id": 0, "name": 0, "title": 0, "slug": 0, "key": 0, + "date": 1, "created": 1, "updated": 1, "createdat": 1, "updatedat": 1, + "status": 2, "state": 2, "statuscode": 2, + "summary": 3, "description": 3, "price": 3, "amount": 3, "total": 3, + "cost": 3, "points": 3, "score": 3, + "type": 4, "kind": 4, "category": 4, "email": 4, "phone": 4, "url": 4, + } + // Suffix patterns — match when the field ends with this word (after splitting) + suffixMatches := map[string]int{ + "id": 0, "name": 0, "title": 0, + "date": 1, "time": 1, + "status": 2, "state": 2, "code": 2, + "price": 3, "amount": 3, "total": 3, "cost": 3, + "summary": 3, "description": 3, "points": 3, "score": 3, + "type": 4, "kind": 4, "category": 4, "method": 4, + } + + numTiers := 5 + + type scored struct { + name string + tier int + index int + } + + var all []scored + idx := 0 + for k, v := range item { + if !includeComplex { + switch v.(type) { + case []any, map[string]any: + continue + } + } + // Skip values that won't render usefully in cards + if includeComplex { + formatted := formatCellValue(v) + if formatted == "" { + continue + } + } + + tier := numTiers // default: unclassified + lower := strings.ToLower(k) + + // 1. Exact match on full field name + if t, ok := exactMatches[lower]; ok { + tier = t + } else { + // 2. Split camelCase into segments and match the last one + segments := splitCamelCase(lower) + if len(segments) > 0 { + lastSeg := segments[len(segments)-1] + if t, ok := suffixMatches[lastSeg]; ok { + // Compound names with identity suffixes (BuildingName, TipTime) + // get demoted one tier because the prefix dilutes the signal + if len(segments) > 1 { + tier = t + 1 + } else { + tier = t + } + } + } + } + + // Demote booleans to last + if _, ok := v.(bool); ok && tier >= numTiers { + tier = numTiers + 1 + } + all = append(all, scored{name: k, tier: tier, index: idx}) + idx++ + } + + sort.Slice(all, func(i, j int) bool { + if all[i].tier != all[j].tier { + return all[i].tier < all[j].tier + } + return all[i].index < all[j].index + }) + + headers := make([]string, len(all)) + for i, s := range all { + headers[i] = s.name + } + return headers +} + +// splitCamelCase splits "OrderDate" → ["order", "date"], "statusCode" → ["status", "code"], +// "page_size" → ["page", "size"]. +func splitCamelCase(s string) []string { + var segments []string + var current strings.Builder + runes := []rune(s) + for i, r := range runes { + if r == '_' || r == '-' { + if current.Len() > 0 { + segments = append(segments, current.String()) + current.Reset() + } + continue + } + if i > 0 && unicode.IsUpper(r) && unicode.IsLower(runes[i-1]) { + if current.Len() > 0 { + segments = append(segments, current.String()) + current.Reset() + } + } + current.WriteRune(unicode.ToLower(r)) + } + if current.Len() > 0 { + segments = append(segments, current.String()) + } + return segments +} + +// printAutoCards renders items as labeled cards — one block per item. +// Used for complex responses with many fields or nested data. +func printAutoCards(w io.Writer, items []map[string]any) error { + headers := prioritizeAllHeaders(items[0]) + + // Find the longest header for alignment (from fields we'll actually show) + maxLen := 0 + for _, h := range headers { + if len(h) > maxLen { + maxLen = len(h) + } + } + + for i, item := range items { + if i > 0 { + fmt.Fprintln(w) + } + + // Card header: use first priority field as the card title + titleVal := formatCellValue(item[headers[0]]) + if len(headers) > 1 { + secondVal := formatCellValue(item[headers[1]]) + if secondVal != "" { + fmt.Fprintf(w, "%s %s — %s\n", bold(strings.ToUpper(headers[0])), titleVal, secondVal) + } else { + fmt.Fprintf(w, "%s %s\n", bold(strings.ToUpper(headers[0])), titleVal) + } + } else { + fmt.Fprintf(w, "%s %s\n", bold(strings.ToUpper(headers[0])), titleVal) + } + + // Remaining fields indented — skip empty, zero, and false values + for _, h := range headers[2:] { + v := formatCellValue(item[h]) + if v == "" || v == "false" || v == "0" || v == "[]" || v == "null" { + continue + } + // Multi-line values (nested arrays) start with \n + if strings.HasPrefix(v, "\n") { + fmt.Fprintf(w, " %s:%s\n", h, v) + } else { + fmt.Fprintf(w, " %-*s %s\n", maxLen, h+":", v) + } + } + } + return nil +} + +func formatCellValue(v any) string { + switch val := v.(type) { + case string: + // Format ISO dates as just the date portion + if len(val) >= 19 && val[4] == '-' && val[7] == '-' && val[10] == 'T' { + return val[:10] + } + return truncate(val, 60) + case float64: + if val == float64(int64(val)) { + return fmt.Sprintf("%d", int64(val)) + } + return fmt.Sprintf("%.2f", val) + case bool: + return fmt.Sprintf("%t", val) + case nil: + return "" + case []any: + if len(val) == 0 { + return "" + } + // If array contains objects, format each as a summary line + if obj, isObj := val[0].(map[string]any); isObj { + _ = obj + return formatObjectArray(val) + } + // Flatten simple arrays into comma-separated string + parts := make([]string, 0, len(val)) + for _, item := range val { + if s, ok := item.(string); ok { + parts = append(parts, s) + } else { + b, _ := json.Marshal(item) + parts = append(parts, string(b)) + } + } + return truncate(strings.Join(parts, ", "), 60) + case map[string]any: + return formatSingleObject(val) + default: + b, _ := json.Marshal(val) + return truncate(string(b), 60) + } +} + +// formatObjectArray renders an array of objects as multi-line summary. +// Each object is summarized by its most descriptive fields: name/title, qty, size, price. +func formatObjectArray(items []any) string { + var lines []string + for _, raw := range items { + obj, ok := raw.(map[string]any) + if !ok { + continue + } + lines = append(lines, formatObjectSummary(obj)) + } + if len(lines) == 0 { + return "" + } + // Multi-line: newline-prefixed so the card renderer can indent + return "\n" + strings.Join(lines, "\n") +} + +// formatObjectSummary extracts the most useful fields from an object into a one-line summary. +// Looks for: qty/count → name/title → size → price, in that order. +func formatObjectSummary(obj map[string]any) string { + var parts []string + + // Quantity + qty := findField(obj, "qty", "count", "quantity") + if qty != "" && qty != "1" && qty != "0" { + parts = append(parts, qty+"x") + } else if qty == "1" { + parts = append(parts, "1x") + } + + // Name — check nested objects too (e.g., Side1.Name) + name := findField(obj, "name", "title", "label", "description") + if name == "" { + // Check nested objects for name + for _, key := range []string{"Side1", "side1", "Item", "item", "Product", "product"} { + if nested, ok := obj[key].(map[string]any); ok { + name = findField(nested, "name", "title", "label") + if name != "" { + break + } + } + } + } + if name != "" { + parts = append(parts, name) + } + + // Size + size := findField(obj, "sizename", "size_name") + if size == "" { + size = findField(obj, "catname", "cat_name", "category") + } + if size != "" { + parts = append(parts, "—") + parts = append(parts, size) + } + + // Price + price := findField(obj, "extprice", "price", "amount", "total") + if price != "" && price != "0" { + parts = append(parts, fmt.Sprintf("($%s)", price)) + } + + if len(parts) == 0 { + // Fallback: JSON summary + b, _ := json.Marshal(obj) + return truncate(string(b), 80) + } + return " " + strings.Join(parts, " ") +} + +// formatSingleObject renders a single object by its most descriptive fields. +func formatSingleObject(obj map[string]any) string { + name := findField(obj, "name", "title", "label", "description") + if name != "" { + return name + } + id := findField(obj, "id", "key", "code") + if id != "" { + return id + } + return "" +} + +// findField searches an object for a field name (case-insensitive) and returns its formatted value. +func findField(obj map[string]any, names ...string) string { + for _, name := range names { + for k, v := range obj { + if strings.EqualFold(k, name) { + return formatCellValue(v) + } + } + } + return "" +} + +// DataProvenance describes where data came from and when it was last synced. +type DataProvenance struct { + Source string `json:"source"` // "live" or "local" + SyncedAt *time.Time `json:"synced_at,omitempty"` // when local data was last synced + Reason string `json:"reason,omitempty"` // why local was used: "user_requested", "api_unreachable", "no_search_endpoint" + ResourceType string `json:"resource_type,omitempty"` // which resource type was queried + Freshness any `json:"freshness,omitempty"` // optional machine-owned freshness metadata for covered command paths + Scope any `json:"scope,omitempty"` // optional scope metadata for dynamic read-only scopes +} + +// printProvenance writes a one-line provenance message to stderr for TTY users. +// Suppressed when stdout is piped or redirected — the JSON response envelope +// already carries meta.source, so the stderr line is redundant and becomes +// noise in agent flows that merge stderr into stdout. +func printProvenance(cmd *cobra.Command, count int, prov DataProvenance) { + if !isTerminal(cmd.OutOrStdout()) { + return + } + if prov.Source == "live" { + fmt.Fprintf(cmd.ErrOrStderr(), "%d results (live)\n", count) + return + } + age := "unknown" + if prov.SyncedAt != nil { + d := time.Since(*prov.SyncedAt) + switch { + case d < time.Minute: + age = "just now" + case d < time.Hour: + age = fmt.Sprintf("%d minutes ago", int(d.Minutes())) + case d < 24*time.Hour: + age = fmt.Sprintf("%d hours ago", int(d.Hours())) + default: + age = fmt.Sprintf("%d days ago", int(d.Hours()/24)) + } + } + prefix := "" + if prov.Reason == "api_unreachable" { + prefix = "API unreachable. " + } + fmt.Fprintf(cmd.ErrOrStderr(), "%s%d results (cached, synced %s)\n", prefix, count, age) +} + +// unwrapSingleKeyArray flattens single-key collection envelopes +// ({"results":[...]}, {"data":[...]}, etc.) so the agent envelope +// emits a stable .results[] across APIs. Multi-key objects pass +// through so cursor/pagination fields stay accessible; non-array +// values pass through so non-collection responses aren't reshaped. +// +// The wrapper-key set is intentionally narrower than +// extractPaginatedItems (which also walks domain-specific keys like +// "messages", "members", "values" used by social/messaging APIs). +// This helper only flattens canonical collection envelopes for +// --json output; the pagination walker has a broader remit. +func unwrapSingleKeyArray(data json.RawMessage) json.RawMessage { + leading := bytes.TrimLeft(data, " \t\r\n") + if len(leading) == 0 || leading[0] != '{' { + return data + } + var obj map[string]json.RawMessage + if err := json.Unmarshal(data, &obj); err != nil { + return data + } + if len(obj) != 1 { + return data + } + for key, val := range obj { + if key != "results" && key != "data" && key != "items" && key != "nodes" && key != "entries" && key != "records" { + return data + } + trimmed := bytes.TrimLeft(val, " \t\r\n") + if len(trimmed) == 0 || trimmed[0] != '[' { + return data + } + return val + } + return data +} + +// wrapWithProvenance wraps response data in a provenance envelope: +// {"results": ..., "meta": {...}}. When data is valid JSON, it embeds as +// the parsed shape; when data is non-JSON (e.g., XML/RSS responses, plain +// text), it embeds as a JSON string so json.Marshal doesn't choke on +// "invalid character '<'" while still passing the raw payload through to +// the consumer. Single-key array envelopes from the API (e.g. +// {"results": [...]}, {"data": [...]}) are unwrapped first so the output +// shape is the same regardless of the API's wrapper key. +func wrapWithProvenance(data json.RawMessage, prov DataProvenance) (json.RawMessage, error) { + meta := map[string]any{"source": prov.Source} + if prov.SyncedAt != nil { + meta["synced_at"] = prov.SyncedAt.UTC().Format(time.RFC3339) + } + if prov.Reason != "" { + meta["reason"] = prov.Reason + } + if prov.ResourceType != "" { + meta["resource_type"] = prov.ResourceType + } + if prov.Freshness != nil { + meta["freshness"] = prov.Freshness + } + if prov.Scope != nil { + meta["scope"] = prov.Scope + } + var results any + if json.Valid(data) { + results = json.RawMessage(unwrapSingleKeyArray(data)) + } else { + results = string(data) + } + envelope := map[string]any{ + "results": results, + "meta": meta, + } + return json.Marshal(envelope) +} + +// truncateJSONArray returns a JSON array containing at most n elements +// from the input. When n <= 0, when the input isn't a JSON array, or +// when the array is already at-or-below the limit, the input is +// returned unchanged. +// +// Used by list-endpoint commands whose API ignores the ?limit=N query +// param (e.g. Firebase-style endpoints that return the full collection +// regardless of the param). The truncation is idempotent — calling it +// when the API already honored the limit is a no-op. The generator +// only emits the call when the spec declares a `limit` param without a +// Pagination block, so paginated APIs are unaffected. +func truncateJSONArray(data json.RawMessage, n int) json.RawMessage { + if n <= 0 { + return data + } + var arr []json.RawMessage + if err := json.Unmarshal(data, &arr); err != nil { + return data + } + if len(arr) <= n { + return data + } + out, err := json.Marshal(arr[:n]) + if err != nil { + return data + } + return out +} + +// defaultDBPath returns the canonical path for the local SQLite database. +func defaultDBPath(name string) string { + home, _ := os.UserHomeDir() + return filepath.Join(home, ".local", "share", name, "data.db") +} diff --git a/library/productivity/devonthink/internal/cli/import.go b/library/productivity/devonthink/internal/cli/import.go new file mode 100644 index 0000000000..1374d9d3f6 --- /dev/null +++ b/library/productivity/devonthink/internal/cli/import.go @@ -0,0 +1,109 @@ +// Copyright 2026 rowdy and contributors. Licensed under Apache-2.0. See LICENSE. +// Generated by CLI Printing Press (https://github.com/mvanhorn/cli-printing-press). DO NOT EDIT. + +package cli + +import ( + "bufio" + "encoding/json" + "fmt" + "io" + "os" + "strings" + + "github.com/spf13/cobra" +) + +func newImportCmd(flags *rootFlags) *cobra.Command { + var inputFile string + var dryRun bool + var batchSize int + + cmd := &cobra.Command{ + Use: "import <resource>", + Short: "Import data from JSONL file via API create/upsert calls", + Long: `Import data from a JSONL file by issuing POST requests for each record. +Each line must be a valid JSON object. Failed records are logged to stderr +but do not stop the import.`, + Example: ` # Import from a JSONL file + devonthink-pp-cli import <resource> --input data.jsonl + + # Dry-run to preview without sending + devonthink-pp-cli import <resource> --input data.jsonl --dry-run + + # Import from stdin + cat data.jsonl | devonthink-pp-cli import <resource> --input -`, + Args: cobra.ExactArgs(1), + RunE: func(cmd *cobra.Command, args []string) error { + c, err := flags.newClient() + if err != nil { + return err + } + c.DryRun = dryRun + + resource := args[0] + path := "/" + resource + + var reader io.Reader + if inputFile == "-" || inputFile == "" { + reader = os.Stdin + } else { + f, err := os.Open(inputFile) // #nosec G304 -- import intentionally reads a user-selected local file. + if err != nil { + return fmt.Errorf("opening input file: %w", err) + } + defer f.Close() + reader = f + } + + scanner := bufio.NewScanner(reader) + scanner.Buffer(make([]byte, 1024*1024), 1024*1024) // 1MB line buffer + + var success, failed, skipped int + for scanner.Scan() { + line := strings.TrimSpace(scanner.Text()) + if line == "" || line[0] == '#' { + skipped++ + continue + } + + var body map[string]any + if err := json.Unmarshal([]byte(line), &body); err != nil { + fmt.Fprintf(os.Stderr, "warning: skipping invalid JSON line: %v\n", err) + failed++ + continue + } + + _, _, err := c.Post(cmd.Context(), path, body) + if err != nil { + fmt.Fprintf(os.Stderr, "warning: failed to import record: %v\n", err) + failed++ + continue + } + success++ + } + + if err := scanner.Err(); err != nil { + return fmt.Errorf("reading input: %w", err) + } + + // JSON envelope: {succeeded, failed, skipped}. + if flags.asJSON { + return printJSONFiltered(cmd.OutOrStdout(), map[string]any{ + "succeeded": success, + "failed": failed, + "skipped": skipped, + }, flags) + } + fmt.Fprintf(os.Stderr, "Import complete: %d succeeded, %d failed, %d skipped\n", success, failed, skipped) + return nil + }, + } + + cmd.Flags().StringVarP(&inputFile, "input", "i", "", "Input JSONL file path (use - for stdin)") + _ = cmd.MarkFlagRequired("input") + cmd.Flags().BoolVar(&dryRun, "dry-run", false, "Preview import without sending requests") + cmd.Flags().IntVar(&batchSize, "batch-size", 1, "Records per batch (future: batch API support)") + + return cmd +} diff --git a/library/productivity/devonthink/internal/cli/ingest.go b/library/productivity/devonthink/internal/cli/ingest.go new file mode 100644 index 0000000000..4303f15bdb --- /dev/null +++ b/library/productivity/devonthink/internal/cli/ingest.go @@ -0,0 +1,22 @@ +// Copyright 2026 rowdy and contributors. Licensed under Apache-2.0. See LICENSE. +// Generated by CLI Printing Press (https://github.com/mvanhorn/cli-printing-press). DO NOT EDIT. + +package cli + +import ( + "github.com/spf13/cobra" +) + +func newIngestCmd(flags *rootFlags) *cobra.Command { + cmd := &cobra.Command{ + Use: "ingest", + Short: "File and URL ingestion", + Hidden: true, + Annotations: map[string]string{"mcp:read-only": "true"}, + RunE: parentNoSubcommandRunE(flags), + } + + cmd.AddCommand(newIngestFileCmd(flags)) + cmd.AddCommand(newIngestUrlCmd(flags)) + return cmd +} diff --git a/library/productivity/devonthink/internal/cli/ingest_file.go b/library/productivity/devonthink/internal/cli/ingest_file.go new file mode 100644 index 0000000000..acf405e7db --- /dev/null +++ b/library/productivity/devonthink/internal/cli/ingest_file.go @@ -0,0 +1,200 @@ +// Copyright 2026 rowdy and contributors. Licensed under Apache-2.0. See LICENSE. +// Generated by CLI Printing Press (https://github.com/mvanhorn/cli-printing-press). DO NOT EDIT. + +package cli + +import ( + "encoding/json" + "fmt" + "io" + "os" + + "github.com/spf13/cobra" +) + +func newIngestFileCmd(flags *rootFlags) *cobra.Command { + var bodyDestination string + var bodyMode string + var stdinBody bool + + cmd := &cobra.Command{ + Use: "file <path>", + Short: "Import or index a file or folder", + Example: " devonthink-pp-cli ingest file ~/Documents/report.pdf --mode import --dry-run", + Annotations: map[string]string{"pp:endpoint": "ingest.file", "pp:method": "POST", "pp:path": "/ingest/file"}, + RunE: func(cmd *cobra.Command, args []string) error { + if len(args) == 0 { + return cmd.Help() + } + if !stdinBody { + } + c, err := flags.newClient() + if err != nil { + return err + } + + path := "/ingest/file" + params := map[string]string{} + params["path"] = args[0] + var body map[string]any + if stdinBody { + stdinData, err := io.ReadAll(os.Stdin) + if err != nil { + return fmt.Errorf("reading stdin: %w", err) + } + var jsonBody map[string]any + if err := json.Unmarshal(stdinData, &jsonBody); err != nil { + return fmt.Errorf("parsing stdin JSON: %w", err) + } + body = jsonBody + } else { + body = map[string]any{} + if bodyDestination != "" { + body["destination"] = bodyDestination + } + if bodyMode != "" { + body["mode"] = bodyMode + } + } + data, statusCode, err := c.PostWithParams(cmd.Context(), path, params, body) + if err != nil { + return classifyAPIError(err, flags) + } + // Inspect the mutate response body for a partial-failure-shaped + // field (e.g. Google Ads `partialFailureError`). Several Google + // APIs return 200 OK with a partial-failure field when some + // operations in the batch failed; ignoring it silently swallows + // real failures. Detection runs before output-mode selection so + // the exit code is consistent regardless of how stdout is + // rendered. --dry-run short-circuits because no real request + // was sent. + var partialFailure *partialFailureReport + if !flags.dryRun && statusCode >= 200 && statusCode < 300 { + partialFailure = detectPartialFailure(data) + if partialFailure != nil { + fmt.Fprintf(os.Stderr, "warning: partial failure detected in %s response: %s\n", "ingest", partialFailure.Message) + if len(partialFailure.ResourceNames) > 0 { + fmt.Fprintf(os.Stderr, " succeeded: %d operation(s)\n", len(partialFailure.ResourceNames)) + } + } + } + if !flags.dryRun && statusCode >= 200 && statusCode < 300 && (partialFailure == nil || flags.allowPartialFailure) { + writeMutationResponseToStore(cmd.Context(), "ingest", data, "") + } + if wantsHumanTable(cmd.OutOrStdout(), flags) { + // Check if response contains an array (directly or wrapped in "data") + var items []map[string]any + if json.Unmarshal(data, &items) == nil && len(items) > 0 { + if err := printAutoTable(cmd.OutOrStdout(), items); err != nil { + fmt.Fprintf(os.Stderr, "warning: table rendering failed, falling back to JSON: %v\n", err) + } else { + if partialFailure != nil && !flags.allowPartialFailure { + return partialFailureErr(fmt.Errorf("partial failure in %s response: %s", "ingest", partialFailure.Message)) + } + return nil + } + } else { + var wrapped struct { + Data []map[string]any `json:"data"` + } + if json.Unmarshal(data, &wrapped) == nil && len(wrapped.Data) > 0 { + if err := printAutoTable(cmd.OutOrStdout(), wrapped.Data); err != nil { + fmt.Fprintf(os.Stderr, "warning: table rendering failed, falling back to JSON: %v\n", err) + } else { + if partialFailure != nil && !flags.allowPartialFailure { + return partialFailureErr(fmt.Errorf("partial failure in %s response: %s", "ingest", partialFailure.Message)) + } + return nil + } + } + } + } + if flags.asJSON || (!isTerminal(cmd.OutOrStdout()) && !flags.csv && !flags.quiet && !flags.plain) { + if flags.quiet { + if partialFailure != nil && !flags.allowPartialFailure { + return partialFailureErr(fmt.Errorf("partial failure in %s response: %s", "ingest", partialFailure.Message)) + } + return nil + } + envelope := map[string]any{ + "action": "post", + "resource": "ingest", + "path": path, + "status": statusCode, + "success": statusCode >= 200 && statusCode < 300 && (partialFailure == nil || flags.allowPartialFailure), + } + if partialFailure != nil { + envelope["partial_failure"] = partialFailure + } + if flags.dryRun { + envelope["dry_run"] = true + envelope["status"] = 0 + envelope["success"] = false + } + // Verify-mode synthetic envelope detection runs against RAW data + // (before --compact/--select filtering) so the sentinel field is + // guaranteed to be visible even if the operator passes a filter + // flag that would otherwise strip it. Surfaces a top-level + // verify_noop signal + flips success to false. Mirrors the dry_run + // shape above. + if len(data) > 0 { + var rawParsed any + if err := json.Unmarshal(data, &rawParsed); err == nil { + if m, ok := rawParsed.(map[string]any); ok { + if v, ok := m["__pp_verify_synthetic__"].(bool); ok && v { + envelope["verify_noop"] = true + envelope["success"] = false + } + } + } + } + // Apply --compact and --select to the API response before wrapping. + // --select wins when both are set: explicit field choice trumps the + // generic high-gravity allow-list. Otherwise --compact still applies + // when --agent is on but the user did not name fields. + filtered := data + if flags.selectFields != "" { + filtered = filterFields(filtered, flags.selectFields) + } else if flags.compact { + filtered = compactFields(filtered) + } + if len(filtered) > 0 { + var parsed any + if err := json.Unmarshal(filtered, &parsed); err == nil { + envelope["data"] = parsed + } + } + envelopeJSON, err := json.Marshal(envelope) + if err != nil { + return err + } + if perr := printOutput(cmd.OutOrStdout(), json.RawMessage(envelopeJSON), true); perr != nil { + return perr + } + if partialFailure != nil && !flags.allowPartialFailure { + return partialFailureErr(fmt.Errorf("partial failure in %s response: %s", "ingest", partialFailure.Message)) + } + return nil + } + // Fall-through for mutate paths that did not hit the table or + // asJSON branches: --quiet, --csv, --plain, and default terminal + // raw output. printOutputWithFlags renders the body, then the + // typed partial-failure exit fires unless --allow-partial-failure + // downgrades it. Without this guard a partial failure would exit + // 0 for these output modes — the exact silent-swallow regression + // the surrounding patch is preventing for asJSON / piped output. + if perr := printOutputWithFlags(cmd.OutOrStdout(), data, flags); perr != nil { + return perr + } + if partialFailure != nil && !flags.allowPartialFailure { + return partialFailureErr(fmt.Errorf("partial failure in %s response: %s", "ingest", partialFailure.Message)) + } + return nil + }, + } + cmd.Flags().StringVar(&bodyDestination, "destination", "", "Destination group path or UUID") + cmd.Flags().StringVar(&bodyMode, "mode", "import", "import or index") + cmd.Flags().BoolVar(&stdinBody, "stdin", false, "Read request body as JSON from stdin") + + return cmd +} diff --git a/library/productivity/devonthink/internal/cli/ingest_url.go b/library/productivity/devonthink/internal/cli/ingest_url.go new file mode 100644 index 0000000000..72317a237f --- /dev/null +++ b/library/productivity/devonthink/internal/cli/ingest_url.go @@ -0,0 +1,200 @@ +// Copyright 2026 rowdy and contributors. Licensed under Apache-2.0. See LICENSE. +// Generated by CLI Printing Press (https://github.com/mvanhorn/cli-printing-press). DO NOT EDIT. + +package cli + +import ( + "encoding/json" + "fmt" + "io" + "os" + + "github.com/spf13/cobra" +) + +func newIngestUrlCmd(flags *rootFlags) *cobra.Command { + var bodyDestination string + var bodyType string + var stdinBody bool + + cmd := &cobra.Command{ + Use: "url <url>", + Short: "Capture a URL as Markdown, HTML, PDF, bookmark, or webarchive", + Example: " devonthink-pp-cli ingest url https://example.com/resource", + Annotations: map[string]string{"pp:endpoint": "ingest.url", "pp:method": "POST", "pp:path": "/ingest/url"}, + RunE: func(cmd *cobra.Command, args []string) error { + if len(args) == 0 { + return cmd.Help() + } + if !stdinBody { + } + c, err := flags.newClient() + if err != nil { + return err + } + + path := "/ingest/url" + params := map[string]string{} + params["url"] = args[0] + var body map[string]any + if stdinBody { + stdinData, err := io.ReadAll(os.Stdin) + if err != nil { + return fmt.Errorf("reading stdin: %w", err) + } + var jsonBody map[string]any + if err := json.Unmarshal(stdinData, &jsonBody); err != nil { + return fmt.Errorf("parsing stdin JSON: %w", err) + } + body = jsonBody + } else { + body = map[string]any{} + if bodyDestination != "" { + body["destination"] = bodyDestination + } + if bodyType != "" { + body["type"] = bodyType + } + } + data, statusCode, err := c.PostWithParams(cmd.Context(), path, params, body) + if err != nil { + return classifyAPIError(err, flags) + } + // Inspect the mutate response body for a partial-failure-shaped + // field (e.g. Google Ads `partialFailureError`). Several Google + // APIs return 200 OK with a partial-failure field when some + // operations in the batch failed; ignoring it silently swallows + // real failures. Detection runs before output-mode selection so + // the exit code is consistent regardless of how stdout is + // rendered. --dry-run short-circuits because no real request + // was sent. + var partialFailure *partialFailureReport + if !flags.dryRun && statusCode >= 200 && statusCode < 300 { + partialFailure = detectPartialFailure(data) + if partialFailure != nil { + fmt.Fprintf(os.Stderr, "warning: partial failure detected in %s response: %s\n", "ingest", partialFailure.Message) + if len(partialFailure.ResourceNames) > 0 { + fmt.Fprintf(os.Stderr, " succeeded: %d operation(s)\n", len(partialFailure.ResourceNames)) + } + } + } + if !flags.dryRun && statusCode >= 200 && statusCode < 300 && (partialFailure == nil || flags.allowPartialFailure) { + writeMutationResponseToStore(cmd.Context(), "ingest", data, "") + } + if wantsHumanTable(cmd.OutOrStdout(), flags) { + // Check if response contains an array (directly or wrapped in "data") + var items []map[string]any + if json.Unmarshal(data, &items) == nil && len(items) > 0 { + if err := printAutoTable(cmd.OutOrStdout(), items); err != nil { + fmt.Fprintf(os.Stderr, "warning: table rendering failed, falling back to JSON: %v\n", err) + } else { + if partialFailure != nil && !flags.allowPartialFailure { + return partialFailureErr(fmt.Errorf("partial failure in %s response: %s", "ingest", partialFailure.Message)) + } + return nil + } + } else { + var wrapped struct { + Data []map[string]any `json:"data"` + } + if json.Unmarshal(data, &wrapped) == nil && len(wrapped.Data) > 0 { + if err := printAutoTable(cmd.OutOrStdout(), wrapped.Data); err != nil { + fmt.Fprintf(os.Stderr, "warning: table rendering failed, falling back to JSON: %v\n", err) + } else { + if partialFailure != nil && !flags.allowPartialFailure { + return partialFailureErr(fmt.Errorf("partial failure in %s response: %s", "ingest", partialFailure.Message)) + } + return nil + } + } + } + } + if flags.asJSON || (!isTerminal(cmd.OutOrStdout()) && !flags.csv && !flags.quiet && !flags.plain) { + if flags.quiet { + if partialFailure != nil && !flags.allowPartialFailure { + return partialFailureErr(fmt.Errorf("partial failure in %s response: %s", "ingest", partialFailure.Message)) + } + return nil + } + envelope := map[string]any{ + "action": "post", + "resource": "ingest", + "path": path, + "status": statusCode, + "success": statusCode >= 200 && statusCode < 300 && (partialFailure == nil || flags.allowPartialFailure), + } + if partialFailure != nil { + envelope["partial_failure"] = partialFailure + } + if flags.dryRun { + envelope["dry_run"] = true + envelope["status"] = 0 + envelope["success"] = false + } + // Verify-mode synthetic envelope detection runs against RAW data + // (before --compact/--select filtering) so the sentinel field is + // guaranteed to be visible even if the operator passes a filter + // flag that would otherwise strip it. Surfaces a top-level + // verify_noop signal + flips success to false. Mirrors the dry_run + // shape above. + if len(data) > 0 { + var rawParsed any + if err := json.Unmarshal(data, &rawParsed); err == nil { + if m, ok := rawParsed.(map[string]any); ok { + if v, ok := m["__pp_verify_synthetic__"].(bool); ok && v { + envelope["verify_noop"] = true + envelope["success"] = false + } + } + } + } + // Apply --compact and --select to the API response before wrapping. + // --select wins when both are set: explicit field choice trumps the + // generic high-gravity allow-list. Otherwise --compact still applies + // when --agent is on but the user did not name fields. + filtered := data + if flags.selectFields != "" { + filtered = filterFields(filtered, flags.selectFields) + } else if flags.compact { + filtered = compactFields(filtered) + } + if len(filtered) > 0 { + var parsed any + if err := json.Unmarshal(filtered, &parsed); err == nil { + envelope["data"] = parsed + } + } + envelopeJSON, err := json.Marshal(envelope) + if err != nil { + return err + } + if perr := printOutput(cmd.OutOrStdout(), json.RawMessage(envelopeJSON), true); perr != nil { + return perr + } + if partialFailure != nil && !flags.allowPartialFailure { + return partialFailureErr(fmt.Errorf("partial failure in %s response: %s", "ingest", partialFailure.Message)) + } + return nil + } + // Fall-through for mutate paths that did not hit the table or + // asJSON branches: --quiet, --csv, --plain, and default terminal + // raw output. printOutputWithFlags renders the body, then the + // typed partial-failure exit fires unless --allow-partial-failure + // downgrades it. Without this guard a partial failure would exit + // 0 for these output modes — the exact silent-swallow regression + // the surrounding patch is preventing for asJSON / piped output. + if perr := printOutputWithFlags(cmd.OutOrStdout(), data, flags); perr != nil { + return perr + } + if partialFailure != nil && !flags.allowPartialFailure { + return partialFailureErr(fmt.Errorf("partial failure in %s response: %s", "ingest", partialFailure.Message)) + } + return nil + }, + } + cmd.Flags().StringVar(&bodyDestination, "destination", "", "Destination group path or UUID") + cmd.Flags().StringVar(&bodyType, "type", "markdown", "Capture format") + cmd.Flags().BoolVar(&stdinBody, "stdin", false, "Read request body as JSON from stdin") + + return cmd +} diff --git a/library/productivity/devonthink/internal/cli/inventory_export.go b/library/productivity/devonthink/internal/cli/inventory_export.go new file mode 100644 index 0000000000..41bd8a68e8 --- /dev/null +++ b/library/productivity/devonthink/internal/cli/inventory_export.go @@ -0,0 +1,50 @@ +// Copyright 2026 rowdy and contributors. Licensed under Apache-2.0. See LICENSE. +// Generated by CLI Printing Press (https://github.com/mvanhorn/cli-printing-press). DO NOT EDIT. + +package cli + +import ( + "github.com/spf13/cobra" +) + +func newNovelInventoryExportCmd(flags *rootFlags) *cobra.Command { + var flagFormat string + var flagLimit int + var flagOutput string + var flagQuery string + + cmd := &cobra.Command{ + Use: "export", + Short: "Export DEVONthink databases, groups, tags, and document metadata for maintenance plugins.", + Annotations: map[string]string{"mcp:read-only": "true"}, + RunE: func(cmd *cobra.Command, args []string) error { + c, err := flags.newClient() + if err != nil { + return err + } + params := map[string]string{} + if flagFormat != "" { + params["format"] = formatCLIParamValue(flagFormat) + } + if flagLimit != 0 { + params["limit"] = formatCLIParamValue(flagLimit) + } + if flagOutput != "" { + params["output"] = formatCLIParamValue(flagOutput) + } + if flagQuery != "" { + params["query"] = formatCLIParamValue(flagQuery) + } + data, err := c.Get(cmd.Context(), "/inventory/export", params) + if err != nil { + return classifyAPIError(err, flags) + } + return printOutputWithFlags(cmd.OutOrStdout(), data, flags) + }, + } + cmd.Flags().StringVar(&flagFormat, "format", "maintenance", "Export format: maintenance, raw, or compact") + cmd.Flags().IntVar(&flagLimit, "limit", 100, "Maximum record metadata rows to include") + cmd.Flags().StringVar(&flagOutput, "output", "", "Write the export JSON to a file path") + cmd.Flags().StringVar(&flagQuery, "query", "kind:document", "DEVONthink query used to select record metadata") + return cmd +} diff --git a/library/productivity/devonthink/internal/cli/inventory_export_test.go b/library/productivity/devonthink/internal/cli/inventory_export_test.go new file mode 100644 index 0000000000..0477b8ae55 --- /dev/null +++ b/library/productivity/devonthink/internal/cli/inventory_export_test.go @@ -0,0 +1,42 @@ +// Copyright 2026 rowdy and contributors. Licensed under Apache-2.0. See LICENSE. +// Generated by CLI Printing Press (https://github.com/mvanhorn/cli-printing-press). DO NOT EDIT. + +package cli + +import ( + "bytes" + "encoding/json" + "testing" +) + +func executeNovelDryRun(t *testing.T, args ...string) map[string]any { + t.Helper() + var flags rootFlags + cmd := newRootCmd(&flags) + var out bytes.Buffer + cmd.SetOut(&out) + cmd.SetErr(&out) + cmd.SetArgs(append([]string{"--json", "--dry-run"}, args...)) + if err := cmd.Execute(); err != nil { + t.Fatalf("command failed: %v\noutput:\n%s", err, out.String()) + } + var payload map[string]any + if err := json.Unmarshal(out.Bytes(), &payload); err != nil { + t.Fatalf("output is not JSON: %v\n%s", err, out.String()) + } + return payload +} + +func TestNovelInventoryExportCommandDryRun(t *testing.T) { + payload := executeNovelDryRun(t, "inventory", "export", "--format", "maintenance", "--query", "kind:pdf", "--limit", "3") + if payload["path"] != "/inventory/export" { + t.Fatalf("path = %v, want /inventory/export", payload["path"]) + } + params, ok := payload["params"].(map[string]any) + if !ok { + t.Fatalf("params missing from payload: %#v", payload) + } + if params["format"] != "maintenance" || params["query"] != "kind:pdf" || params["limit"] != "3" { + t.Fatalf("params = %#v", params) + } +} diff --git a/library/productivity/devonthink/internal/cli/ledger.go b/library/productivity/devonthink/internal/cli/ledger.go new file mode 100644 index 0000000000..4728b9e3a4 --- /dev/null +++ b/library/productivity/devonthink/internal/cli/ledger.go @@ -0,0 +1,22 @@ +// Copyright 2026 rowdy and contributors. Licensed under Apache-2.0. See LICENSE. +// Generated by CLI Printing Press (https://github.com/mvanhorn/cli-printing-press). DO NOT EDIT. + +package cli + +import ( + "github.com/spf13/cobra" +) + +func newLedgerCmd(flags *rootFlags) *cobra.Command { + cmd := &cobra.Command{ + Use: "ledger", + Short: "List and get ledger", + Hidden: true, + Annotations: map[string]string{"mcp:read-only": "true"}, + RunE: parentNoSubcommandRunE(flags), + } + + cmd.AddCommand(newLedgerListCmd(flags)) + cmd.AddCommand(newLedgerShowCmd(flags)) + return cmd +} diff --git a/library/productivity/devonthink/internal/cli/ledger_list.go b/library/productivity/devonthink/internal/cli/ledger_list.go new file mode 100644 index 0000000000..c6821588f3 --- /dev/null +++ b/library/productivity/devonthink/internal/cli/ledger_list.go @@ -0,0 +1,91 @@ +// Copyright 2026 rowdy and contributors. Licensed under Apache-2.0. See LICENSE. +// Generated by CLI Printing Press (https://github.com/mvanhorn/cli-printing-press). DO NOT EDIT. + +package cli + +import ( + "encoding/json" + "fmt" + "os" + + "github.com/spf13/cobra" +) + +func newLedgerListCmd(flags *rootFlags) *cobra.Command { + var flagSince string + var flagLimit int + + cmd := &cobra.Command{ + Use: "list", + Short: "List recent CLI operation ledger entries", + Example: " devonthink-pp-cli ledger list", + Annotations: map[string]string{"pp:endpoint": "ledger.list", "pp:method": "GET", "pp:path": "/ledger", "mcp:read-only": "true"}, + RunE: func(cmd *cobra.Command, args []string) error { + c, err := flags.newClient() + if err != nil { + return err + } + + path := "/ledger" + params := map[string]string{} + if flagSince != "" { + params["since"] = formatCLIParamValue(flagSince) + } + if flagLimit != 0 { + params["limit"] = formatCLIParamValue(flagLimit) + } + data, prov, err := resolveReadWithStrategy(cmd.Context(), c, flags, "auto", "ledger", true, path, params, nil, cmd.ErrOrStderr()) + if err != nil { + return classifyAPIError(err, flags) + } + // Honor --limit when the API accepts but ignores ?limit=N. + data = truncateJSONArray(data, flagLimit) + // Print provenance to stderr for human-facing output only. + // Machine-format flags (--json, --csv, --compact, --quiet, --plain, + // --select) and piped stdout suppress this line; the JSON envelope + // already carries meta.source for those consumers. + // SYNC: keep this gate aligned with command_promoted.go.tmpl. + if wantsHumanTable(cmd.OutOrStdout(), flags) { + var countItems []json.RawMessage + _ = json.Unmarshal(data, &countItems) + printProvenance(cmd, len(countItems), prov) + } + // For JSON output, wrap with provenance envelope before passing through flags. + // --select wins over --compact when both are set; --compact only runs when + // no explicit fields were requested. Explicit format flags (--csv, --quiet, + // --plain) opt out of the auto-JSON path so piped consumers that asked for + // a non-JSON format reach the standard pipeline below. + if flags.asJSON || (!isTerminal(cmd.OutOrStdout()) && !flags.csv && !flags.quiet && !flags.plain) { + filtered := data + if flags.selectFields != "" { + filtered = filterFields(filtered, flags.selectFields) + } else if flags.compact { + filtered = compactFields(filtered) + } + wrapped, wrapErr := wrapWithProvenance(filtered, prov) + if wrapErr != nil { + return wrapErr + } + return printOutput(cmd.OutOrStdout(), wrapped, true) + } + // For all other output modes (table, csv, plain, quiet), use the standard pipeline + if wantsHumanTable(cmd.OutOrStdout(), flags) { + var items []map[string]any + if json.Unmarshal(data, &items) == nil && len(items) > 0 { + if err := printAutoTable(cmd.OutOrStdout(), items); err != nil { + return err + } + if len(items) >= 25 { + fmt.Fprintf(os.Stderr, "\nShowing %d results. To narrow: add --limit, --json --select, or filter flags.\n", len(items)) + } + return nil + } + } + return printOutputWithFlags(cmd.OutOrStdout(), data, flags) + }, + } + cmd.Flags().StringVar(&flagSince, "since", "", "Time window such as 7d") + cmd.Flags().IntVar(&flagLimit, "limit", 50, "Maximum entries") + + return cmd +} diff --git a/library/productivity/devonthink/internal/cli/ledger_show.go b/library/productivity/devonthink/internal/cli/ledger_show.go new file mode 100644 index 0000000000..1ac9577e3d --- /dev/null +++ b/library/productivity/devonthink/internal/cli/ledger_show.go @@ -0,0 +1,83 @@ +// Copyright 2026 rowdy and contributors. Licensed under Apache-2.0. See LICENSE. +// Generated by CLI Printing Press (https://github.com/mvanhorn/cli-printing-press). DO NOT EDIT. + +package cli + +import ( + "encoding/json" + "fmt" + "os" + + "github.com/spf13/cobra" +) + +func newLedgerShowCmd(flags *rootFlags) *cobra.Command { + + cmd := &cobra.Command{ + Use: "show <id>", + Short: "Show one ledger entry with target proofs and rollback hints", + Example: " devonthink-pp-cli ledger show 550e8400-e29b-41d4-a716-446655440000", + Annotations: map[string]string{"pp:endpoint": "ledger.show", "pp:method": "GET", "pp:path": "/ledger/{id}", "mcp:read-only": "true"}, + RunE: func(cmd *cobra.Command, args []string) error { + if len(args) == 0 { + return cmd.Help() + } + c, err := flags.newClient() + if err != nil { + return err + } + + path := "/ledger/{id}" + path = replacePathParam(path, "id", args[0]) + params := map[string]string{} + data, prov, err := resolveReadWithStrategy(cmd.Context(), c, flags, "auto", "ledger", false, path, params, nil, cmd.ErrOrStderr()) + if err != nil { + return classifyAPIError(err, flags) + } + // Print provenance to stderr for human-facing output only. + // Machine-format flags (--json, --csv, --compact, --quiet, --plain, + // --select) and piped stdout suppress this line; the JSON envelope + // already carries meta.source for those consumers. + // SYNC: keep this gate aligned with command_promoted.go.tmpl. + if wantsHumanTable(cmd.OutOrStdout(), flags) { + var countItems []json.RawMessage + _ = json.Unmarshal(data, &countItems) + printProvenance(cmd, len(countItems), prov) + } + // For JSON output, wrap with provenance envelope before passing through flags. + // --select wins over --compact when both are set; --compact only runs when + // no explicit fields were requested. Explicit format flags (--csv, --quiet, + // --plain) opt out of the auto-JSON path so piped consumers that asked for + // a non-JSON format reach the standard pipeline below. + if flags.asJSON || (!isTerminal(cmd.OutOrStdout()) && !flags.csv && !flags.quiet && !flags.plain) { + filtered := data + if flags.selectFields != "" { + filtered = filterFields(filtered, flags.selectFields) + } else if flags.compact { + filtered = compactFields(filtered) + } + wrapped, wrapErr := wrapWithProvenance(filtered, prov) + if wrapErr != nil { + return wrapErr + } + return printOutput(cmd.OutOrStdout(), wrapped, true) + } + // For all other output modes (table, csv, plain, quiet), use the standard pipeline + if wantsHumanTable(cmd.OutOrStdout(), flags) { + var items []map[string]any + if json.Unmarshal(data, &items) == nil && len(items) > 0 { + if err := printAutoTable(cmd.OutOrStdout(), items); err != nil { + return err + } + if len(items) >= 25 { + fmt.Fprintf(os.Stderr, "\nShowing %d results. To narrow: add --limit, --json --select, or filter flags.\n", len(items)) + } + return nil + } + } + return printOutputWithFlags(cmd.OutOrStdout(), data, flags) + }, + } + + return cmd +} diff --git a/library/productivity/devonthink/internal/cli/mcp.go b/library/productivity/devonthink/internal/cli/mcp.go new file mode 100644 index 0000000000..93620010b8 --- /dev/null +++ b/library/productivity/devonthink/internal/cli/mcp.go @@ -0,0 +1,23 @@ +// Copyright 2026 rowdy and contributors. Licensed under Apache-2.0. See LICENSE. +// Generated by CLI Printing Press (https://github.com/mvanhorn/cli-printing-press). DO NOT EDIT. + +package cli + +import ( + "github.com/spf13/cobra" +) + +func newMcpCmd(flags *rootFlags) *cobra.Command { + cmd := &cobra.Command{ + Use: "mcp", + Short: "Optional local official MCP passthrough", + Hidden: true, + Annotations: map[string]string{"mcp:read-only": "true"}, + RunE: parentNoSubcommandRunE(flags), + } + + cmd.AddCommand(newMcpCallCmd(flags)) + cmd.AddCommand(newMcpSchemaCmd(flags)) + cmd.AddCommand(newMcpToolsCmd(flags)) + return cmd +} diff --git a/library/productivity/devonthink/internal/cli/mcp_call.go b/library/productivity/devonthink/internal/cli/mcp_call.go new file mode 100644 index 0000000000..d9f4ded366 --- /dev/null +++ b/library/productivity/devonthink/internal/cli/mcp_call.go @@ -0,0 +1,195 @@ +// Copyright 2026 rowdy and contributors. Licensed under Apache-2.0. See LICENSE. +// Generated by CLI Printing Press (https://github.com/mvanhorn/cli-printing-press). DO NOT EDIT. + +package cli + +import ( + "encoding/json" + "fmt" + "io" + "os" + + "github.com/spf13/cobra" +) + +func newMcpCallCmd(flags *rootFlags) *cobra.Command { + var bodyArgs string + var stdinBody bool + + cmd := &cobra.Command{ + Use: "call <tool>", + Short: "Call a local official DEVONthink MCP tool by name", + Example: " devonthink-pp-cli mcp call is_running --json", + Annotations: map[string]string{"pp:endpoint": "mcp.call", "pp:method": "POST", "pp:path": "/mcp/call"}, + RunE: func(cmd *cobra.Command, args []string) error { + if len(args) == 0 { + return cmd.Help() + } + if !stdinBody { + } + c, err := flags.newClient() + if err != nil { + return err + } + + path := "/mcp/call" + params := map[string]string{} + params["tool"] = args[0] + var body map[string]any + if stdinBody { + stdinData, err := io.ReadAll(os.Stdin) + if err != nil { + return fmt.Errorf("reading stdin: %w", err) + } + var jsonBody map[string]any + if err := json.Unmarshal(stdinData, &jsonBody); err != nil { + return fmt.Errorf("parsing stdin JSON: %w", err) + } + body = jsonBody + } else { + body = map[string]any{} + if bodyArgs != "" { + body["args"] = bodyArgs + } + } + data, statusCode, err := c.PostWithParams(cmd.Context(), path, params, body) + if err != nil { + return classifyAPIError(err, flags) + } + // Inspect the mutate response body for a partial-failure-shaped + // field (e.g. Google Ads `partialFailureError`). Several Google + // APIs return 200 OK with a partial-failure field when some + // operations in the batch failed; ignoring it silently swallows + // real failures. Detection runs before output-mode selection so + // the exit code is consistent regardless of how stdout is + // rendered. --dry-run short-circuits because no real request + // was sent. + var partialFailure *partialFailureReport + if !flags.dryRun && statusCode >= 200 && statusCode < 300 { + partialFailure = detectPartialFailure(data) + if partialFailure != nil { + fmt.Fprintf(os.Stderr, "warning: partial failure detected in %s response: %s\n", "mcp", partialFailure.Message) + if len(partialFailure.ResourceNames) > 0 { + fmt.Fprintf(os.Stderr, " succeeded: %d operation(s)\n", len(partialFailure.ResourceNames)) + } + } + } + if !flags.dryRun && statusCode >= 200 && statusCode < 300 && (partialFailure == nil || flags.allowPartialFailure) { + writeMutationResponseToStore(cmd.Context(), "mcp", data, "") + } + if wantsHumanTable(cmd.OutOrStdout(), flags) { + // Check if response contains an array (directly or wrapped in "data") + var items []map[string]any + if json.Unmarshal(data, &items) == nil && len(items) > 0 { + if err := printAutoTable(cmd.OutOrStdout(), items); err != nil { + fmt.Fprintf(os.Stderr, "warning: table rendering failed, falling back to JSON: %v\n", err) + } else { + if partialFailure != nil && !flags.allowPartialFailure { + return partialFailureErr(fmt.Errorf("partial failure in %s response: %s", "mcp", partialFailure.Message)) + } + return nil + } + } else { + var wrapped struct { + Data []map[string]any `json:"data"` + } + if json.Unmarshal(data, &wrapped) == nil && len(wrapped.Data) > 0 { + if err := printAutoTable(cmd.OutOrStdout(), wrapped.Data); err != nil { + fmt.Fprintf(os.Stderr, "warning: table rendering failed, falling back to JSON: %v\n", err) + } else { + if partialFailure != nil && !flags.allowPartialFailure { + return partialFailureErr(fmt.Errorf("partial failure in %s response: %s", "mcp", partialFailure.Message)) + } + return nil + } + } + } + } + if flags.asJSON || (!isTerminal(cmd.OutOrStdout()) && !flags.csv && !flags.quiet && !flags.plain) { + if flags.quiet { + if partialFailure != nil && !flags.allowPartialFailure { + return partialFailureErr(fmt.Errorf("partial failure in %s response: %s", "mcp", partialFailure.Message)) + } + return nil + } + envelope := map[string]any{ + "action": "post", + "resource": "mcp", + "path": path, + "status": statusCode, + "success": statusCode >= 200 && statusCode < 300 && (partialFailure == nil || flags.allowPartialFailure), + } + if partialFailure != nil { + envelope["partial_failure"] = partialFailure + } + if flags.dryRun { + envelope["dry_run"] = true + envelope["status"] = 0 + envelope["success"] = false + } + // Verify-mode synthetic envelope detection runs against RAW data + // (before --compact/--select filtering) so the sentinel field is + // guaranteed to be visible even if the operator passes a filter + // flag that would otherwise strip it. Surfaces a top-level + // verify_noop signal + flips success to false. Mirrors the dry_run + // shape above. + if len(data) > 0 { + var rawParsed any + if err := json.Unmarshal(data, &rawParsed); err == nil { + if m, ok := rawParsed.(map[string]any); ok { + if v, ok := m["__pp_verify_synthetic__"].(bool); ok && v { + envelope["verify_noop"] = true + envelope["success"] = false + } + } + } + } + // Apply --compact and --select to the API response before wrapping. + // --select wins when both are set: explicit field choice trumps the + // generic high-gravity allow-list. Otherwise --compact still applies + // when --agent is on but the user did not name fields. + filtered := data + if flags.selectFields != "" { + filtered = filterFields(filtered, flags.selectFields) + } else if flags.compact { + filtered = compactFields(filtered) + } + if len(filtered) > 0 { + var parsed any + if err := json.Unmarshal(filtered, &parsed); err == nil { + envelope["data"] = parsed + } + } + envelopeJSON, err := json.Marshal(envelope) + if err != nil { + return err + } + if perr := printOutput(cmd.OutOrStdout(), json.RawMessage(envelopeJSON), true); perr != nil { + return perr + } + if partialFailure != nil && !flags.allowPartialFailure { + return partialFailureErr(fmt.Errorf("partial failure in %s response: %s", "mcp", partialFailure.Message)) + } + return nil + } + // Fall-through for mutate paths that did not hit the table or + // asJSON branches: --quiet, --csv, --plain, and default terminal + // raw output. printOutputWithFlags renders the body, then the + // typed partial-failure exit fires unless --allow-partial-failure + // downgrades it. Without this guard a partial failure would exit + // 0 for these output modes — the exact silent-swallow regression + // the surrounding patch is preventing for asJSON / piped output. + if perr := printOutputWithFlags(cmd.OutOrStdout(), data, flags); perr != nil { + return perr + } + if partialFailure != nil && !flags.allowPartialFailure { + return partialFailureErr(fmt.Errorf("partial failure in %s response: %s", "mcp", partialFailure.Message)) + } + return nil + }, + } + cmd.Flags().StringVar(&bodyArgs, "args", "", "JSON argument object") + cmd.Flags().BoolVar(&stdinBody, "stdin", false, "Read request body as JSON from stdin") + + return cmd +} diff --git a/library/productivity/devonthink/internal/cli/mcp_schema.go b/library/productivity/devonthink/internal/cli/mcp_schema.go new file mode 100644 index 0000000000..319b364e6a --- /dev/null +++ b/library/productivity/devonthink/internal/cli/mcp_schema.go @@ -0,0 +1,79 @@ +// Copyright 2026 rowdy and contributors. Licensed under Apache-2.0. See LICENSE. +// Generated by CLI Printing Press (https://github.com/mvanhorn/cli-printing-press). DO NOT EDIT. + +package cli + +import ( + "encoding/json" + "fmt" + "os" + + "github.com/spf13/cobra" +) + +func newMcpSchemaCmd(flags *rootFlags) *cobra.Command { + + cmd := &cobra.Command{ + Use: "schema", + Short: "Emit cached MCP tool schemas", + Example: " devonthink-pp-cli mcp schema", + Annotations: map[string]string{"pp:endpoint": "mcp.schema", "pp:method": "GET", "pp:path": "/mcp/schema", "mcp:read-only": "true"}, + RunE: func(cmd *cobra.Command, args []string) error { + c, err := flags.newClient() + if err != nil { + return err + } + + path := "/mcp/schema" + params := map[string]string{} + data, prov, err := resolveReadWithStrategy(cmd.Context(), c, flags, "auto", "mcp", false, path, params, nil, cmd.ErrOrStderr()) + if err != nil { + return classifyAPIError(err, flags) + } + // Print provenance to stderr for human-facing output only. + // Machine-format flags (--json, --csv, --compact, --quiet, --plain, + // --select) and piped stdout suppress this line; the JSON envelope + // already carries meta.source for those consumers. + // SYNC: keep this gate aligned with command_promoted.go.tmpl. + if wantsHumanTable(cmd.OutOrStdout(), flags) { + var countItems []json.RawMessage + _ = json.Unmarshal(data, &countItems) + printProvenance(cmd, len(countItems), prov) + } + // For JSON output, wrap with provenance envelope before passing through flags. + // --select wins over --compact when both are set; --compact only runs when + // no explicit fields were requested. Explicit format flags (--csv, --quiet, + // --plain) opt out of the auto-JSON path so piped consumers that asked for + // a non-JSON format reach the standard pipeline below. + if flags.asJSON || (!isTerminal(cmd.OutOrStdout()) && !flags.csv && !flags.quiet && !flags.plain) { + filtered := data + if flags.selectFields != "" { + filtered = filterFields(filtered, flags.selectFields) + } else if flags.compact { + filtered = compactFields(filtered) + } + wrapped, wrapErr := wrapWithProvenance(filtered, prov) + if wrapErr != nil { + return wrapErr + } + return printOutput(cmd.OutOrStdout(), wrapped, true) + } + // For all other output modes (table, csv, plain, quiet), use the standard pipeline + if wantsHumanTable(cmd.OutOrStdout(), flags) { + var items []map[string]any + if json.Unmarshal(data, &items) == nil && len(items) > 0 { + if err := printAutoTable(cmd.OutOrStdout(), items); err != nil { + return err + } + if len(items) >= 25 { + fmt.Fprintf(os.Stderr, "\nShowing %d results. To narrow: add --limit, --json --select, or filter flags.\n", len(items)) + } + return nil + } + } + return printOutputWithFlags(cmd.OutOrStdout(), data, flags) + }, + } + + return cmd +} diff --git a/library/productivity/devonthink/internal/cli/mcp_tools.go b/library/productivity/devonthink/internal/cli/mcp_tools.go new file mode 100644 index 0000000000..3f3650e720 --- /dev/null +++ b/library/productivity/devonthink/internal/cli/mcp_tools.go @@ -0,0 +1,79 @@ +// Copyright 2026 rowdy and contributors. Licensed under Apache-2.0. See LICENSE. +// Generated by CLI Printing Press (https://github.com/mvanhorn/cli-printing-press). DO NOT EDIT. + +package cli + +import ( + "encoding/json" + "fmt" + "os" + + "github.com/spf13/cobra" +) + +func newMcpToolsCmd(flags *rootFlags) *cobra.Command { + + cmd := &cobra.Command{ + Use: "tools", + Short: "List official DEVONthink MCP tools when local MCP HTTP is enabled", + Example: " devonthink-pp-cli mcp tools", + Annotations: map[string]string{"pp:endpoint": "mcp.tools", "pp:method": "GET", "pp:path": "/mcp/tools", "mcp:read-only": "true"}, + RunE: func(cmd *cobra.Command, args []string) error { + c, err := flags.newClient() + if err != nil { + return err + } + + path := "/mcp/tools" + params := map[string]string{} + data, prov, err := resolveReadWithStrategy(cmd.Context(), c, flags, "auto", "mcp", false, path, params, nil, cmd.ErrOrStderr()) + if err != nil { + return classifyAPIError(err, flags) + } + // Print provenance to stderr for human-facing output only. + // Machine-format flags (--json, --csv, --compact, --quiet, --plain, + // --select) and piped stdout suppress this line; the JSON envelope + // already carries meta.source for those consumers. + // SYNC: keep this gate aligned with command_promoted.go.tmpl. + if wantsHumanTable(cmd.OutOrStdout(), flags) { + var countItems []json.RawMessage + _ = json.Unmarshal(data, &countItems) + printProvenance(cmd, len(countItems), prov) + } + // For JSON output, wrap with provenance envelope before passing through flags. + // --select wins over --compact when both are set; --compact only runs when + // no explicit fields were requested. Explicit format flags (--csv, --quiet, + // --plain) opt out of the auto-JSON path so piped consumers that asked for + // a non-JSON format reach the standard pipeline below. + if flags.asJSON || (!isTerminal(cmd.OutOrStdout()) && !flags.csv && !flags.quiet && !flags.plain) { + filtered := data + if flags.selectFields != "" { + filtered = filterFields(filtered, flags.selectFields) + } else if flags.compact { + filtered = compactFields(filtered) + } + wrapped, wrapErr := wrapWithProvenance(filtered, prov) + if wrapErr != nil { + return wrapErr + } + return printOutput(cmd.OutOrStdout(), wrapped, true) + } + // For all other output modes (table, csv, plain, quiet), use the standard pipeline + if wantsHumanTable(cmd.OutOrStdout(), flags) { + var items []map[string]any + if json.Unmarshal(data, &items) == nil && len(items) > 0 { + if err := printAutoTable(cmd.OutOrStdout(), items); err != nil { + return err + } + if len(items) >= 25 { + fmt.Fprintf(os.Stderr, "\nShowing %d results. To narrow: add --limit, --json --select, or filter flags.\n", len(items)) + } + return nil + } + } + return printOutputWithFlags(cmd.OutOrStdout(), data, flags) + }, + } + + return cmd +} diff --git a/library/productivity/devonthink/internal/cli/media.go b/library/productivity/devonthink/internal/cli/media.go new file mode 100644 index 0000000000..8665febcbc --- /dev/null +++ b/library/productivity/devonthink/internal/cli/media.go @@ -0,0 +1,22 @@ +// Copyright 2026 rowdy and contributors. Licensed under Apache-2.0. See LICENSE. +// Generated by CLI Printing Press (https://github.com/mvanhorn/cli-printing-press). DO NOT EDIT. + +package cli + +import ( + "github.com/spf13/cobra" +) + +func newMediaCmd(flags *rootFlags) *cobra.Command { + cmd := &cobra.Command{ + Use: "media", + Short: "Manage media command groups", + Hidden: true, + Annotations: map[string]string{"mcp:read-only": "true"}, + RunE: parentNoSubcommandRunE(flags), + } + + cmd.AddCommand(newMediaOcrCmd(flags)) + cmd.AddCommand(newMediaTranscribeCmd(flags)) + return cmd +} diff --git a/library/productivity/devonthink/internal/cli/media_ocr.go b/library/productivity/devonthink/internal/cli/media_ocr.go new file mode 100644 index 0000000000..ddbb7311e9 --- /dev/null +++ b/library/productivity/devonthink/internal/cli/media_ocr.go @@ -0,0 +1,195 @@ +// Copyright 2026 rowdy and contributors. Licensed under Apache-2.0. See LICENSE. +// Generated by CLI Printing Press (https://github.com/mvanhorn/cli-printing-press). DO NOT EDIT. + +package cli + +import ( + "encoding/json" + "fmt" + "io" + "os" + + "github.com/spf13/cobra" +) + +func newMediaOcrCmd(flags *rootFlags) *cobra.Command { + var bodyFormat string + var stdinBody bool + + cmd := &cobra.Command{ + Use: "ocr <uuid>", + Short: "OCR an image or scanned PDF", + Example: " devonthink-pp-cli media ocr 550e8400-e29b-41d4-a716-446655440000", + Annotations: map[string]string{"pp:endpoint": "media.ocr", "pp:method": "POST", "pp:path": "/media/{uuid}/ocr"}, + RunE: func(cmd *cobra.Command, args []string) error { + if len(args) == 0 { + return cmd.Help() + } + if !stdinBody { + } + c, err := flags.newClient() + if err != nil { + return err + } + + path := "/media/{uuid}/ocr" + path = replacePathParam(path, "uuid", args[0]) + params := map[string]string{} + var body map[string]any + if stdinBody { + stdinData, err := io.ReadAll(os.Stdin) + if err != nil { + return fmt.Errorf("reading stdin: %w", err) + } + var jsonBody map[string]any + if err := json.Unmarshal(stdinData, &jsonBody); err != nil { + return fmt.Errorf("parsing stdin JSON: %w", err) + } + body = jsonBody + } else { + body = map[string]any{} + if bodyFormat != "" { + body["format"] = bodyFormat + } + } + data, statusCode, err := c.PostWithParams(cmd.Context(), path, params, body) + if err != nil { + return classifyAPIError(err, flags) + } + // Inspect the mutate response body for a partial-failure-shaped + // field (e.g. Google Ads `partialFailureError`). Several Google + // APIs return 200 OK with a partial-failure field when some + // operations in the batch failed; ignoring it silently swallows + // real failures. Detection runs before output-mode selection so + // the exit code is consistent regardless of how stdout is + // rendered. --dry-run short-circuits because no real request + // was sent. + var partialFailure *partialFailureReport + if !flags.dryRun && statusCode >= 200 && statusCode < 300 { + partialFailure = detectPartialFailure(data) + if partialFailure != nil { + fmt.Fprintf(os.Stderr, "warning: partial failure detected in %s response: %s\n", "media", partialFailure.Message) + if len(partialFailure.ResourceNames) > 0 { + fmt.Fprintf(os.Stderr, " succeeded: %d operation(s)\n", len(partialFailure.ResourceNames)) + } + } + } + if !flags.dryRun && statusCode >= 200 && statusCode < 300 && (partialFailure == nil || flags.allowPartialFailure) { + writeMutationResponseToStore(cmd.Context(), "media", data, "") + } + if wantsHumanTable(cmd.OutOrStdout(), flags) { + // Check if response contains an array (directly or wrapped in "data") + var items []map[string]any + if json.Unmarshal(data, &items) == nil && len(items) > 0 { + if err := printAutoTable(cmd.OutOrStdout(), items); err != nil { + fmt.Fprintf(os.Stderr, "warning: table rendering failed, falling back to JSON: %v\n", err) + } else { + if partialFailure != nil && !flags.allowPartialFailure { + return partialFailureErr(fmt.Errorf("partial failure in %s response: %s", "media", partialFailure.Message)) + } + return nil + } + } else { + var wrapped struct { + Data []map[string]any `json:"data"` + } + if json.Unmarshal(data, &wrapped) == nil && len(wrapped.Data) > 0 { + if err := printAutoTable(cmd.OutOrStdout(), wrapped.Data); err != nil { + fmt.Fprintf(os.Stderr, "warning: table rendering failed, falling back to JSON: %v\n", err) + } else { + if partialFailure != nil && !flags.allowPartialFailure { + return partialFailureErr(fmt.Errorf("partial failure in %s response: %s", "media", partialFailure.Message)) + } + return nil + } + } + } + } + if flags.asJSON || (!isTerminal(cmd.OutOrStdout()) && !flags.csv && !flags.quiet && !flags.plain) { + if flags.quiet { + if partialFailure != nil && !flags.allowPartialFailure { + return partialFailureErr(fmt.Errorf("partial failure in %s response: %s", "media", partialFailure.Message)) + } + return nil + } + envelope := map[string]any{ + "action": "post", + "resource": "media", + "path": path, + "status": statusCode, + "success": statusCode >= 200 && statusCode < 300 && (partialFailure == nil || flags.allowPartialFailure), + } + if partialFailure != nil { + envelope["partial_failure"] = partialFailure + } + if flags.dryRun { + envelope["dry_run"] = true + envelope["status"] = 0 + envelope["success"] = false + } + // Verify-mode synthetic envelope detection runs against RAW data + // (before --compact/--select filtering) so the sentinel field is + // guaranteed to be visible even if the operator passes a filter + // flag that would otherwise strip it. Surfaces a top-level + // verify_noop signal + flips success to false. Mirrors the dry_run + // shape above. + if len(data) > 0 { + var rawParsed any + if err := json.Unmarshal(data, &rawParsed); err == nil { + if m, ok := rawParsed.(map[string]any); ok { + if v, ok := m["__pp_verify_synthetic__"].(bool); ok && v { + envelope["verify_noop"] = true + envelope["success"] = false + } + } + } + } + // Apply --compact and --select to the API response before wrapping. + // --select wins when both are set: explicit field choice trumps the + // generic high-gravity allow-list. Otherwise --compact still applies + // when --agent is on but the user did not name fields. + filtered := data + if flags.selectFields != "" { + filtered = filterFields(filtered, flags.selectFields) + } else if flags.compact { + filtered = compactFields(filtered) + } + if len(filtered) > 0 { + var parsed any + if err := json.Unmarshal(filtered, &parsed); err == nil { + envelope["data"] = parsed + } + } + envelopeJSON, err := json.Marshal(envelope) + if err != nil { + return err + } + if perr := printOutput(cmd.OutOrStdout(), json.RawMessage(envelopeJSON), true); perr != nil { + return perr + } + if partialFailure != nil && !flags.allowPartialFailure { + return partialFailureErr(fmt.Errorf("partial failure in %s response: %s", "media", partialFailure.Message)) + } + return nil + } + // Fall-through for mutate paths that did not hit the table or + // asJSON branches: --quiet, --csv, --plain, and default terminal + // raw output. printOutputWithFlags renders the body, then the + // typed partial-failure exit fires unless --allow-partial-failure + // downgrades it. Without this guard a partial failure would exit + // 0 for these output modes — the exact silent-swallow regression + // the surrounding patch is preventing for asJSON / piped output. + if perr := printOutputWithFlags(cmd.OutOrStdout(), data, flags); perr != nil { + return perr + } + if partialFailure != nil && !flags.allowPartialFailure { + return partialFailureErr(fmt.Errorf("partial failure in %s response: %s", "media", partialFailure.Message)) + } + return nil + }, + } + cmd.Flags().StringVar(&bodyFormat, "format", "pdf", "Output format") + cmd.Flags().BoolVar(&stdinBody, "stdin", false, "Read request body as JSON from stdin") + + return cmd +} diff --git a/library/productivity/devonthink/internal/cli/media_transcribe.go b/library/productivity/devonthink/internal/cli/media_transcribe.go new file mode 100644 index 0000000000..3a53917af6 --- /dev/null +++ b/library/productivity/devonthink/internal/cli/media_transcribe.go @@ -0,0 +1,200 @@ +// Copyright 2026 rowdy and contributors. Licensed under Apache-2.0. See LICENSE. +// Generated by CLI Printing Press (https://github.com/mvanhorn/cli-printing-press). DO NOT EDIT. + +package cli + +import ( + "encoding/json" + "fmt" + "io" + "os" + + "github.com/spf13/cobra" +) + +func newMediaTranscribeCmd(flags *rootFlags) *cobra.Command { + var bodyLanguage string + var bodyTimestamps bool + var stdinBody bool + + cmd := &cobra.Command{ + Use: "transcribe <uuid>", + Short: "Transcribe audio, video, image, or PDF content", + Example: " devonthink-pp-cli media transcribe 550e8400-e29b-41d4-a716-446655440000", + Annotations: map[string]string{"pp:endpoint": "media.transcribe", "pp:method": "POST", "pp:path": "/media/{uuid}/transcribe"}, + RunE: func(cmd *cobra.Command, args []string) error { + if len(args) == 0 { + return cmd.Help() + } + if !stdinBody { + } + c, err := flags.newClient() + if err != nil { + return err + } + + path := "/media/{uuid}/transcribe" + path = replacePathParam(path, "uuid", args[0]) + params := map[string]string{} + var body map[string]any + if stdinBody { + stdinData, err := io.ReadAll(os.Stdin) + if err != nil { + return fmt.Errorf("reading stdin: %w", err) + } + var jsonBody map[string]any + if err := json.Unmarshal(stdinData, &jsonBody); err != nil { + return fmt.Errorf("parsing stdin JSON: %w", err) + } + body = jsonBody + } else { + body = map[string]any{} + if bodyLanguage != "" { + body["language"] = bodyLanguage + } + if cmd.Flags().Changed("timestamps") { + body["timestamps"] = bodyTimestamps + } + } + data, statusCode, err := c.PostWithParams(cmd.Context(), path, params, body) + if err != nil { + return classifyAPIError(err, flags) + } + // Inspect the mutate response body for a partial-failure-shaped + // field (e.g. Google Ads `partialFailureError`). Several Google + // APIs return 200 OK with a partial-failure field when some + // operations in the batch failed; ignoring it silently swallows + // real failures. Detection runs before output-mode selection so + // the exit code is consistent regardless of how stdout is + // rendered. --dry-run short-circuits because no real request + // was sent. + var partialFailure *partialFailureReport + if !flags.dryRun && statusCode >= 200 && statusCode < 300 { + partialFailure = detectPartialFailure(data) + if partialFailure != nil { + fmt.Fprintf(os.Stderr, "warning: partial failure detected in %s response: %s\n", "media", partialFailure.Message) + if len(partialFailure.ResourceNames) > 0 { + fmt.Fprintf(os.Stderr, " succeeded: %d operation(s)\n", len(partialFailure.ResourceNames)) + } + } + } + if !flags.dryRun && statusCode >= 200 && statusCode < 300 && (partialFailure == nil || flags.allowPartialFailure) { + writeMutationResponseToStore(cmd.Context(), "media", data, "") + } + if wantsHumanTable(cmd.OutOrStdout(), flags) { + // Check if response contains an array (directly or wrapped in "data") + var items []map[string]any + if json.Unmarshal(data, &items) == nil && len(items) > 0 { + if err := printAutoTable(cmd.OutOrStdout(), items); err != nil { + fmt.Fprintf(os.Stderr, "warning: table rendering failed, falling back to JSON: %v\n", err) + } else { + if partialFailure != nil && !flags.allowPartialFailure { + return partialFailureErr(fmt.Errorf("partial failure in %s response: %s", "media", partialFailure.Message)) + } + return nil + } + } else { + var wrapped struct { + Data []map[string]any `json:"data"` + } + if json.Unmarshal(data, &wrapped) == nil && len(wrapped.Data) > 0 { + if err := printAutoTable(cmd.OutOrStdout(), wrapped.Data); err != nil { + fmt.Fprintf(os.Stderr, "warning: table rendering failed, falling back to JSON: %v\n", err) + } else { + if partialFailure != nil && !flags.allowPartialFailure { + return partialFailureErr(fmt.Errorf("partial failure in %s response: %s", "media", partialFailure.Message)) + } + return nil + } + } + } + } + if flags.asJSON || (!isTerminal(cmd.OutOrStdout()) && !flags.csv && !flags.quiet && !flags.plain) { + if flags.quiet { + if partialFailure != nil && !flags.allowPartialFailure { + return partialFailureErr(fmt.Errorf("partial failure in %s response: %s", "media", partialFailure.Message)) + } + return nil + } + envelope := map[string]any{ + "action": "post", + "resource": "media", + "path": path, + "status": statusCode, + "success": statusCode >= 200 && statusCode < 300 && (partialFailure == nil || flags.allowPartialFailure), + } + if partialFailure != nil { + envelope["partial_failure"] = partialFailure + } + if flags.dryRun { + envelope["dry_run"] = true + envelope["status"] = 0 + envelope["success"] = false + } + // Verify-mode synthetic envelope detection runs against RAW data + // (before --compact/--select filtering) so the sentinel field is + // guaranteed to be visible even if the operator passes a filter + // flag that would otherwise strip it. Surfaces a top-level + // verify_noop signal + flips success to false. Mirrors the dry_run + // shape above. + if len(data) > 0 { + var rawParsed any + if err := json.Unmarshal(data, &rawParsed); err == nil { + if m, ok := rawParsed.(map[string]any); ok { + if v, ok := m["__pp_verify_synthetic__"].(bool); ok && v { + envelope["verify_noop"] = true + envelope["success"] = false + } + } + } + } + // Apply --compact and --select to the API response before wrapping. + // --select wins when both are set: explicit field choice trumps the + // generic high-gravity allow-list. Otherwise --compact still applies + // when --agent is on but the user did not name fields. + filtered := data + if flags.selectFields != "" { + filtered = filterFields(filtered, flags.selectFields) + } else if flags.compact { + filtered = compactFields(filtered) + } + if len(filtered) > 0 { + var parsed any + if err := json.Unmarshal(filtered, &parsed); err == nil { + envelope["data"] = parsed + } + } + envelopeJSON, err := json.Marshal(envelope) + if err != nil { + return err + } + if perr := printOutput(cmd.OutOrStdout(), json.RawMessage(envelopeJSON), true); perr != nil { + return perr + } + if partialFailure != nil && !flags.allowPartialFailure { + return partialFailureErr(fmt.Errorf("partial failure in %s response: %s", "media", partialFailure.Message)) + } + return nil + } + // Fall-through for mutate paths that did not hit the table or + // asJSON branches: --quiet, --csv, --plain, and default terminal + // raw output. printOutputWithFlags renders the body, then the + // typed partial-failure exit fires unless --allow-partial-failure + // downgrades it. Without this guard a partial failure would exit + // 0 for these output modes — the exact silent-swallow regression + // the surrounding patch is preventing for asJSON / piped output. + if perr := printOutputWithFlags(cmd.OutOrStdout(), data, flags); perr != nil { + return perr + } + if partialFailure != nil && !flags.allowPartialFailure { + return partialFailureErr(fmt.Errorf("partial failure in %s response: %s", "media", partialFailure.Message)) + } + return nil + }, + } + cmd.Flags().StringVar(&bodyLanguage, "language", "", "ISO language code") + cmd.Flags().BoolVar(&bodyTimestamps, "timestamps", false, "Include timestamps when available") + cmd.Flags().BoolVar(&stdinBody, "stdin", false, "Read request body as JSON from stdin") + + return cmd +} diff --git a/library/productivity/devonthink/internal/cli/mirror.go b/library/productivity/devonthink/internal/cli/mirror.go new file mode 100644 index 0000000000..ee1853b6c7 --- /dev/null +++ b/library/productivity/devonthink/internal/cli/mirror.go @@ -0,0 +1,22 @@ +// Copyright 2026 rowdy and contributors. Licensed under Apache-2.0. See LICENSE. +// Generated by CLI Printing Press (https://github.com/mvanhorn/cli-printing-press). DO NOT EDIT. + +package cli + +import ( + "github.com/spf13/cobra" +) + +func newMirrorCmd(flags *rootFlags) *cobra.Command { + cmd := &cobra.Command{ + Use: "mirror", + Short: "Get and search mirror", + Hidden: true, + Annotations: map[string]string{"mcp:read-only": "true"}, + RunE: parentNoSubcommandRunE(flags), + } + + cmd.AddCommand(newMirrorSearchCmd(flags)) + cmd.AddCommand(newMirrorSyncCmd(flags)) + return cmd +} diff --git a/library/productivity/devonthink/internal/cli/mirror_search.go b/library/productivity/devonthink/internal/cli/mirror_search.go new file mode 100644 index 0000000000..387bf06730 --- /dev/null +++ b/library/productivity/devonthink/internal/cli/mirror_search.go @@ -0,0 +1,90 @@ +// Copyright 2026 rowdy and contributors. Licensed under Apache-2.0. See LICENSE. +// Generated by CLI Printing Press (https://github.com/mvanhorn/cli-printing-press). DO NOT EDIT. + +package cli + +import ( + "encoding/json" + "fmt" + "os" + + "github.com/spf13/cobra" +) + +func newMirrorSearchCmd(flags *rootFlags) *cobra.Command { + var flagLimit int + + cmd := &cobra.Command{ + Use: "search <query>", + Short: "Search the local mirror with FTS", + Example: " devonthink-pp-cli mirror search \"tag:tax\" --limit 20 --agent --select uuid,name,path", + Annotations: map[string]string{"pp:endpoint": "mirror.search", "pp:method": "GET", "pp:path": "/mirror/search", "mcp:read-only": "true"}, + RunE: func(cmd *cobra.Command, args []string) error { + if len(args) == 0 { + return cmd.Help() + } + c, err := flags.newClient() + if err != nil { + return err + } + + path := "/mirror/search" + params := map[string]string{} + params["query"] = args[0] + if flagLimit != 0 { + params["limit"] = formatCLIParamValue(flagLimit) + } + data, prov, err := resolveReadWithStrategy(cmd.Context(), c, flags, "auto", "mirror", false, path, params, nil, cmd.ErrOrStderr()) + if err != nil { + return classifyAPIError(err, flags) + } + // Honor --limit when the API accepts but ignores ?limit=N. + data = truncateJSONArray(data, flagLimit) + // Print provenance to stderr for human-facing output only. + // Machine-format flags (--json, --csv, --compact, --quiet, --plain, + // --select) and piped stdout suppress this line; the JSON envelope + // already carries meta.source for those consumers. + // SYNC: keep this gate aligned with command_promoted.go.tmpl. + if wantsHumanTable(cmd.OutOrStdout(), flags) { + var countItems []json.RawMessage + _ = json.Unmarshal(data, &countItems) + printProvenance(cmd, len(countItems), prov) + } + // For JSON output, wrap with provenance envelope before passing through flags. + // --select wins over --compact when both are set; --compact only runs when + // no explicit fields were requested. Explicit format flags (--csv, --quiet, + // --plain) opt out of the auto-JSON path so piped consumers that asked for + // a non-JSON format reach the standard pipeline below. + if flags.asJSON || (!isTerminal(cmd.OutOrStdout()) && !flags.csv && !flags.quiet && !flags.plain) { + filtered := data + if flags.selectFields != "" { + filtered = filterFields(filtered, flags.selectFields) + } else if flags.compact { + filtered = compactFields(filtered) + } + wrapped, wrapErr := wrapWithProvenance(filtered, prov) + if wrapErr != nil { + return wrapErr + } + return printOutput(cmd.OutOrStdout(), wrapped, true) + } + // For all other output modes (table, csv, plain, quiet), use the standard pipeline + if wantsHumanTable(cmd.OutOrStdout(), flags) { + var items []map[string]any + if json.Unmarshal(data, &items) == nil && len(items) > 0 { + if err := printAutoTable(cmd.OutOrStdout(), items); err != nil { + return err + } + if len(items) >= 25 { + fmt.Fprintf(os.Stderr, "\nShowing %d results. To narrow: add --limit, --json --select, or filter flags.\n", len(items)) + } + return nil + } + } + return printOutputWithFlags(cmd.OutOrStdout(), data, flags) + }, + } + cmd.Flags().IntVar(&flagLimit, "limit", 20, "Maximum rows") + + return cmd +} diff --git a/library/productivity/devonthink/internal/cli/mirror_sync.go b/library/productivity/devonthink/internal/cli/mirror_sync.go new file mode 100644 index 0000000000..9eba015ef4 --- /dev/null +++ b/library/productivity/devonthink/internal/cli/mirror_sync.go @@ -0,0 +1,89 @@ +// Copyright 2026 rowdy and contributors. Licensed under Apache-2.0. See LICENSE. +// Generated by CLI Printing Press (https://github.com/mvanhorn/cli-printing-press). DO NOT EDIT. + +package cli + +import ( + "encoding/json" + "fmt" + "os" + + "github.com/spf13/cobra" +) + +func newMirrorSyncCmd(flags *rootFlags) *cobra.Command { + var flagDatabase string + var flagFull bool + + cmd := &cobra.Command{ + Use: "sync", + Short: "Refresh the local SQLite mirror from open DEVONthink databases", + Example: " devonthink-pp-cli mirror sync", + Annotations: map[string]string{"pp:endpoint": "mirror.sync", "pp:method": "GET", "pp:path": "/mirror/sync", "mcp:read-only": "true"}, + RunE: func(cmd *cobra.Command, args []string) error { + c, err := flags.newClient() + if err != nil { + return err + } + + path := "/mirror/sync" + params := map[string]string{} + if flagDatabase != "" { + params["database"] = formatCLIParamValue(flagDatabase) + } + if flagFull != false { + params["full"] = formatCLIParamValue(flagFull) + } + data, prov, err := resolveReadWithStrategy(cmd.Context(), c, flags, "auto", "mirror", false, path, params, nil, cmd.ErrOrStderr()) + if err != nil { + return classifyAPIError(err, flags) + } + // Print provenance to stderr for human-facing output only. + // Machine-format flags (--json, --csv, --compact, --quiet, --plain, + // --select) and piped stdout suppress this line; the JSON envelope + // already carries meta.source for those consumers. + // SYNC: keep this gate aligned with command_promoted.go.tmpl. + if wantsHumanTable(cmd.OutOrStdout(), flags) { + var countItems []json.RawMessage + _ = json.Unmarshal(data, &countItems) + printProvenance(cmd, len(countItems), prov) + } + // For JSON output, wrap with provenance envelope before passing through flags. + // --select wins over --compact when both are set; --compact only runs when + // no explicit fields were requested. Explicit format flags (--csv, --quiet, + // --plain) opt out of the auto-JSON path so piped consumers that asked for + // a non-JSON format reach the standard pipeline below. + if flags.asJSON || (!isTerminal(cmd.OutOrStdout()) && !flags.csv && !flags.quiet && !flags.plain) { + filtered := data + if flags.selectFields != "" { + filtered = filterFields(filtered, flags.selectFields) + } else if flags.compact { + filtered = compactFields(filtered) + } + wrapped, wrapErr := wrapWithProvenance(filtered, prov) + if wrapErr != nil { + return wrapErr + } + return printOutput(cmd.OutOrStdout(), wrapped, true) + } + // For all other output modes (table, csv, plain, quiet), use the standard pipeline + if wantsHumanTable(cmd.OutOrStdout(), flags) { + var items []map[string]any + if json.Unmarshal(data, &items) == nil && len(items) > 0 { + if err := printAutoTable(cmd.OutOrStdout(), items); err != nil { + return err + } + if len(items) >= 25 { + fmt.Fprintf(os.Stderr, "\nShowing %d results. To narrow: add --limit, --json --select, or filter flags.\n", len(items)) + } + return nil + } + } + return printOutputWithFlags(cmd.OutOrStdout(), data, flags) + }, + } + cmd.Flags().StringVar(&flagDatabase, "database", "", "Database name or UUID") + cmd.Flags().BoolVar(&flagFull, "full", false, "Force full sync") + + return cmd +} diff --git a/library/productivity/devonthink/internal/cli/privacy_audit.go b/library/productivity/devonthink/internal/cli/privacy_audit.go new file mode 100644 index 0000000000..86bafaf187 --- /dev/null +++ b/library/productivity/devonthink/internal/cli/privacy_audit.go @@ -0,0 +1,40 @@ +// Copyright 2026 rowdy and contributors. Licensed under Apache-2.0. See LICENSE. +// Generated by CLI Printing Press (https://github.com/mvanhorn/cli-printing-press). DO NOT EDIT. + +package cli + +import ( + "github.com/spf13/cobra" +) + +func newNovelPrivacyAuditCmd(flags *rootFlags) *cobra.Command { + var flagQuery string + var flagLimit int + + cmd := &cobra.Command{ + Use: "audit", + Short: "Preview what a workflow may expose before content leaves the local machine.", + Annotations: map[string]string{"mcp:read-only": "true"}, + RunE: func(cmd *cobra.Command, args []string) error { + c, err := flags.newClient() + if err != nil { + return err + } + params := map[string]string{} + if flagQuery != "" { + params["query"] = formatCLIParamValue(flagQuery) + } + if flagLimit != 0 { + params["limit"] = formatCLIParamValue(flagLimit) + } + data, err := c.Get(cmd.Context(), "/privacy/audit", params) + if err != nil { + return classifyAPIError(err, flags) + } + return printOutputWithFlags(cmd.OutOrStdout(), data, flags) + }, + } + cmd.Flags().StringVar(&flagQuery, "query", "", "DEVONthink query whose exposure should be audited") + cmd.Flags().IntVar(&flagLimit, "limit", 25, "Maximum records to include in the audit preview") + return cmd +} diff --git a/library/productivity/devonthink/internal/cli/privacy_audit_test.go b/library/productivity/devonthink/internal/cli/privacy_audit_test.go new file mode 100644 index 0000000000..5cdeebf278 --- /dev/null +++ b/library/productivity/devonthink/internal/cli/privacy_audit_test.go @@ -0,0 +1,20 @@ +// Copyright 2026 rowdy and contributors. Licensed under Apache-2.0. See LICENSE. +// Generated by CLI Printing Press (https://github.com/mvanhorn/cli-printing-press). DO NOT EDIT. + +package cli + +import "testing" + +func TestNovelPrivacyAuditCommandDryRun(t *testing.T) { + payload := executeNovelDryRun(t, "privacy", "audit", "--query", "invoice", "--limit", "5") + if payload["path"] != "/privacy/audit" { + t.Fatalf("path = %v, want /privacy/audit", payload["path"]) + } + params, ok := payload["params"].(map[string]any) + if !ok { + t.Fatalf("params missing from payload: %#v", payload) + } + if params["query"] != "invoice" || params["limit"] != "5" { + t.Fatalf("params = %#v", params) + } +} diff --git a/library/productivity/devonthink/internal/cli/profile.go b/library/productivity/devonthink/internal/cli/profile.go new file mode 100644 index 0000000000..dd59b99a59 --- /dev/null +++ b/library/productivity/devonthink/internal/cli/profile.go @@ -0,0 +1,351 @@ +// Copyright 2026 rowdy and contributors. Licensed under Apache-2.0. See LICENSE. +// Generated by CLI Printing Press (https://github.com/mvanhorn/cli-printing-press). DO NOT EDIT. + +package cli + +import ( + "encoding/json" + "fmt" + "os" + "path/filepath" + "sort" + "strings" + + "github.com/spf13/cobra" + "github.com/spf13/pflag" +) + +// Profile is a named set of flag values saved for reuse across invocations. +// HeyGen's "Beacon" pattern: one named context that a scheduled agent reuses +// day after day with the same voice/format but different input each run. +type Profile struct { + Name string `json:"name"` + Description string `json:"description,omitempty"` + Values map[string]string `json:"values"` +} + +type profileStore struct { + Profiles map[string]Profile `json:"profiles"` +} + +func profileStorePath() (string, error) { + home, err := os.UserHomeDir() + if err != nil { + return "", fmt.Errorf("resolving home dir: %w", err) + } + dir := filepath.Join(home, ".devonthink-pp-cli") + if err := os.MkdirAll(dir, 0o700); err != nil { + return "", fmt.Errorf("creating state dir: %w", err) + } + return filepath.Join(dir, "profiles.json"), nil +} + +func loadProfileStore() (*profileStore, error) { + p, err := profileStorePath() + if err != nil { + return nil, err + } + data, err := os.ReadFile(p) // #nosec G304 -- profile store path is resolved under the CLI config directory. + if err != nil { + if os.IsNotExist(err) { + return &profileStore{Profiles: map[string]Profile{}}, nil + } + return nil, fmt.Errorf("reading profiles: %w", err) + } + var s profileStore + if err := json.Unmarshal(data, &s); err != nil { + return nil, fmt.Errorf("parsing profiles: %w", err) + } + if s.Profiles == nil { + s.Profiles = map[string]Profile{} + } + return &s, nil +} + +func saveProfileStore(s *profileStore) error { + p, err := profileStorePath() + if err != nil { + return err + } + data, err := json.MarshalIndent(s, "", " ") + if err != nil { + return fmt.Errorf("marshaling profiles: %w", err) + } + tmp := p + ".tmp" + if err := os.WriteFile(tmp, data, 0o600); err != nil { + return fmt.Errorf("writing profiles: %w", err) + } + return os.Rename(tmp, p) +} + +// GetProfile returns a profile by name, or (nil, nil) if not found. +func GetProfile(name string) (*Profile, error) { + s, err := loadProfileStore() + if err != nil { + return nil, err + } + if p, ok := s.Profiles[name]; ok { + return &p, nil + } + return nil, nil +} + +// ApplyProfileToFlags overlays profile values onto flags that the user has +// not set explicitly on the command line. Used from root.go's +// PersistentPreRunE so profile values feed the whole command tree. +func ApplyProfileToFlags(cmd *cobra.Command, profile *Profile) error { + if profile == nil || len(profile.Values) == 0 { + return nil + } + // Reserved flags that never come from a profile - they control profile + // resolution itself or are dangerous to overlay. + reserved := map[string]bool{ + "profile": true, "config": true, "help": true, + } + for name, value := range profile.Values { + if reserved[name] { + continue + } + flag := cmd.Flags().Lookup(name) + if flag == nil { + flag = cmd.InheritedFlags().Lookup(name) + } + if flag == nil { + continue + } + if flag.Changed { + continue + } + if err := flag.Value.Set(value); err != nil { + return fmt.Errorf("applying profile value %s=%q: %w", name, value, err) + } + } + return nil +} + +// ListProfileNames returns profile names sorted alphabetically. Used by the +// agent-context subcommand to expose available_profiles at runtime. +func ListProfileNames() []string { + s, err := loadProfileStore() + if err != nil { + return nil + } + names := make([]string, 0, len(s.Profiles)) + for name := range s.Profiles { + names = append(names, name) + } + sort.Strings(names) + return names +} + +func newProfileCmd(flags *rootFlags) *cobra.Command { + cmd := &cobra.Command{ + Use: "profile", + Short: "Named sets of flags saved for reuse", + Long: `Profiles capture a set of flag values under a name so a scheduled +agent can invoke the same command with the same configuration each run. + + profile save <name> captures the current invocation's set flags + profile use <name> prints the values (for inspection) + profile list lists all saved profiles + profile show <name> shows the values of one profile + profile delete <name> removes a profile + +Use --profile <name> on any command to apply that profile's values. +Explicit flags override profile values.`, + RunE: parentNoSubcommandRunE(flags), + } + cmd.AddCommand(newProfileSaveCmd(flags)) + cmd.AddCommand(newProfileUseCmd(flags)) + cmd.AddCommand(newProfileListCmd(flags)) + cmd.AddCommand(newProfileShowCmd(flags)) + cmd.AddCommand(newProfileDeleteCmd(flags)) + return cmd +} + +func newProfileSaveCmd(flags *rootFlags) *cobra.Command { + var description string + cmd := &cobra.Command{ + Use: "save <name> [--<flag> <value> ...]", + Short: "Save the current invocation's non-default flags as a named profile", + Long: `Captures every flag explicitly set on the invocation and stores +them under <name>. To update an existing profile, run save again; the +entry is replaced. + +To avoid creating empty profiles, at least one non-default flag must be +present (other than --profile and --config).`, + Example: ` devonthink-pp-cli profile save my-defaults --json --compact + devonthink-pp-cli profile save tonight-defaults --region US`, + Args: cobra.ExactArgs(1), + RunE: func(cmd *cobra.Command, args []string) error { + name := args[0] + if strings.ContainsAny(name, `/\: `) { + return fmt.Errorf("profile name %q contains reserved characters", name) + } + values := map[string]string{} + // Walk inherited + local flags, capture only those the user set. + skip := map[string]bool{"profile": true, "config": true, "help": true, "description": true} + visit := func(fl *pflag.Flag) { + if fl.Changed && !skip[fl.Name] { + values[fl.Name] = fl.Value.String() + } + } + cmd.InheritedFlags().VisitAll(visit) + cmd.Flags().VisitAll(visit) + if len(values) == 0 { + return fmt.Errorf("no non-default flags set - pass at least one flag to save into %q", name) + } + s, err := loadProfileStore() + if err != nil { + return err + } + s.Profiles[name] = Profile{Name: name, Description: description, Values: values} + if err := saveProfileStore(s); err != nil { + return err + } + if flags.asJSON { + return printJSONFiltered(cmd.OutOrStdout(), s.Profiles[name], flags) + } + fmt.Fprintf(cmd.OutOrStdout(), "saved profile %q with %d values\n", name, len(values)) + return nil + }, + } + cmd.Flags().StringVar(&description, "description", "", "Short description shown in 'profile list'") + return cmd +} + +func newProfileUseCmd(flags *rootFlags) *cobra.Command { + return &cobra.Command{ + Use: "use <name>", + Short: "Print the flag values a profile will apply (does not execute anything)", + Example: ` devonthink-pp-cli profile use my-defaults + devonthink-pp-cli profile use tonight-defaults --json`, + Args: cobra.ExactArgs(1), + RunE: func(cmd *cobra.Command, args []string) error { + p, err := GetProfile(args[0]) + if err != nil { + return err + } + if p == nil { + return fmt.Errorf("profile %q not found", args[0]) + } + if flags.asJSON { + return printJSONFiltered(cmd.OutOrStdout(), p, flags) + } + fmt.Fprintf(cmd.OutOrStdout(), "profile %q:\n", p.Name) + if p.Description != "" { + fmt.Fprintf(cmd.OutOrStdout(), " description: %s\n", p.Description) + } + keys := make([]string, 0, len(p.Values)) + for k := range p.Values { + keys = append(keys, k) + } + sort.Strings(keys) + for _, k := range keys { + fmt.Fprintf(cmd.OutOrStdout(), " --%s %s\n", k, p.Values[k]) + } + return nil + }, + } +} + +func newProfileListCmd(flags *rootFlags) *cobra.Command { + return &cobra.Command{ + Use: "list", + Short: "List saved profiles", + Annotations: map[string]string{ + "mcp:read-only": "true", + }, + Example: ` devonthink-pp-cli profile list + devonthink-pp-cli profile list --json`, + RunE: func(cmd *cobra.Command, _ []string) error { + s, err := loadProfileStore() + if err != nil { + return err + } + names := make([]string, 0, len(s.Profiles)) + for n := range s.Profiles { + names = append(names, n) + } + sort.Strings(names) + if flags.asJSON { + out := make([]map[string]any, 0, len(names)) + for _, n := range names { + p := s.Profiles[n] + out = append(out, map[string]any{ + "name": p.Name, + "description": p.Description, + "field_count": len(p.Values), + }) + } + return printJSONFiltered(cmd.OutOrStdout(), out, flags) + } + headers := []string{"NAME", "FIELDS", "DESCRIPTION"} + rows := make([][]string, 0, len(names)) + for _, n := range names { + p := s.Profiles[n] + rows = append(rows, []string{p.Name, fmt.Sprintf("%d", len(p.Values)), p.Description}) + } + return flags.printTable(cmd, headers, rows) + }, + } +} + +func newProfileShowCmd(flags *rootFlags) *cobra.Command { + return &cobra.Command{ + Use: "show <name>", + Short: "Show a profile's values as JSON", + Annotations: map[string]string{ + "mcp:read-only": "true", + }, + Example: ` devonthink-pp-cli profile show my-defaults + devonthink-pp-cli profile show tonight-defaults --json`, + Args: cobra.ExactArgs(1), + RunE: func(cmd *cobra.Command, args []string) error { + p, err := GetProfile(args[0]) + if err != nil { + return err + } + if p == nil { + return fmt.Errorf("profile %q not found", args[0]) + } + return printJSONFiltered(cmd.OutOrStdout(), p, flags) + }, + } +} + +func newProfileDeleteCmd(flags *rootFlags) *cobra.Command { + return &cobra.Command{ + Use: "delete <name>", + Short: "Remove a profile", + Example: ` devonthink-pp-cli profile delete my-defaults --yes + devonthink-pp-cli profile delete old-profile --yes --json`, + Args: cobra.ExactArgs(1), + RunE: func(cmd *cobra.Command, args []string) error { + name := args[0] + s, err := loadProfileStore() + if err != nil { + return err + } + if _, ok := s.Profiles[name]; !ok { + return fmt.Errorf("profile %q not found", name) + } + if !flags.yes { + fmt.Fprintf(cmd.ErrOrStderr(), "refusing to delete %q without --yes\n", name) + return fmt.Errorf("confirmation required: pass --yes") + } + delete(s.Profiles, name) + if err := saveProfileStore(s); err != nil { + return err + } + // JSON envelope: {deleted: name}. + if flags.asJSON { + return printJSONFiltered(cmd.OutOrStdout(), map[string]any{ + "deleted": name, + }, flags) + } + fmt.Fprintf(cmd.OutOrStdout(), "deleted profile %q\n", name) + return nil + }, + } +} diff --git a/library/productivity/devonthink/internal/cli/promoted_context.go b/library/productivity/devonthink/internal/cli/promoted_context.go new file mode 100644 index 0000000000..8620fc761b --- /dev/null +++ b/library/productivity/devonthink/internal/cli/promoted_context.go @@ -0,0 +1,105 @@ +// Copyright 2026 rowdy and contributors. Licensed under Apache-2.0. See LICENSE. +// Generated by CLI Printing Press (https://github.com/mvanhorn/cli-printing-press). DO NOT EDIT. + +package cli + +import ( + "encoding/json" + "fmt" + "os" + + "github.com/spf13/cobra" +) + +func newContextPromotedCmd(flags *rootFlags) *cobra.Command { + var flagQuery string + var flagUuid string + var flagSelection bool + var flagTokenBudget int + + cmd := &cobra.Command{ + Use: "context", + Short: "Build a compact local context pack from records, selection, or search", + Long: "Build a compact local context pack from records, selection, or search", + Example: " devonthink-pp-cli context", + Annotations: map[string]string{"pp:endpoint": "context.pack", "pp:method": "GET", "pp:path": "/context/pack", "mcp:read-only": "true"}, + RunE: func(cmd *cobra.Command, args []string) error { + c, err := flags.newClient() + if err != nil { + return err + } + + path := "/context/pack" + params := map[string]string{} + if flagQuery != "" { + params["query"] = formatCLIParamValue(flagQuery) + } + if flagUuid != "" { + params["uuid"] = formatCLIParamValue(flagUuid) + } + if flagSelection != false { + params["selection"] = formatCLIParamValue(flagSelection) + } + if flagTokenBudget != 0 { + params["token_budget"] = formatCLIParamValue(flagTokenBudget) + } + data, prov, err := resolveReadWithStrategy(cmd.Context(), c, flags, "auto", "context", false, path, params, nil, cmd.ErrOrStderr()) + if err != nil { + return classifyAPIError(err, flags) + } + // Print provenance to stderr for human-facing output only. + // Machine-format flags (--json, --csv, --compact, --quiet, --plain, + // --select) and piped stdout suppress this line; the JSON envelope + // already carries meta.source for those consumers. + // SYNC: keep this gate aligned with command_endpoint.go.tmpl. + if wantsHumanTable(cmd.OutOrStdout(), flags) { + var countItems []json.RawMessage + if json.Unmarshal(data, &countItems) != nil { + // Single object, not an array + countItems = []json.RawMessage{data} + } + printProvenance(cmd, len(countItems), prov) + } + // For JSON output, wrap with provenance envelope. --select wins over + // --compact when both are set; --compact only runs when no explicit + // fields were requested. Explicit format flags (--csv, --quiet, --plain) + // opt out of the auto-JSON path so piped consumers that asked for a + // non-JSON format reach the standard pipeline below. + if flags.asJSON || (!isTerminal(cmd.OutOrStdout()) && !flags.csv && !flags.quiet && !flags.plain) { + filtered := data + if flags.selectFields != "" { + filtered = filterFields(filtered, flags.selectFields) + } else if flags.compact { + filtered = compactFields(filtered) + } + wrapped, wrapErr := wrapWithProvenance(filtered, prov) + if wrapErr != nil { + return wrapErr + } + return printOutput(cmd.OutOrStdout(), wrapped, true) + } + if wantsHumanTable(cmd.OutOrStdout(), flags) { + var items []map[string]any + if json.Unmarshal(data, &items) == nil && len(items) > 0 { + if err := printAutoTable(cmd.OutOrStdout(), items); err != nil { + return err + } + if len(items) >= 25 { + fmt.Fprintf(os.Stderr, "\nShowing %d results. To narrow: add --limit, --json --select, or filter flags.\n", len(items)) + } + return nil + } + } + return printOutputWithFlags(cmd.OutOrStdout(), data, flags) + }, + } + cmd.Flags().StringVar(&flagQuery, "query", "", "Search query to seed the pack") + cmd.Flags().StringVar(&flagUuid, "uuid", "", "Record UUID to seed the pack") + cmd.Flags().BoolVar(&flagSelection, "selection", false, "Use current DEVONthink selection") + cmd.Flags().IntVar(&flagTokenBudget, "token-budget", 6000, "Approximate token budget") + + // Wire sibling endpoints and sub-resources as subcommands + cmd.AddCommand(newNovelContextPackCmd(flags)) + + return cmd +} diff --git a/library/productivity/devonthink/internal/cli/promoted_databases.go b/library/productivity/devonthink/internal/cli/promoted_databases.go new file mode 100644 index 0000000000..22b2960155 --- /dev/null +++ b/library/productivity/devonthink/internal/cli/promoted_databases.go @@ -0,0 +1,84 @@ +// Copyright 2026 rowdy and contributors. Licensed under Apache-2.0. See LICENSE. +// Generated by CLI Printing Press (https://github.com/mvanhorn/cli-printing-press). DO NOT EDIT. + +package cli + +import ( + "encoding/json" + "fmt" + "os" + + "github.com/spf13/cobra" +) + +func newDatabasesPromotedCmd(flags *rootFlags) *cobra.Command { + + cmd := &cobra.Command{ + Use: "databases", + Short: "List open databases", + Long: "List open databases", + Example: " devonthink-pp-cli databases", + Annotations: map[string]string{"pp:endpoint": "databases.list", "pp:method": "GET", "pp:path": "/databases", "mcp:read-only": "true"}, + RunE: func(cmd *cobra.Command, args []string) error { + c, err := flags.newClient() + if err != nil { + return err + } + + path := "/databases" + params := map[string]string{} + data, prov, err := resolveReadWithStrategy(cmd.Context(), c, flags, "auto", "databases", true, path, params, nil, cmd.ErrOrStderr()) + if err != nil { + return classifyAPIError(err, flags) + } + // Print provenance to stderr for human-facing output only. + // Machine-format flags (--json, --csv, --compact, --quiet, --plain, + // --select) and piped stdout suppress this line; the JSON envelope + // already carries meta.source for those consumers. + // SYNC: keep this gate aligned with command_endpoint.go.tmpl. + if wantsHumanTable(cmd.OutOrStdout(), flags) { + var countItems []json.RawMessage + if json.Unmarshal(data, &countItems) != nil { + // Single object, not an array + countItems = []json.RawMessage{data} + } + printProvenance(cmd, len(countItems), prov) + } + // For JSON output, wrap with provenance envelope. --select wins over + // --compact when both are set; --compact only runs when no explicit + // fields were requested. Explicit format flags (--csv, --quiet, --plain) + // opt out of the auto-JSON path so piped consumers that asked for a + // non-JSON format reach the standard pipeline below. + if flags.asJSON || (!isTerminal(cmd.OutOrStdout()) && !flags.csv && !flags.quiet && !flags.plain) { + filtered := data + if flags.selectFields != "" { + filtered = filterFields(filtered, flags.selectFields) + } else if flags.compact { + filtered = compactFields(filtered) + } + wrapped, wrapErr := wrapWithProvenance(filtered, prov) + if wrapErr != nil { + return wrapErr + } + return printOutput(cmd.OutOrStdout(), wrapped, true) + } + if wantsHumanTable(cmd.OutOrStdout(), flags) { + var items []map[string]any + if json.Unmarshal(data, &items) == nil && len(items) > 0 { + if err := printAutoTable(cmd.OutOrStdout(), items); err != nil { + return err + } + if len(items) >= 25 { + fmt.Fprintf(os.Stderr, "\nShowing %d results. To narrow: add --limit, --json --select, or filter flags.\n", len(items)) + } + return nil + } + } + return printOutputWithFlags(cmd.OutOrStdout(), data, flags) + }, + } + + // Wire sibling endpoints and sub-resources as subcommands + + return cmd +} diff --git a/library/productivity/devonthink/internal/cli/promoted_groups.go b/library/productivity/devonthink/internal/cli/promoted_groups.go new file mode 100644 index 0000000000..abee2963ea --- /dev/null +++ b/library/productivity/devonthink/internal/cli/promoted_groups.go @@ -0,0 +1,99 @@ +// Copyright 2026 rowdy and contributors. Licensed under Apache-2.0. See LICENSE. +// Generated by CLI Printing Press (https://github.com/mvanhorn/cli-printing-press). DO NOT EDIT. + +package cli + +import ( + "encoding/json" + "fmt" + "os" + + "github.com/spf13/cobra" +) + +func newGroupsPromotedCmd(flags *rootFlags) *cobra.Command { + var flagDatabase string + var flagGroup string + var flagDepth int + + cmd := &cobra.Command{ + Use: "groups", + Short: "Render a bounded group tree", + Long: "Render a bounded group tree", + Example: " devonthink-pp-cli groups", + Annotations: map[string]string{"pp:endpoint": "groups.tree", "pp:method": "GET", "pp:path": "/groups/tree", "mcp:read-only": "true"}, + RunE: func(cmd *cobra.Command, args []string) error { + c, err := flags.newClient() + if err != nil { + return err + } + + path := "/groups/tree" + params := map[string]string{} + if flagDatabase != "" { + params["database"] = formatCLIParamValue(flagDatabase) + } + if flagGroup != "" { + params["group"] = formatCLIParamValue(flagGroup) + } + if flagDepth != 0 { + params["depth"] = formatCLIParamValue(flagDepth) + } + data, prov, err := resolveReadWithStrategy(cmd.Context(), c, flags, "auto", "groups", false, path, params, nil, cmd.ErrOrStderr()) + if err != nil { + return classifyAPIError(err, flags) + } + // Print provenance to stderr for human-facing output only. + // Machine-format flags (--json, --csv, --compact, --quiet, --plain, + // --select) and piped stdout suppress this line; the JSON envelope + // already carries meta.source for those consumers. + // SYNC: keep this gate aligned with command_endpoint.go.tmpl. + if wantsHumanTable(cmd.OutOrStdout(), flags) { + var countItems []json.RawMessage + if json.Unmarshal(data, &countItems) != nil { + // Single object, not an array + countItems = []json.RawMessage{data} + } + printProvenance(cmd, len(countItems), prov) + } + // For JSON output, wrap with provenance envelope. --select wins over + // --compact when both are set; --compact only runs when no explicit + // fields were requested. Explicit format flags (--csv, --quiet, --plain) + // opt out of the auto-JSON path so piped consumers that asked for a + // non-JSON format reach the standard pipeline below. + if flags.asJSON || (!isTerminal(cmd.OutOrStdout()) && !flags.csv && !flags.quiet && !flags.plain) { + filtered := data + if flags.selectFields != "" { + filtered = filterFields(filtered, flags.selectFields) + } else if flags.compact { + filtered = compactFields(filtered) + } + wrapped, wrapErr := wrapWithProvenance(filtered, prov) + if wrapErr != nil { + return wrapErr + } + return printOutput(cmd.OutOrStdout(), wrapped, true) + } + if wantsHumanTable(cmd.OutOrStdout(), flags) { + var items []map[string]any + if json.Unmarshal(data, &items) == nil && len(items) > 0 { + if err := printAutoTable(cmd.OutOrStdout(), items); err != nil { + return err + } + if len(items) >= 25 { + fmt.Fprintf(os.Stderr, "\nShowing %d results. To narrow: add --limit, --json --select, or filter flags.\n", len(items)) + } + return nil + } + } + return printOutputWithFlags(cmd.OutOrStdout(), data, flags) + }, + } + cmd.Flags().StringVar(&flagDatabase, "database", "", "Database name or UUID") + cmd.Flags().StringVar(&flagGroup, "group", "", "Root group UUID or path") + cmd.Flags().IntVar(&flagDepth, "depth", 2, "Maximum tree depth") + + // Wire sibling endpoints and sub-resources as subcommands + + return cmd +} diff --git a/library/productivity/devonthink/internal/cli/promoted_inventory.go b/library/productivity/devonthink/internal/cli/promoted_inventory.go new file mode 100644 index 0000000000..787926637f --- /dev/null +++ b/library/productivity/devonthink/internal/cli/promoted_inventory.go @@ -0,0 +1,105 @@ +// Copyright 2026 rowdy and contributors. Licensed under Apache-2.0. See LICENSE. +// Generated by CLI Printing Press (https://github.com/mvanhorn/cli-printing-press). DO NOT EDIT. + +package cli + +import ( + "encoding/json" + "fmt" + "os" + + "github.com/spf13/cobra" +) + +func newInventoryPromotedCmd(flags *rootFlags) *cobra.Command { + var flagDatabase string + var flagFormat string + var flagIncludeText bool + var flagOutput string + + cmd := &cobra.Command{ + Use: "inventory", + Short: "Export databases, groups, tags, and selected document metadata for downstream tools", + Long: "Export databases, groups, tags, and selected document metadata for downstream tools", + Example: " devonthink-pp-cli inventory", + Annotations: map[string]string{"pp:endpoint": "inventory.export", "pp:method": "GET", "pp:path": "/inventory/export", "mcp:read-only": "true"}, + RunE: func(cmd *cobra.Command, args []string) error { + c, err := flags.newClient() + if err != nil { + return err + } + + path := "/inventory/export" + params := map[string]string{} + if flagDatabase != "" { + params["database"] = formatCLIParamValue(flagDatabase) + } + if flagFormat != "" { + params["format"] = formatCLIParamValue(flagFormat) + } + if flagIncludeText != false { + params["include_text"] = formatCLIParamValue(flagIncludeText) + } + if flagOutput != "" { + params["output"] = formatCLIParamValue(flagOutput) + } + data, prov, err := resolveReadWithStrategy(cmd.Context(), c, flags, "auto", "inventory", false, path, params, nil, cmd.ErrOrStderr()) + if err != nil { + return classifyAPIError(err, flags) + } + // Print provenance to stderr for human-facing output only. + // Machine-format flags (--json, --csv, --compact, --quiet, --plain, + // --select) and piped stdout suppress this line; the JSON envelope + // already carries meta.source for those consumers. + // SYNC: keep this gate aligned with command_endpoint.go.tmpl. + if wantsHumanTable(cmd.OutOrStdout(), flags) { + var countItems []json.RawMessage + if json.Unmarshal(data, &countItems) != nil { + // Single object, not an array + countItems = []json.RawMessage{data} + } + printProvenance(cmd, len(countItems), prov) + } + // For JSON output, wrap with provenance envelope. --select wins over + // --compact when both are set; --compact only runs when no explicit + // fields were requested. Explicit format flags (--csv, --quiet, --plain) + // opt out of the auto-JSON path so piped consumers that asked for a + // non-JSON format reach the standard pipeline below. + if flags.asJSON || (!isTerminal(cmd.OutOrStdout()) && !flags.csv && !flags.quiet && !flags.plain) { + filtered := data + if flags.selectFields != "" { + filtered = filterFields(filtered, flags.selectFields) + } else if flags.compact { + filtered = compactFields(filtered) + } + wrapped, wrapErr := wrapWithProvenance(filtered, prov) + if wrapErr != nil { + return wrapErr + } + return printOutput(cmd.OutOrStdout(), wrapped, true) + } + if wantsHumanTable(cmd.OutOrStdout(), flags) { + var items []map[string]any + if json.Unmarshal(data, &items) == nil && len(items) > 0 { + if err := printAutoTable(cmd.OutOrStdout(), items); err != nil { + return err + } + if len(items) >= 25 { + fmt.Fprintf(os.Stderr, "\nShowing %d results. To narrow: add --limit, --json --select, or filter flags.\n", len(items)) + } + return nil + } + } + return printOutputWithFlags(cmd.OutOrStdout(), data, flags) + }, + } + cmd.Flags().StringVar(&flagDatabase, "database", "", "Database name or UUID") + cmd.Flags().StringVar(&flagFormat, "format", "maintenance", "Export format: maintenance, llm-wiki, raw") + cmd.Flags().BoolVar(&flagIncludeText, "include-text", false, "Include extracted text where safe") + cmd.Flags().StringVar(&flagOutput, "output", "", "Output JSON path") + + // Wire sibling endpoints and sub-resources as subcommands + cmd.AddCommand(newNovelInventoryExportCmd(flags)) + + return cmd +} diff --git a/library/productivity/devonthink/internal/cli/promoted_privacy.go b/library/productivity/devonthink/internal/cli/promoted_privacy.go new file mode 100644 index 0000000000..60a9f85846 --- /dev/null +++ b/library/productivity/devonthink/internal/cli/promoted_privacy.go @@ -0,0 +1,102 @@ +// Copyright 2026 rowdy and contributors. Licensed under Apache-2.0. See LICENSE. +// Generated by CLI Printing Press (https://github.com/mvanhorn/cli-printing-press). DO NOT EDIT. + +package cli + +import ( + "encoding/json" + "fmt" + "os" + + "github.com/spf13/cobra" +) + +func newPrivacyPromotedCmd(flags *rootFlags) *cobra.Command { + var flagQuery string + var flagUuid string + var flagLimit int + + cmd := &cobra.Command{ + Use: "privacy", + Short: "Preview database scope, content-size budget, and cloud/MCP exposure before handoff", + Long: "Preview database scope, content-size budget, and cloud/MCP exposure before handoff", + Example: " devonthink-pp-cli privacy", + Annotations: map[string]string{"pp:endpoint": "privacy.audit", "pp:method": "GET", "pp:path": "/privacy/audit", "mcp:read-only": "true"}, + RunE: func(cmd *cobra.Command, args []string) error { + c, err := flags.newClient() + if err != nil { + return err + } + + path := "/privacy/audit" + params := map[string]string{} + if flagQuery != "" { + params["query"] = formatCLIParamValue(flagQuery) + } + if flagUuid != "" { + params["uuid"] = formatCLIParamValue(flagUuid) + } + if flagLimit != 0 { + params["limit"] = formatCLIParamValue(flagLimit) + } + data, prov, err := resolveReadWithStrategy(cmd.Context(), c, flags, "auto", "privacy", false, path, params, nil, cmd.ErrOrStderr()) + if err != nil { + return classifyAPIError(err, flags) + } + // Honor --limit when the API accepts but ignores ?limit=N. + data = truncateJSONArray(data, flagLimit) + // Print provenance to stderr for human-facing output only. + // Machine-format flags (--json, --csv, --compact, --quiet, --plain, + // --select) and piped stdout suppress this line; the JSON envelope + // already carries meta.source for those consumers. + // SYNC: keep this gate aligned with command_endpoint.go.tmpl. + if wantsHumanTable(cmd.OutOrStdout(), flags) { + var countItems []json.RawMessage + if json.Unmarshal(data, &countItems) != nil { + // Single object, not an array + countItems = []json.RawMessage{data} + } + printProvenance(cmd, len(countItems), prov) + } + // For JSON output, wrap with provenance envelope. --select wins over + // --compact when both are set; --compact only runs when no explicit + // fields were requested. Explicit format flags (--csv, --quiet, --plain) + // opt out of the auto-JSON path so piped consumers that asked for a + // non-JSON format reach the standard pipeline below. + if flags.asJSON || (!isTerminal(cmd.OutOrStdout()) && !flags.csv && !flags.quiet && !flags.plain) { + filtered := data + if flags.selectFields != "" { + filtered = filterFields(filtered, flags.selectFields) + } else if flags.compact { + filtered = compactFields(filtered) + } + wrapped, wrapErr := wrapWithProvenance(filtered, prov) + if wrapErr != nil { + return wrapErr + } + return printOutput(cmd.OutOrStdout(), wrapped, true) + } + if wantsHumanTable(cmd.OutOrStdout(), flags) { + var items []map[string]any + if json.Unmarshal(data, &items) == nil && len(items) > 0 { + if err := printAutoTable(cmd.OutOrStdout(), items); err != nil { + return err + } + if len(items) >= 25 { + fmt.Fprintf(os.Stderr, "\nShowing %d results. To narrow: add --limit, --json --select, or filter flags.\n", len(items)) + } + return nil + } + } + return printOutputWithFlags(cmd.OutOrStdout(), data, flags) + }, + } + cmd.Flags().StringVar(&flagQuery, "query", "", "Search query to audit") + cmd.Flags().StringVar(&flagUuid, "uuid", "", "Record UUID or item link") + cmd.Flags().IntVar(&flagLimit, "limit", 20, "Maximum records to inspect") + + // Wire sibling endpoints and sub-resources as subcommands + cmd.AddCommand(newNovelPrivacyAuditCmd(flags)) + + return cmd +} diff --git a/library/productivity/devonthink/internal/cli/promoted_runtime.go b/library/productivity/devonthink/internal/cli/promoted_runtime.go new file mode 100644 index 0000000000..b99aeeee1e --- /dev/null +++ b/library/productivity/devonthink/internal/cli/promoted_runtime.go @@ -0,0 +1,84 @@ +// Copyright 2026 rowdy and contributors. Licensed under Apache-2.0. See LICENSE. +// Generated by CLI Printing Press (https://github.com/mvanhorn/cli-printing-press). DO NOT EDIT. + +package cli + +import ( + "encoding/json" + "fmt" + "os" + + "github.com/spf13/cobra" +) + +func newRuntimePromotedCmd(flags *rootFlags) *cobra.Command { + + cmd := &cobra.Command{ + Use: "runtime", + Short: "Check DEVONthink app, AppleScript, optional MCP, and local mirror readiness", + Long: "Check DEVONthink app, AppleScript, optional MCP, and local mirror readiness", + Example: " devonthink-pp-cli runtime", + Annotations: map[string]string{"pp:endpoint": "runtime.doctor", "pp:method": "GET", "pp:path": "/runtime/doctor", "mcp:read-only": "true"}, + RunE: func(cmd *cobra.Command, args []string) error { + c, err := flags.newClient() + if err != nil { + return err + } + + path := "/runtime/doctor" + params := map[string]string{} + data, prov, err := resolveReadWithStrategy(cmd.Context(), c, flags, "auto", "runtime", false, path, params, nil, cmd.ErrOrStderr()) + if err != nil { + return classifyAPIError(err, flags) + } + // Print provenance to stderr for human-facing output only. + // Machine-format flags (--json, --csv, --compact, --quiet, --plain, + // --select) and piped stdout suppress this line; the JSON envelope + // already carries meta.source for those consumers. + // SYNC: keep this gate aligned with command_endpoint.go.tmpl. + if wantsHumanTable(cmd.OutOrStdout(), flags) { + var countItems []json.RawMessage + if json.Unmarshal(data, &countItems) != nil { + // Single object, not an array + countItems = []json.RawMessage{data} + } + printProvenance(cmd, len(countItems), prov) + } + // For JSON output, wrap with provenance envelope. --select wins over + // --compact when both are set; --compact only runs when no explicit + // fields were requested. Explicit format flags (--csv, --quiet, --plain) + // opt out of the auto-JSON path so piped consumers that asked for a + // non-JSON format reach the standard pipeline below. + if flags.asJSON || (!isTerminal(cmd.OutOrStdout()) && !flags.csv && !flags.quiet && !flags.plain) { + filtered := data + if flags.selectFields != "" { + filtered = filterFields(filtered, flags.selectFields) + } else if flags.compact { + filtered = compactFields(filtered) + } + wrapped, wrapErr := wrapWithProvenance(filtered, prov) + if wrapErr != nil { + return wrapErr + } + return printOutput(cmd.OutOrStdout(), wrapped, true) + } + if wantsHumanTable(cmd.OutOrStdout(), flags) { + var items []map[string]any + if json.Unmarshal(data, &items) == nil && len(items) > 0 { + if err := printAutoTable(cmd.OutOrStdout(), items); err != nil { + return err + } + if len(items) >= 25 { + fmt.Fprintf(os.Stderr, "\nShowing %d results. To narrow: add --limit, --json --select, or filter flags.\n", len(items)) + } + return nil + } + } + return printOutputWithFlags(cmd.OutOrStdout(), data, flags) + }, + } + + // Wire sibling endpoints and sub-resources as subcommands + + return cmd +} diff --git a/library/productivity/devonthink/internal/cli/promoted_sheets.go b/library/productivity/devonthink/internal/cli/promoted_sheets.go new file mode 100644 index 0000000000..e48c89f4cd --- /dev/null +++ b/library/productivity/devonthink/internal/cli/promoted_sheets.go @@ -0,0 +1,98 @@ +// Copyright 2026 rowdy and contributors. Licensed under Apache-2.0. See LICENSE. +// Generated by CLI Printing Press (https://github.com/mvanhorn/cli-printing-press). DO NOT EDIT. + +package cli + +import ( + "encoding/json" + "fmt" + "os" + + "github.com/spf13/cobra" +) + +func newSheetsPromotedCmd(flags *rootFlags) *cobra.Command { + + cmd := &cobra.Command{ + Use: "sheets <uuid>", + Short: "Read a sheet as structured rows", + Long: "Read a sheet as structured rows", + Example: " devonthink-pp-cli sheets 550e8400-e29b-41d4-a716-446655440000", + Annotations: map[string]string{"pp:endpoint": "sheets.get", "pp:method": "GET", "pp:path": "/sheets/{uuid}", "mcp:read-only": "true"}, + RunE: func(cmd *cobra.Command, args []string) error { + c, err := flags.newClient() + if err != nil { + return err + } + + path := "/sheets/{uuid}" + if len(args) < 1 { + // JSON envelope: {error, usage}. Written first; the + // usageErr return preserves exit code 2 across modes. + if flags.asJSON { + if printErr := printJSONFiltered(cmd.OutOrStdout(), map[string]any{ + "error": "uuid is required", + "usage": fmt.Sprintf("%s <%s>", cmd.CommandPath(), "uuid"), + }, flags); printErr != nil { + return printErr + } + } + return usageErr(fmt.Errorf("uuid is required\nUsage: %s <%s>", cmd.CommandPath(), "uuid")) + } + path = replacePathParam(path, "uuid", args[0]) + params := map[string]string{} + data, prov, err := resolveReadWithStrategy(cmd.Context(), c, flags, "auto", "sheets", false, path, params, nil, cmd.ErrOrStderr()) + if err != nil { + return classifyAPIError(err, flags) + } + // Print provenance to stderr for human-facing output only. + // Machine-format flags (--json, --csv, --compact, --quiet, --plain, + // --select) and piped stdout suppress this line; the JSON envelope + // already carries meta.source for those consumers. + // SYNC: keep this gate aligned with command_endpoint.go.tmpl. + if wantsHumanTable(cmd.OutOrStdout(), flags) { + var countItems []json.RawMessage + if json.Unmarshal(data, &countItems) != nil { + // Single object, not an array + countItems = []json.RawMessage{data} + } + printProvenance(cmd, len(countItems), prov) + } + // For JSON output, wrap with provenance envelope. --select wins over + // --compact when both are set; --compact only runs when no explicit + // fields were requested. Explicit format flags (--csv, --quiet, --plain) + // opt out of the auto-JSON path so piped consumers that asked for a + // non-JSON format reach the standard pipeline below. + if flags.asJSON || (!isTerminal(cmd.OutOrStdout()) && !flags.csv && !flags.quiet && !flags.plain) { + filtered := data + if flags.selectFields != "" { + filtered = filterFields(filtered, flags.selectFields) + } else if flags.compact { + filtered = compactFields(filtered) + } + wrapped, wrapErr := wrapWithProvenance(filtered, prov) + if wrapErr != nil { + return wrapErr + } + return printOutput(cmd.OutOrStdout(), wrapped, true) + } + if wantsHumanTable(cmd.OutOrStdout(), flags) { + var items []map[string]any + if json.Unmarshal(data, &items) == nil && len(items) > 0 { + if err := printAutoTable(cmd.OutOrStdout(), items); err != nil { + return err + } + if len(items) >= 25 { + fmt.Fprintf(os.Stderr, "\nShowing %d results. To narrow: add --limit, --json --select, or filter flags.\n", len(items)) + } + return nil + } + } + return printOutputWithFlags(cmd.OutOrStdout(), data, flags) + }, + } + + // Wire sibling endpoints and sub-resources as subcommands + + return cmd +} diff --git a/library/productivity/devonthink/internal/cli/promoted_tags.go b/library/productivity/devonthink/internal/cli/promoted_tags.go new file mode 100644 index 0000000000..dd39a492e2 --- /dev/null +++ b/library/productivity/devonthink/internal/cli/promoted_tags.go @@ -0,0 +1,89 @@ +// Copyright 2026 rowdy and contributors. Licensed under Apache-2.0. See LICENSE. +// Generated by CLI Printing Press (https://github.com/mvanhorn/cli-printing-press). DO NOT EDIT. + +package cli + +import ( + "encoding/json" + "fmt" + "os" + + "github.com/spf13/cobra" +) + +func newTagsPromotedCmd(flags *rootFlags) *cobra.Command { + var flagDatabase string + + cmd := &cobra.Command{ + Use: "tags", + Short: "Analyze tags for duplicates, case drift, action tags, and maintenance tags", + Long: "Analyze tags for duplicates, case drift, action tags, and maintenance tags", + Example: " devonthink-pp-cli tags", + Annotations: map[string]string{"pp:endpoint": "tags.analyze", "pp:method": "GET", "pp:path": "/tags/analyze", "mcp:read-only": "true"}, + RunE: func(cmd *cobra.Command, args []string) error { + c, err := flags.newClient() + if err != nil { + return err + } + + path := "/tags/analyze" + params := map[string]string{} + if flagDatabase != "" { + params["database"] = formatCLIParamValue(flagDatabase) + } + data, prov, err := resolveReadWithStrategy(cmd.Context(), c, flags, "auto", "tags", false, path, params, nil, cmd.ErrOrStderr()) + if err != nil { + return classifyAPIError(err, flags) + } + // Print provenance to stderr for human-facing output only. + // Machine-format flags (--json, --csv, --compact, --quiet, --plain, + // --select) and piped stdout suppress this line; the JSON envelope + // already carries meta.source for those consumers. + // SYNC: keep this gate aligned with command_endpoint.go.tmpl. + if wantsHumanTable(cmd.OutOrStdout(), flags) { + var countItems []json.RawMessage + if json.Unmarshal(data, &countItems) != nil { + // Single object, not an array + countItems = []json.RawMessage{data} + } + printProvenance(cmd, len(countItems), prov) + } + // For JSON output, wrap with provenance envelope. --select wins over + // --compact when both are set; --compact only runs when no explicit + // fields were requested. Explicit format flags (--csv, --quiet, --plain) + // opt out of the auto-JSON path so piped consumers that asked for a + // non-JSON format reach the standard pipeline below. + if flags.asJSON || (!isTerminal(cmd.OutOrStdout()) && !flags.csv && !flags.quiet && !flags.plain) { + filtered := data + if flags.selectFields != "" { + filtered = filterFields(filtered, flags.selectFields) + } else if flags.compact { + filtered = compactFields(filtered) + } + wrapped, wrapErr := wrapWithProvenance(filtered, prov) + if wrapErr != nil { + return wrapErr + } + return printOutput(cmd.OutOrStdout(), wrapped, true) + } + if wantsHumanTable(cmd.OutOrStdout(), flags) { + var items []map[string]any + if json.Unmarshal(data, &items) == nil && len(items) > 0 { + if err := printAutoTable(cmd.OutOrStdout(), items); err != nil { + return err + } + if len(items) >= 25 { + fmt.Fprintf(os.Stderr, "\nShowing %d results. To narrow: add --limit, --json --select, or filter flags.\n", len(items)) + } + return nil + } + } + return printOutputWithFlags(cmd.OutOrStdout(), data, flags) + }, + } + cmd.Flags().StringVar(&flagDatabase, "database", "", "Database name or UUID") + + // Wire sibling endpoints and sub-resources as subcommands + + return cmd +} diff --git a/library/productivity/devonthink/internal/cli/records.go b/library/productivity/devonthink/internal/cli/records.go new file mode 100644 index 0000000000..7d554583e9 --- /dev/null +++ b/library/productivity/devonthink/internal/cli/records.go @@ -0,0 +1,30 @@ +// Copyright 2026 rowdy and contributors. Licensed under Apache-2.0. See LICENSE. +// Generated by CLI Printing Press (https://github.com/mvanhorn/cli-printing-press). DO NOT EDIT. + +package cli + +import ( + "github.com/spf13/cobra" +) + +func newRecordsCmd(flags *rootFlags) *cobra.Command { + cmd := &cobra.Command{ + Use: "records", + Short: "Get, search, create, and update records", + Hidden: true, + Annotations: map[string]string{"mcp:read-only": "true"}, + RunE: parentNoSubcommandRunE(flags), + } + + cmd.AddCommand(newRecordsContentCmd(flags)) + cmd.AddCommand(newRecordsCreateCmd(flags)) + cmd.AddCommand(newRecordsGetCmd(flags)) + cmd.AddCommand(newRecordsHighlightsCmd(flags)) + cmd.AddCommand(newRecordsLookupCmd(flags)) + cmd.AddCommand(newRecordsMoveCmd(flags)) + cmd.AddCommand(newRecordsRelatedCmd(flags)) + cmd.AddCommand(newRecordsSearchCmd(flags)) + cmd.AddCommand(newRecordsUpdateCmd(flags)) + cmd.AddCommand(newRecordsVersionsCmd(flags)) + return cmd +} diff --git a/library/productivity/devonthink/internal/cli/records_content.go b/library/productivity/devonthink/internal/cli/records_content.go new file mode 100644 index 0000000000..4350658387 --- /dev/null +++ b/library/productivity/devonthink/internal/cli/records_content.go @@ -0,0 +1,88 @@ +// Copyright 2026 rowdy and contributors. Licensed under Apache-2.0. See LICENSE. +// Generated by CLI Printing Press (https://github.com/mvanhorn/cli-printing-press). DO NOT EDIT. + +package cli + +import ( + "encoding/json" + "fmt" + "os" + + "github.com/spf13/cobra" +) + +func newRecordsContentCmd(flags *rootFlags) *cobra.Command { + var flagMaxChars int + + cmd := &cobra.Command{ + Use: "content <uuid>", + Short: "Extract text content with length and redaction controls", + Example: " devonthink-pp-cli records content 550e8400-e29b-41d4-a716-446655440000", + Annotations: map[string]string{"pp:endpoint": "records.content", "pp:method": "GET", "pp:path": "/records/{uuid}/content", "mcp:read-only": "true"}, + RunE: func(cmd *cobra.Command, args []string) error { + if len(args) == 0 { + return cmd.Help() + } + c, err := flags.newClient() + if err != nil { + return err + } + + path := "/records/{uuid}/content" + path = replacePathParam(path, "uuid", args[0]) + params := map[string]string{} + if flagMaxChars != 0 { + params["max_chars"] = formatCLIParamValue(flagMaxChars) + } + data, prov, err := resolveReadWithStrategy(cmd.Context(), c, flags, "auto", "records", false, path, params, nil, cmd.ErrOrStderr()) + if err != nil { + return classifyAPIError(err, flags) + } + // Print provenance to stderr for human-facing output only. + // Machine-format flags (--json, --csv, --compact, --quiet, --plain, + // --select) and piped stdout suppress this line; the JSON envelope + // already carries meta.source for those consumers. + // SYNC: keep this gate aligned with command_promoted.go.tmpl. + if wantsHumanTable(cmd.OutOrStdout(), flags) { + var countItems []json.RawMessage + _ = json.Unmarshal(data, &countItems) + printProvenance(cmd, len(countItems), prov) + } + // For JSON output, wrap with provenance envelope before passing through flags. + // --select wins over --compact when both are set; --compact only runs when + // no explicit fields were requested. Explicit format flags (--csv, --quiet, + // --plain) opt out of the auto-JSON path so piped consumers that asked for + // a non-JSON format reach the standard pipeline below. + if flags.asJSON || (!isTerminal(cmd.OutOrStdout()) && !flags.csv && !flags.quiet && !flags.plain) { + filtered := data + if flags.selectFields != "" { + filtered = filterFields(filtered, flags.selectFields) + } else if flags.compact { + filtered = compactFields(filtered) + } + wrapped, wrapErr := wrapWithProvenance(filtered, prov) + if wrapErr != nil { + return wrapErr + } + return printOutput(cmd.OutOrStdout(), wrapped, true) + } + // For all other output modes (table, csv, plain, quiet), use the standard pipeline + if wantsHumanTable(cmd.OutOrStdout(), flags) { + var items []map[string]any + if json.Unmarshal(data, &items) == nil && len(items) > 0 { + if err := printAutoTable(cmd.OutOrStdout(), items); err != nil { + return err + } + if len(items) >= 25 { + fmt.Fprintf(os.Stderr, "\nShowing %d results. To narrow: add --limit, --json --select, or filter flags.\n", len(items)) + } + return nil + } + } + return printOutputWithFlags(cmd.OutOrStdout(), data, flags) + }, + } + cmd.Flags().IntVar(&flagMaxChars, "max-chars", 8000, "Maximum characters to emit") + + return cmd +} diff --git a/library/productivity/devonthink/internal/cli/records_create.go b/library/productivity/devonthink/internal/cli/records_create.go new file mode 100644 index 0000000000..4103869728 --- /dev/null +++ b/library/productivity/devonthink/internal/cli/records_create.go @@ -0,0 +1,220 @@ +// Copyright 2026 rowdy and contributors. Licensed under Apache-2.0. See LICENSE. +// Generated by CLI Printing Press (https://github.com/mvanhorn/cli-printing-press). DO NOT EDIT. + +package cli + +import ( + "encoding/json" + "fmt" + "io" + "os" + + "github.com/spf13/cobra" +) + +func newRecordsCreateCmd(flags *rootFlags) *cobra.Command { + var bodyTitle string + var bodyType string + var bodyDestination string + var bodyContent string + var bodyTags string + var stdinBody bool + + cmd := &cobra.Command{ + Use: "create", + Short: "Create a record or group after validating destination", + Example: " devonthink-pp-cli records create --title example-resource", + Annotations: map[string]string{"pp:endpoint": "records.create", "pp:method": "POST", "pp:path": "/records/create"}, + RunE: func(cmd *cobra.Command, args []string) error { + // Bare invocation of a command with required input prints help + // instead of pflag's terse "required flag not set" error. Optional- + // only read commands fall through so a bare call still executes. + if cmd.Flags().NFlag() == 0 && len(args) == 0 && !flags.dryRun { + return cmd.Help() + } + if !stdinBody { + if !cmd.Flags().Changed("title") && !flags.dryRun { + return fmt.Errorf("required flag \"%s\" not set", "title") + } + } + c, err := flags.newClient() + if err != nil { + return err + } + + path := "/records/create" + params := map[string]string{} + var body map[string]any + if stdinBody { + stdinData, err := io.ReadAll(os.Stdin) + if err != nil { + return fmt.Errorf("reading stdin: %w", err) + } + var jsonBody map[string]any + if err := json.Unmarshal(stdinData, &jsonBody); err != nil { + return fmt.Errorf("parsing stdin JSON: %w", err) + } + body = jsonBody + } else { + body = map[string]any{} + if bodyTitle != "" { + body["title"] = bodyTitle + } + if bodyType != "" { + body["type"] = bodyType + } + if bodyDestination != "" { + body["destination"] = bodyDestination + } + if bodyContent != "" { + body["content"] = bodyContent + } + if bodyTags != "" { + body["tags"] = bodyTags + } + } + data, statusCode, err := c.PostWithParams(cmd.Context(), path, params, body) + if err != nil { + return classifyAPIError(err, flags) + } + // Inspect the mutate response body for a partial-failure-shaped + // field (e.g. Google Ads `partialFailureError`). Several Google + // APIs return 200 OK with a partial-failure field when some + // operations in the batch failed; ignoring it silently swallows + // real failures. Detection runs before output-mode selection so + // the exit code is consistent regardless of how stdout is + // rendered. --dry-run short-circuits because no real request + // was sent. + var partialFailure *partialFailureReport + if !flags.dryRun && statusCode >= 200 && statusCode < 300 { + partialFailure = detectPartialFailure(data) + if partialFailure != nil { + fmt.Fprintf(os.Stderr, "warning: partial failure detected in %s response: %s\n", "records", partialFailure.Message) + if len(partialFailure.ResourceNames) > 0 { + fmt.Fprintf(os.Stderr, " succeeded: %d operation(s)\n", len(partialFailure.ResourceNames)) + } + } + } + if !flags.dryRun && statusCode >= 200 && statusCode < 300 && (partialFailure == nil || flags.allowPartialFailure) { + writeMutationResponseToStore(cmd.Context(), "records", data, "") + } + if wantsHumanTable(cmd.OutOrStdout(), flags) { + // Check if response contains an array (directly or wrapped in "data") + var items []map[string]any + if json.Unmarshal(data, &items) == nil && len(items) > 0 { + if err := printAutoTable(cmd.OutOrStdout(), items); err != nil { + fmt.Fprintf(os.Stderr, "warning: table rendering failed, falling back to JSON: %v\n", err) + } else { + if partialFailure != nil && !flags.allowPartialFailure { + return partialFailureErr(fmt.Errorf("partial failure in %s response: %s", "records", partialFailure.Message)) + } + return nil + } + } else { + var wrapped struct { + Data []map[string]any `json:"data"` + } + if json.Unmarshal(data, &wrapped) == nil && len(wrapped.Data) > 0 { + if err := printAutoTable(cmd.OutOrStdout(), wrapped.Data); err != nil { + fmt.Fprintf(os.Stderr, "warning: table rendering failed, falling back to JSON: %v\n", err) + } else { + if partialFailure != nil && !flags.allowPartialFailure { + return partialFailureErr(fmt.Errorf("partial failure in %s response: %s", "records", partialFailure.Message)) + } + return nil + } + } + } + } + if flags.asJSON || (!isTerminal(cmd.OutOrStdout()) && !flags.csv && !flags.quiet && !flags.plain) { + if flags.quiet { + if partialFailure != nil && !flags.allowPartialFailure { + return partialFailureErr(fmt.Errorf("partial failure in %s response: %s", "records", partialFailure.Message)) + } + return nil + } + envelope := map[string]any{ + "action": "post", + "resource": "records", + "path": path, + "status": statusCode, + "success": statusCode >= 200 && statusCode < 300 && (partialFailure == nil || flags.allowPartialFailure), + } + if partialFailure != nil { + envelope["partial_failure"] = partialFailure + } + if flags.dryRun { + envelope["dry_run"] = true + envelope["status"] = 0 + envelope["success"] = false + } + // Verify-mode synthetic envelope detection runs against RAW data + // (before --compact/--select filtering) so the sentinel field is + // guaranteed to be visible even if the operator passes a filter + // flag that would otherwise strip it. Surfaces a top-level + // verify_noop signal + flips success to false. Mirrors the dry_run + // shape above. + if len(data) > 0 { + var rawParsed any + if err := json.Unmarshal(data, &rawParsed); err == nil { + if m, ok := rawParsed.(map[string]any); ok { + if v, ok := m["__pp_verify_synthetic__"].(bool); ok && v { + envelope["verify_noop"] = true + envelope["success"] = false + } + } + } + } + // Apply --compact and --select to the API response before wrapping. + // --select wins when both are set: explicit field choice trumps the + // generic high-gravity allow-list. Otherwise --compact still applies + // when --agent is on but the user did not name fields. + filtered := data + if flags.selectFields != "" { + filtered = filterFields(filtered, flags.selectFields) + } else if flags.compact { + filtered = compactFields(filtered) + } + if len(filtered) > 0 { + var parsed any + if err := json.Unmarshal(filtered, &parsed); err == nil { + envelope["data"] = parsed + } + } + envelopeJSON, err := json.Marshal(envelope) + if err != nil { + return err + } + if perr := printOutput(cmd.OutOrStdout(), json.RawMessage(envelopeJSON), true); perr != nil { + return perr + } + if partialFailure != nil && !flags.allowPartialFailure { + return partialFailureErr(fmt.Errorf("partial failure in %s response: %s", "records", partialFailure.Message)) + } + return nil + } + // Fall-through for mutate paths that did not hit the table or + // asJSON branches: --quiet, --csv, --plain, and default terminal + // raw output. printOutputWithFlags renders the body, then the + // typed partial-failure exit fires unless --allow-partial-failure + // downgrades it. Without this guard a partial failure would exit + // 0 for these output modes — the exact silent-swallow regression + // the surrounding patch is preventing for asJSON / piped output. + if perr := printOutputWithFlags(cmd.OutOrStdout(), data, flags); perr != nil { + return perr + } + if partialFailure != nil && !flags.allowPartialFailure { + return partialFailureErr(fmt.Errorf("partial failure in %s response: %s", "records", partialFailure.Message)) + } + return nil + }, + } + cmd.Flags().StringVar(&bodyTitle, "title", "", "Record title") + cmd.Flags().StringVar(&bodyType, "type", "markdown", "Record type") + cmd.Flags().StringVar(&bodyDestination, "destination", "", "Destination group path or UUID") + cmd.Flags().StringVar(&bodyContent, "content", "", "Inline text content") + cmd.Flags().StringVar(&bodyTags, "tags", "", "Comma-separated tags") + cmd.Flags().BoolVar(&stdinBody, "stdin", false, "Read request body as JSON from stdin") + + return cmd +} diff --git a/library/productivity/devonthink/internal/cli/records_get.go b/library/productivity/devonthink/internal/cli/records_get.go new file mode 100644 index 0000000000..657435d47f --- /dev/null +++ b/library/productivity/devonthink/internal/cli/records_get.go @@ -0,0 +1,83 @@ +// Copyright 2026 rowdy and contributors. Licensed under Apache-2.0. See LICENSE. +// Generated by CLI Printing Press (https://github.com/mvanhorn/cli-printing-press). DO NOT EDIT. + +package cli + +import ( + "encoding/json" + "fmt" + "os" + + "github.com/spf13/cobra" +) + +func newRecordsGetCmd(flags *rootFlags) *cobra.Command { + + cmd := &cobra.Command{ + Use: "get <uuid>", + Short: "Get record metadata", + Example: " devonthink-pp-cli records get 550e8400-e29b-41d4-a716-446655440000", + Annotations: map[string]string{"pp:endpoint": "records.get", "pp:method": "GET", "pp:path": "/records/{uuid}", "mcp:read-only": "true"}, + RunE: func(cmd *cobra.Command, args []string) error { + if len(args) == 0 { + return cmd.Help() + } + c, err := flags.newClient() + if err != nil { + return err + } + + path := "/records/{uuid}" + path = replacePathParam(path, "uuid", args[0]) + params := map[string]string{} + data, prov, err := resolveReadWithStrategy(cmd.Context(), c, flags, "auto", "records", false, path, params, nil, cmd.ErrOrStderr()) + if err != nil { + return classifyAPIError(err, flags) + } + // Print provenance to stderr for human-facing output only. + // Machine-format flags (--json, --csv, --compact, --quiet, --plain, + // --select) and piped stdout suppress this line; the JSON envelope + // already carries meta.source for those consumers. + // SYNC: keep this gate aligned with command_promoted.go.tmpl. + if wantsHumanTable(cmd.OutOrStdout(), flags) { + var countItems []json.RawMessage + _ = json.Unmarshal(data, &countItems) + printProvenance(cmd, len(countItems), prov) + } + // For JSON output, wrap with provenance envelope before passing through flags. + // --select wins over --compact when both are set; --compact only runs when + // no explicit fields were requested. Explicit format flags (--csv, --quiet, + // --plain) opt out of the auto-JSON path so piped consumers that asked for + // a non-JSON format reach the standard pipeline below. + if flags.asJSON || (!isTerminal(cmd.OutOrStdout()) && !flags.csv && !flags.quiet && !flags.plain) { + filtered := data + if flags.selectFields != "" { + filtered = filterFields(filtered, flags.selectFields) + } else if flags.compact { + filtered = compactFields(filtered) + } + wrapped, wrapErr := wrapWithProvenance(filtered, prov) + if wrapErr != nil { + return wrapErr + } + return printOutput(cmd.OutOrStdout(), wrapped, true) + } + // For all other output modes (table, csv, plain, quiet), use the standard pipeline + if wantsHumanTable(cmd.OutOrStdout(), flags) { + var items []map[string]any + if json.Unmarshal(data, &items) == nil && len(items) > 0 { + if err := printAutoTable(cmd.OutOrStdout(), items); err != nil { + return err + } + if len(items) >= 25 { + fmt.Fprintf(os.Stderr, "\nShowing %d results. To narrow: add --limit, --json --select, or filter flags.\n", len(items)) + } + return nil + } + } + return printOutputWithFlags(cmd.OutOrStdout(), data, flags) + }, + } + + return cmd +} diff --git a/library/productivity/devonthink/internal/cli/records_highlights.go b/library/productivity/devonthink/internal/cli/records_highlights.go new file mode 100644 index 0000000000..13abc23e11 --- /dev/null +++ b/library/productivity/devonthink/internal/cli/records_highlights.go @@ -0,0 +1,83 @@ +// Copyright 2026 rowdy and contributors. Licensed under Apache-2.0. See LICENSE. +// Generated by CLI Printing Press (https://github.com/mvanhorn/cli-printing-press). DO NOT EDIT. + +package cli + +import ( + "encoding/json" + "fmt" + "os" + + "github.com/spf13/cobra" +) + +func newRecordsHighlightsCmd(flags *rootFlags) *cobra.Command { + + cmd := &cobra.Command{ + Use: "highlights <uuid>", + Short: "Extract highlights and annotations", + Example: " devonthink-pp-cli records highlights 550e8400-e29b-41d4-a716-446655440000", + Annotations: map[string]string{"pp:endpoint": "records.highlights", "pp:method": "GET", "pp:path": "/records/{uuid}/highlights", "mcp:read-only": "true"}, + RunE: func(cmd *cobra.Command, args []string) error { + if len(args) == 0 { + return cmd.Help() + } + c, err := flags.newClient() + if err != nil { + return err + } + + path := "/records/{uuid}/highlights" + path = replacePathParam(path, "uuid", args[0]) + params := map[string]string{} + data, prov, err := resolveReadWithStrategy(cmd.Context(), c, flags, "auto", "records", false, path, params, nil, cmd.ErrOrStderr()) + if err != nil { + return classifyAPIError(err, flags) + } + // Print provenance to stderr for human-facing output only. + // Machine-format flags (--json, --csv, --compact, --quiet, --plain, + // --select) and piped stdout suppress this line; the JSON envelope + // already carries meta.source for those consumers. + // SYNC: keep this gate aligned with command_promoted.go.tmpl. + if wantsHumanTable(cmd.OutOrStdout(), flags) { + var countItems []json.RawMessage + _ = json.Unmarshal(data, &countItems) + printProvenance(cmd, len(countItems), prov) + } + // For JSON output, wrap with provenance envelope before passing through flags. + // --select wins over --compact when both are set; --compact only runs when + // no explicit fields were requested. Explicit format flags (--csv, --quiet, + // --plain) opt out of the auto-JSON path so piped consumers that asked for + // a non-JSON format reach the standard pipeline below. + if flags.asJSON || (!isTerminal(cmd.OutOrStdout()) && !flags.csv && !flags.quiet && !flags.plain) { + filtered := data + if flags.selectFields != "" { + filtered = filterFields(filtered, flags.selectFields) + } else if flags.compact { + filtered = compactFields(filtered) + } + wrapped, wrapErr := wrapWithProvenance(filtered, prov) + if wrapErr != nil { + return wrapErr + } + return printOutput(cmd.OutOrStdout(), wrapped, true) + } + // For all other output modes (table, csv, plain, quiet), use the standard pipeline + if wantsHumanTable(cmd.OutOrStdout(), flags) { + var items []map[string]any + if json.Unmarshal(data, &items) == nil && len(items) > 0 { + if err := printAutoTable(cmd.OutOrStdout(), items); err != nil { + return err + } + if len(items) >= 25 { + fmt.Fprintf(os.Stderr, "\nShowing %d results. To narrow: add --limit, --json --select, or filter flags.\n", len(items)) + } + return nil + } + } + return printOutputWithFlags(cmd.OutOrStdout(), data, flags) + }, + } + + return cmd +} diff --git a/library/productivity/devonthink/internal/cli/records_lookup.go b/library/productivity/devonthink/internal/cli/records_lookup.go new file mode 100644 index 0000000000..68bac4f3b4 --- /dev/null +++ b/library/productivity/devonthink/internal/cli/records_lookup.go @@ -0,0 +1,109 @@ +// Copyright 2026 rowdy and contributors. Licensed under Apache-2.0. See LICENSE. +// Generated by CLI Printing Press (https://github.com/mvanhorn/cli-printing-press). DO NOT EDIT. + +package cli + +import ( + "encoding/json" + "fmt" + "os" + + "github.com/spf13/cobra" +) + +func newRecordsLookupCmd(flags *rootFlags) *cobra.Command { + var flagName string + var flagUrl string + var flagPath string + var flagLocation string + var flagFilename string + var flagComment string + + cmd := &cobra.Command{ + Use: "lookup", + Short: "Look up records by exact name, URL, path, filename, location, or comment", + Example: " devonthink-pp-cli records lookup", + Annotations: map[string]string{"pp:endpoint": "records.lookup", "pp:method": "GET", "pp:path": "/records/lookup", "mcp:read-only": "true"}, + RunE: func(cmd *cobra.Command, args []string) error { + c, err := flags.newClient() + if err != nil { + return err + } + + path := "/records/lookup" + params := map[string]string{} + if flagName != "" { + params["name"] = formatCLIParamValue(flagName) + } + if flagUrl != "" { + params["url"] = formatCLIParamValue(flagUrl) + } + if flagPath != "" { + params["path"] = formatCLIParamValue(flagPath) + } + if flagLocation != "" { + params["location"] = formatCLIParamValue(flagLocation) + } + if flagFilename != "" { + params["filename"] = formatCLIParamValue(flagFilename) + } + if flagComment != "" { + params["comment"] = formatCLIParamValue(flagComment) + } + data, prov, err := resolveReadWithStrategy(cmd.Context(), c, flags, "auto", "records", false, path, params, nil, cmd.ErrOrStderr()) + if err != nil { + return classifyAPIError(err, flags) + } + // Print provenance to stderr for human-facing output only. + // Machine-format flags (--json, --csv, --compact, --quiet, --plain, + // --select) and piped stdout suppress this line; the JSON envelope + // already carries meta.source for those consumers. + // SYNC: keep this gate aligned with command_promoted.go.tmpl. + if wantsHumanTable(cmd.OutOrStdout(), flags) { + var countItems []json.RawMessage + _ = json.Unmarshal(data, &countItems) + printProvenance(cmd, len(countItems), prov) + } + // For JSON output, wrap with provenance envelope before passing through flags. + // --select wins over --compact when both are set; --compact only runs when + // no explicit fields were requested. Explicit format flags (--csv, --quiet, + // --plain) opt out of the auto-JSON path so piped consumers that asked for + // a non-JSON format reach the standard pipeline below. + if flags.asJSON || (!isTerminal(cmd.OutOrStdout()) && !flags.csv && !flags.quiet && !flags.plain) { + filtered := data + if flags.selectFields != "" { + filtered = filterFields(filtered, flags.selectFields) + } else if flags.compact { + filtered = compactFields(filtered) + } + wrapped, wrapErr := wrapWithProvenance(filtered, prov) + if wrapErr != nil { + return wrapErr + } + return printOutput(cmd.OutOrStdout(), wrapped, true) + } + // For all other output modes (table, csv, plain, quiet), use the standard pipeline + if wantsHumanTable(cmd.OutOrStdout(), flags) { + var items []map[string]any + if json.Unmarshal(data, &items) == nil && len(items) > 0 { + if err := printAutoTable(cmd.OutOrStdout(), items); err != nil { + return err + } + if len(items) >= 25 { + fmt.Fprintf(os.Stderr, "\nShowing %d results. To narrow: add --limit, --json --select, or filter flags.\n", len(items)) + } + return nil + } + } + return printOutputWithFlags(cmd.OutOrStdout(), data, flags) + }, + } + cmd.Flags().StringVar(&flagName, "name", "", "Exact record name") + cmd.Flags().StringVar(&flagUrl, "url", "", "Exact record URL") + cmd.Flags().StringVar(&flagPath, "path", "", "Exact filesystem path") + cmd.Flags().StringVar(&flagLocation, "location", "", "Exact DEVONthink location") + cmd.Flags().StringVar(&flagFilename, "filename", "", "Exact filename") + cmd.Flags().StringVar(&flagComment, "comment", "", "Exact comment") + + return cmd +} diff --git a/library/productivity/devonthink/internal/cli/records_move.go b/library/productivity/devonthink/internal/cli/records_move.go new file mode 100644 index 0000000000..835ff3527f --- /dev/null +++ b/library/productivity/devonthink/internal/cli/records_move.go @@ -0,0 +1,205 @@ +// Copyright 2026 rowdy and contributors. Licensed under Apache-2.0. See LICENSE. +// Generated by CLI Printing Press (https://github.com/mvanhorn/cli-printing-press). DO NOT EDIT. + +package cli + +import ( + "encoding/json" + "fmt" + "io" + "os" + + "github.com/spf13/cobra" +) + +func newRecordsMoveCmd(flags *rootFlags) *cobra.Command { + var bodyDestination string + var bodyMode string + var bodyDryRun bool + var stdinBody bool + + cmd := &cobra.Command{ + Use: "move <uuid>", + Short: "Move, duplicate, replicate, or trash a record with dry-run proof", + Example: " devonthink-pp-cli records move 550e8400-e29b-41d4-a716-446655440000", + Annotations: map[string]string{"pp:endpoint": "records.move", "pp:method": "POST", "pp:path": "/records/{uuid}/move"}, + RunE: func(cmd *cobra.Command, args []string) error { + if len(args) == 0 { + return cmd.Help() + } + if !stdinBody { + } + c, err := flags.newClient() + if err != nil { + return err + } + + path := "/records/{uuid}/move" + path = replacePathParam(path, "uuid", args[0]) + params := map[string]string{} + var body map[string]any + if stdinBody { + stdinData, err := io.ReadAll(os.Stdin) + if err != nil { + return fmt.Errorf("reading stdin: %w", err) + } + var jsonBody map[string]any + if err := json.Unmarshal(stdinData, &jsonBody); err != nil { + return fmt.Errorf("parsing stdin JSON: %w", err) + } + body = jsonBody + } else { + body = map[string]any{} + if bodyDestination != "" { + body["destination"] = bodyDestination + } + if bodyMode != "" { + body["mode"] = bodyMode + } + if cmd.Flags().Changed("dry-run") { + body["dry_run"] = bodyDryRun + } + } + data, statusCode, err := c.PostWithParams(cmd.Context(), path, params, body) + if err != nil { + return classifyAPIError(err, flags) + } + // Inspect the mutate response body for a partial-failure-shaped + // field (e.g. Google Ads `partialFailureError`). Several Google + // APIs return 200 OK with a partial-failure field when some + // operations in the batch failed; ignoring it silently swallows + // real failures. Detection runs before output-mode selection so + // the exit code is consistent regardless of how stdout is + // rendered. --dry-run short-circuits because no real request + // was sent. + var partialFailure *partialFailureReport + if !flags.dryRun && statusCode >= 200 && statusCode < 300 { + partialFailure = detectPartialFailure(data) + if partialFailure != nil { + fmt.Fprintf(os.Stderr, "warning: partial failure detected in %s response: %s\n", "records", partialFailure.Message) + if len(partialFailure.ResourceNames) > 0 { + fmt.Fprintf(os.Stderr, " succeeded: %d operation(s)\n", len(partialFailure.ResourceNames)) + } + } + } + if !flags.dryRun && statusCode >= 200 && statusCode < 300 && (partialFailure == nil || flags.allowPartialFailure) { + writeMutationResponseToStore(cmd.Context(), "records", data, "") + } + if wantsHumanTable(cmd.OutOrStdout(), flags) { + // Check if response contains an array (directly or wrapped in "data") + var items []map[string]any + if json.Unmarshal(data, &items) == nil && len(items) > 0 { + if err := printAutoTable(cmd.OutOrStdout(), items); err != nil { + fmt.Fprintf(os.Stderr, "warning: table rendering failed, falling back to JSON: %v\n", err) + } else { + if partialFailure != nil && !flags.allowPartialFailure { + return partialFailureErr(fmt.Errorf("partial failure in %s response: %s", "records", partialFailure.Message)) + } + return nil + } + } else { + var wrapped struct { + Data []map[string]any `json:"data"` + } + if json.Unmarshal(data, &wrapped) == nil && len(wrapped.Data) > 0 { + if err := printAutoTable(cmd.OutOrStdout(), wrapped.Data); err != nil { + fmt.Fprintf(os.Stderr, "warning: table rendering failed, falling back to JSON: %v\n", err) + } else { + if partialFailure != nil && !flags.allowPartialFailure { + return partialFailureErr(fmt.Errorf("partial failure in %s response: %s", "records", partialFailure.Message)) + } + return nil + } + } + } + } + if flags.asJSON || (!isTerminal(cmd.OutOrStdout()) && !flags.csv && !flags.quiet && !flags.plain) { + if flags.quiet { + if partialFailure != nil && !flags.allowPartialFailure { + return partialFailureErr(fmt.Errorf("partial failure in %s response: %s", "records", partialFailure.Message)) + } + return nil + } + envelope := map[string]any{ + "action": "post", + "resource": "records", + "path": path, + "status": statusCode, + "success": statusCode >= 200 && statusCode < 300 && (partialFailure == nil || flags.allowPartialFailure), + } + if partialFailure != nil { + envelope["partial_failure"] = partialFailure + } + if flags.dryRun { + envelope["dry_run"] = true + envelope["status"] = 0 + envelope["success"] = false + } + // Verify-mode synthetic envelope detection runs against RAW data + // (before --compact/--select filtering) so the sentinel field is + // guaranteed to be visible even if the operator passes a filter + // flag that would otherwise strip it. Surfaces a top-level + // verify_noop signal + flips success to false. Mirrors the dry_run + // shape above. + if len(data) > 0 { + var rawParsed any + if err := json.Unmarshal(data, &rawParsed); err == nil { + if m, ok := rawParsed.(map[string]any); ok { + if v, ok := m["__pp_verify_synthetic__"].(bool); ok && v { + envelope["verify_noop"] = true + envelope["success"] = false + } + } + } + } + // Apply --compact and --select to the API response before wrapping. + // --select wins when both are set: explicit field choice trumps the + // generic high-gravity allow-list. Otherwise --compact still applies + // when --agent is on but the user did not name fields. + filtered := data + if flags.selectFields != "" { + filtered = filterFields(filtered, flags.selectFields) + } else if flags.compact { + filtered = compactFields(filtered) + } + if len(filtered) > 0 { + var parsed any + if err := json.Unmarshal(filtered, &parsed); err == nil { + envelope["data"] = parsed + } + } + envelopeJSON, err := json.Marshal(envelope) + if err != nil { + return err + } + if perr := printOutput(cmd.OutOrStdout(), json.RawMessage(envelopeJSON), true); perr != nil { + return perr + } + if partialFailure != nil && !flags.allowPartialFailure { + return partialFailureErr(fmt.Errorf("partial failure in %s response: %s", "records", partialFailure.Message)) + } + return nil + } + // Fall-through for mutate paths that did not hit the table or + // asJSON branches: --quiet, --csv, --plain, and default terminal + // raw output. printOutputWithFlags renders the body, then the + // typed partial-failure exit fires unless --allow-partial-failure + // downgrades it. Without this guard a partial failure would exit + // 0 for these output modes — the exact silent-swallow regression + // the surrounding patch is preventing for asJSON / piped output. + if perr := printOutputWithFlags(cmd.OutOrStdout(), data, flags); perr != nil { + return perr + } + if partialFailure != nil && !flags.allowPartialFailure { + return partialFailureErr(fmt.Errorf("partial failure in %s response: %s", "records", partialFailure.Message)) + } + return nil + }, + } + cmd.Flags().StringVar(&bodyDestination, "destination", "", "Destination group path or UUID") + cmd.Flags().StringVar(&bodyMode, "mode", "move", "Operation mode: move, duplicate, replicate, trash") + cmd.Flags().BoolVar(&bodyDryRun, "dry-run", false, "Show the planned move without writing") + cmd.Flags().BoolVar(&stdinBody, "stdin", false, "Read request body as JSON from stdin") + + return cmd +} diff --git a/library/productivity/devonthink/internal/cli/records_related.go b/library/productivity/devonthink/internal/cli/records_related.go new file mode 100644 index 0000000000..2d3f0ee815 --- /dev/null +++ b/library/productivity/devonthink/internal/cli/records_related.go @@ -0,0 +1,90 @@ +// Copyright 2026 rowdy and contributors. Licensed under Apache-2.0. See LICENSE. +// Generated by CLI Printing Press (https://github.com/mvanhorn/cli-printing-press). DO NOT EDIT. + +package cli + +import ( + "encoding/json" + "fmt" + "os" + + "github.com/spf13/cobra" +) + +func newRecordsRelatedCmd(flags *rootFlags) *cobra.Command { + var flagLimit int + + cmd := &cobra.Command{ + Use: "related <uuid>", + Short: "Find related records using DEVONthink similarity", + Example: " devonthink-pp-cli records related 550e8400-e29b-41d4-a716-446655440000", + Annotations: map[string]string{"pp:endpoint": "records.related", "pp:method": "GET", "pp:path": "/records/{uuid}/related", "mcp:read-only": "true"}, + RunE: func(cmd *cobra.Command, args []string) error { + if len(args) == 0 { + return cmd.Help() + } + c, err := flags.newClient() + if err != nil { + return err + } + + path := "/records/{uuid}/related" + path = replacePathParam(path, "uuid", args[0]) + params := map[string]string{} + if flagLimit != 0 { + params["limit"] = formatCLIParamValue(flagLimit) + } + data, prov, err := resolveReadWithStrategy(cmd.Context(), c, flags, "auto", "records", false, path, params, nil, cmd.ErrOrStderr()) + if err != nil { + return classifyAPIError(err, flags) + } + // Honor --limit when the API accepts but ignores ?limit=N. + data = truncateJSONArray(data, flagLimit) + // Print provenance to stderr for human-facing output only. + // Machine-format flags (--json, --csv, --compact, --quiet, --plain, + // --select) and piped stdout suppress this line; the JSON envelope + // already carries meta.source for those consumers. + // SYNC: keep this gate aligned with command_promoted.go.tmpl. + if wantsHumanTable(cmd.OutOrStdout(), flags) { + var countItems []json.RawMessage + _ = json.Unmarshal(data, &countItems) + printProvenance(cmd, len(countItems), prov) + } + // For JSON output, wrap with provenance envelope before passing through flags. + // --select wins over --compact when both are set; --compact only runs when + // no explicit fields were requested. Explicit format flags (--csv, --quiet, + // --plain) opt out of the auto-JSON path so piped consumers that asked for + // a non-JSON format reach the standard pipeline below. + if flags.asJSON || (!isTerminal(cmd.OutOrStdout()) && !flags.csv && !flags.quiet && !flags.plain) { + filtered := data + if flags.selectFields != "" { + filtered = filterFields(filtered, flags.selectFields) + } else if flags.compact { + filtered = compactFields(filtered) + } + wrapped, wrapErr := wrapWithProvenance(filtered, prov) + if wrapErr != nil { + return wrapErr + } + return printOutput(cmd.OutOrStdout(), wrapped, true) + } + // For all other output modes (table, csv, plain, quiet), use the standard pipeline + if wantsHumanTable(cmd.OutOrStdout(), flags) { + var items []map[string]any + if json.Unmarshal(data, &items) == nil && len(items) > 0 { + if err := printAutoTable(cmd.OutOrStdout(), items); err != nil { + return err + } + if len(items) >= 25 { + fmt.Fprintf(os.Stderr, "\nShowing %d results. To narrow: add --limit, --json --select, or filter flags.\n", len(items)) + } + return nil + } + } + return printOutputWithFlags(cmd.OutOrStdout(), data, flags) + }, + } + cmd.Flags().IntVar(&flagLimit, "limit", 20, "Maximum related records") + + return cmd +} diff --git a/library/productivity/devonthink/internal/cli/records_search.go b/library/productivity/devonthink/internal/cli/records_search.go new file mode 100644 index 0000000000..9d9a0aaf2e --- /dev/null +++ b/library/productivity/devonthink/internal/cli/records_search.go @@ -0,0 +1,120 @@ +// Copyright 2026 rowdy and contributors. Licensed under Apache-2.0. See LICENSE. +// Generated by CLI Printing Press (https://github.com/mvanhorn/cli-printing-press). DO NOT EDIT. + +package cli + +import ( + "encoding/json" + "fmt" + "os" + + "github.com/spf13/cobra" +) + +func newRecordsSearchCmd(flags *rootFlags) *cobra.Command { + var flagDatabase string + var flagGroup string + var flagSmartGroup string + var flagLimit int + + cmd := &cobra.Command{ + Use: "search <query>", + Short: "Search records using DEVONthink query syntax or local mirror fallback", + Example: " devonthink-pp-cli records search \"kind:pdf\" --limit 5 --agent --select uuid,name,item_link", + Annotations: map[string]string{"pp:endpoint": "records.search", "pp:method": "GET", "pp:path": "/records/search", "mcp:read-only": "true"}, + RunE: func(cmd *cobra.Command, args []string) error { + if len(args) == 0 { + return cmd.Help() + } + c, err := flags.newClient() + if err != nil { + return err + } + + path := "/records/search" + params := map[string]string{} + params["query"] = args[0] + if flagDatabase != "" { + params["database"] = formatCLIParamValue(flagDatabase) + } + var scope any + if flagGroup != "" && flagSmartGroup != "" { + return usageErr(fmt.Errorf("--group and --smart-group are mutually exclusive")) + } + if flagSmartGroup != "" { + if flags.dataSource == "local" { + return usageErr(fmt.Errorf("--smart-group requires live DEVONthink MCP search; Smart Groups are dynamic scopes and cannot be resolved from --data-source local")) + } + resolved, err := c.ResolveSmartGroupScope(cmd.Context(), flagSmartGroup) + if err != nil { + return err + } + params["group"] = resolved.UUID + scope = resolved + } + if flagGroup != "" { + params["group"] = formatCLIParamValue(flagGroup) + } + if flagLimit != 0 { + params["limit"] = formatCLIParamValue(flagLimit) + } + data, prov, err := resolveReadWithStrategy(cmd.Context(), c, flags, "auto", "records", false, path, params, nil, cmd.ErrOrStderr()) + if err != nil { + return classifyAPIError(err, flags) + } + if scope != nil { + prov.Scope = scope + } + // Honor --limit when the API accepts but ignores ?limit=N. + data = truncateJSONArray(data, flagLimit) + // Print provenance to stderr for human-facing output only. + // Machine-format flags (--json, --csv, --compact, --quiet, --plain, + // --select) and piped stdout suppress this line; the JSON envelope + // already carries meta.source for those consumers. + // SYNC: keep this gate aligned with command_promoted.go.tmpl. + if wantsHumanTable(cmd.OutOrStdout(), flags) { + var countItems []json.RawMessage + _ = json.Unmarshal(data, &countItems) + printProvenance(cmd, len(countItems), prov) + } + // For JSON output, wrap with provenance envelope before passing through flags. + // --select wins over --compact when both are set; --compact only runs when + // no explicit fields were requested. Explicit format flags (--csv, --quiet, + // --plain) opt out of the auto-JSON path so piped consumers that asked for + // a non-JSON format reach the standard pipeline below. + if flags.asJSON || (!isTerminal(cmd.OutOrStdout()) && !flags.csv && !flags.quiet && !flags.plain) { + filtered := data + if flags.selectFields != "" { + filtered = filterFields(filtered, flags.selectFields) + } else if flags.compact { + filtered = compactFields(filtered) + } + wrapped, wrapErr := wrapWithProvenance(filtered, prov) + if wrapErr != nil { + return wrapErr + } + return printOutput(cmd.OutOrStdout(), wrapped, true) + } + // For all other output modes (table, csv, plain, quiet), use the standard pipeline + if wantsHumanTable(cmd.OutOrStdout(), flags) { + var items []map[string]any + if json.Unmarshal(data, &items) == nil && len(items) > 0 { + if err := printAutoTable(cmd.OutOrStdout(), items); err != nil { + return err + } + if len(items) >= 25 { + fmt.Fprintf(os.Stderr, "\nShowing %d results. To narrow: add --limit, --json --select, or filter flags.\n", len(items)) + } + return nil + } + } + return printOutputWithFlags(cmd.OutOrStdout(), data, flags) + }, + } + cmd.Flags().StringVar(&flagDatabase, "database", "", "Database name or UUID") + cmd.Flags().StringVar(&flagGroup, "group", "", "Group UUID or path") + cmd.Flags().StringVar(&flagSmartGroup, "smart-group", "", "Smart Group UUID, exact name, or DEVONthink path used as a read-only search scope") + cmd.Flags().IntVar(&flagLimit, "limit", 20, "Maximum records to return") + + return cmd +} diff --git a/library/productivity/devonthink/internal/cli/records_update.go b/library/productivity/devonthink/internal/cli/records_update.go new file mode 100644 index 0000000000..171160a097 --- /dev/null +++ b/library/productivity/devonthink/internal/cli/records_update.go @@ -0,0 +1,220 @@ +// Copyright 2026 rowdy and contributors. Licensed under Apache-2.0. See LICENSE. +// Generated by CLI Printing Press (https://github.com/mvanhorn/cli-printing-press). DO NOT EDIT. + +package cli + +import ( + "encoding/json" + "fmt" + "io" + "os" + + "github.com/spf13/cobra" +) + +func newRecordsUpdateCmd(flags *rootFlags) *cobra.Command { + var bodyTitle string + var bodyTags string + var bodyAddTag string + var bodyComment string + var bodyAppend string + var bodyDryRun bool + var stdinBody bool + + cmd := &cobra.Command{ + Use: "update <uuid>", + Short: "Update record text, properties, tags, comment, URL, aliases, or rating", + Example: " devonthink-pp-cli records update 550e8400-e29b-41d4-a716-446655440000", + Annotations: map[string]string{"pp:endpoint": "records.update", "pp:method": "PATCH", "pp:path": "/records/{uuid}"}, + RunE: func(cmd *cobra.Command, args []string) error { + if len(args) == 0 { + return cmd.Help() + } + if !stdinBody { + } + c, err := flags.newClient() + if err != nil { + return err + } + + path := "/records/{uuid}" + path = replacePathParam(path, "uuid", args[0]) + params := map[string]string{} + var body map[string]any + if stdinBody { + stdinData, err := io.ReadAll(os.Stdin) + if err != nil { + return fmt.Errorf("reading stdin: %w", err) + } + var jsonBody map[string]any + if err := json.Unmarshal(stdinData, &jsonBody); err != nil { + return fmt.Errorf("parsing stdin JSON: %w", err) + } + body = jsonBody + } else { + body = map[string]any{} + if bodyTitle != "" { + body["title"] = bodyTitle + } + if bodyTags != "" { + body["tags"] = bodyTags + } + if bodyAddTag != "" { + body["add_tag"] = bodyAddTag + } + if bodyComment != "" { + body["comment"] = bodyComment + } + if bodyAppend != "" { + body["append"] = bodyAppend + } + if cmd.Flags().Changed("dry-run") { + body["dry_run"] = bodyDryRun + } + } + data, statusCode, err := c.PatchWithParams(cmd.Context(), path, params, body) + if err != nil { + return classifyAPIError(err, flags) + } + // Inspect the mutate response body for a partial-failure-shaped + // field (e.g. Google Ads `partialFailureError`). Several Google + // APIs return 200 OK with a partial-failure field when some + // operations in the batch failed; ignoring it silently swallows + // real failures. Detection runs before output-mode selection so + // the exit code is consistent regardless of how stdout is + // rendered. --dry-run short-circuits because no real request + // was sent. + var partialFailure *partialFailureReport + if !flags.dryRun && statusCode >= 200 && statusCode < 300 { + partialFailure = detectPartialFailure(data) + if partialFailure != nil { + fmt.Fprintf(os.Stderr, "warning: partial failure detected in %s response: %s\n", "records", partialFailure.Message) + if len(partialFailure.ResourceNames) > 0 { + fmt.Fprintf(os.Stderr, " succeeded: %d operation(s)\n", len(partialFailure.ResourceNames)) + } + } + } + if !flags.dryRun && statusCode >= 200 && statusCode < 300 && (partialFailure == nil || flags.allowPartialFailure) { + writeMutationResponseToStore(cmd.Context(), "records", data, "") + } + if wantsHumanTable(cmd.OutOrStdout(), flags) { + // Check if response contains an array (directly or wrapped in "data") + var items []map[string]any + if json.Unmarshal(data, &items) == nil && len(items) > 0 { + if err := printAutoTable(cmd.OutOrStdout(), items); err != nil { + fmt.Fprintf(os.Stderr, "warning: table rendering failed, falling back to JSON: %v\n", err) + } else { + if partialFailure != nil && !flags.allowPartialFailure { + return partialFailureErr(fmt.Errorf("partial failure in %s response: %s", "records", partialFailure.Message)) + } + return nil + } + } else { + var wrapped struct { + Data []map[string]any `json:"data"` + } + if json.Unmarshal(data, &wrapped) == nil && len(wrapped.Data) > 0 { + if err := printAutoTable(cmd.OutOrStdout(), wrapped.Data); err != nil { + fmt.Fprintf(os.Stderr, "warning: table rendering failed, falling back to JSON: %v\n", err) + } else { + if partialFailure != nil && !flags.allowPartialFailure { + return partialFailureErr(fmt.Errorf("partial failure in %s response: %s", "records", partialFailure.Message)) + } + return nil + } + } + } + } + if flags.asJSON || (!isTerminal(cmd.OutOrStdout()) && !flags.csv && !flags.quiet && !flags.plain) { + if flags.quiet { + if partialFailure != nil && !flags.allowPartialFailure { + return partialFailureErr(fmt.Errorf("partial failure in %s response: %s", "records", partialFailure.Message)) + } + return nil + } + envelope := map[string]any{ + "action": "patch", + "resource": "records", + "path": path, + "status": statusCode, + "success": statusCode >= 200 && statusCode < 300 && (partialFailure == nil || flags.allowPartialFailure), + } + if partialFailure != nil { + envelope["partial_failure"] = partialFailure + } + if flags.dryRun { + envelope["dry_run"] = true + envelope["status"] = 0 + envelope["success"] = false + } + // Verify-mode synthetic envelope detection runs against RAW data + // (before --compact/--select filtering) so the sentinel field is + // guaranteed to be visible even if the operator passes a filter + // flag that would otherwise strip it. Surfaces a top-level + // verify_noop signal + flips success to false. Mirrors the dry_run + // shape above. + if len(data) > 0 { + var rawParsed any + if err := json.Unmarshal(data, &rawParsed); err == nil { + if m, ok := rawParsed.(map[string]any); ok { + if v, ok := m["__pp_verify_synthetic__"].(bool); ok && v { + envelope["verify_noop"] = true + envelope["success"] = false + } + } + } + } + // Apply --compact and --select to the API response before wrapping. + // --select wins when both are set: explicit field choice trumps the + // generic high-gravity allow-list. Otherwise --compact still applies + // when --agent is on but the user did not name fields. + filtered := data + if flags.selectFields != "" { + filtered = filterFields(filtered, flags.selectFields) + } else if flags.compact { + filtered = compactFields(filtered) + } + if len(filtered) > 0 { + var parsed any + if err := json.Unmarshal(filtered, &parsed); err == nil { + envelope["data"] = parsed + } + } + envelopeJSON, err := json.Marshal(envelope) + if err != nil { + return err + } + if perr := printOutput(cmd.OutOrStdout(), json.RawMessage(envelopeJSON), true); perr != nil { + return perr + } + if partialFailure != nil && !flags.allowPartialFailure { + return partialFailureErr(fmt.Errorf("partial failure in %s response: %s", "records", partialFailure.Message)) + } + return nil + } + // Fall-through for mutate paths that did not hit the table or + // asJSON branches: --quiet, --csv, --plain, and default terminal + // raw output. printOutputWithFlags renders the body, then the + // typed partial-failure exit fires unless --allow-partial-failure + // downgrades it. Without this guard a partial failure would exit + // 0 for these output modes — the exact silent-swallow regression + // the surrounding patch is preventing for asJSON / piped output. + if perr := printOutputWithFlags(cmd.OutOrStdout(), data, flags); perr != nil { + return perr + } + if partialFailure != nil && !flags.allowPartialFailure { + return partialFailureErr(fmt.Errorf("partial failure in %s response: %s", "records", partialFailure.Message)) + } + return nil + }, + } + cmd.Flags().StringVar(&bodyTitle, "title", "", "New title") + cmd.Flags().StringVar(&bodyTags, "tags", "", "Replacement comma-separated tags") + cmd.Flags().StringVar(&bodyAddTag, "add-tag", "", "Tag to add") + cmd.Flags().StringVar(&bodyComment, "comment", "", "New comment") + cmd.Flags().StringVar(&bodyAppend, "append", "", "Text to append") + cmd.Flags().BoolVar(&bodyDryRun, "dry-run", false, "Show the planned update without writing") + cmd.Flags().BoolVar(&stdinBody, "stdin", false, "Read request body as JSON from stdin") + + return cmd +} diff --git a/library/productivity/devonthink/internal/cli/records_versions.go b/library/productivity/devonthink/internal/cli/records_versions.go new file mode 100644 index 0000000000..8b2d75fe92 --- /dev/null +++ b/library/productivity/devonthink/internal/cli/records_versions.go @@ -0,0 +1,83 @@ +// Copyright 2026 rowdy and contributors. Licensed under Apache-2.0. See LICENSE. +// Generated by CLI Printing Press (https://github.com/mvanhorn/cli-printing-press). DO NOT EDIT. + +package cli + +import ( + "encoding/json" + "fmt" + "os" + + "github.com/spf13/cobra" +) + +func newRecordsVersionsCmd(flags *rootFlags) *cobra.Command { + + cmd := &cobra.Command{ + Use: "versions <uuid>", + Short: "List saved record versions", + Example: " devonthink-pp-cli records versions 550e8400-e29b-41d4-a716-446655440000", + Annotations: map[string]string{"pp:endpoint": "records.versions", "pp:method": "GET", "pp:path": "/records/{uuid}/versions", "mcp:read-only": "true"}, + RunE: func(cmd *cobra.Command, args []string) error { + if len(args) == 0 { + return cmd.Help() + } + c, err := flags.newClient() + if err != nil { + return err + } + + path := "/records/{uuid}/versions" + path = replacePathParam(path, "uuid", args[0]) + params := map[string]string{} + data, prov, err := resolveReadWithStrategy(cmd.Context(), c, flags, "auto", "records", false, path, params, nil, cmd.ErrOrStderr()) + if err != nil { + return classifyAPIError(err, flags) + } + // Print provenance to stderr for human-facing output only. + // Machine-format flags (--json, --csv, --compact, --quiet, --plain, + // --select) and piped stdout suppress this line; the JSON envelope + // already carries meta.source for those consumers. + // SYNC: keep this gate aligned with command_promoted.go.tmpl. + if wantsHumanTable(cmd.OutOrStdout(), flags) { + var countItems []json.RawMessage + _ = json.Unmarshal(data, &countItems) + printProvenance(cmd, len(countItems), prov) + } + // For JSON output, wrap with provenance envelope before passing through flags. + // --select wins over --compact when both are set; --compact only runs when + // no explicit fields were requested. Explicit format flags (--csv, --quiet, + // --plain) opt out of the auto-JSON path so piped consumers that asked for + // a non-JSON format reach the standard pipeline below. + if flags.asJSON || (!isTerminal(cmd.OutOrStdout()) && !flags.csv && !flags.quiet && !flags.plain) { + filtered := data + if flags.selectFields != "" { + filtered = filterFields(filtered, flags.selectFields) + } else if flags.compact { + filtered = compactFields(filtered) + } + wrapped, wrapErr := wrapWithProvenance(filtered, prov) + if wrapErr != nil { + return wrapErr + } + return printOutput(cmd.OutOrStdout(), wrapped, true) + } + // For all other output modes (table, csv, plain, quiet), use the standard pipeline + if wantsHumanTable(cmd.OutOrStdout(), flags) { + var items []map[string]any + if json.Unmarshal(data, &items) == nil && len(items) > 0 { + if err := printAutoTable(cmd.OutOrStdout(), items); err != nil { + return err + } + if len(items) >= 25 { + fmt.Fprintf(os.Stderr, "\nShowing %d results. To narrow: add --limit, --json --select, or filter flags.\n", len(items)) + } + return nil + } + } + return printOutputWithFlags(cmd.OutOrStdout(), data, flags) + }, + } + + return cmd +} diff --git a/library/productivity/devonthink/internal/cli/root.go b/library/productivity/devonthink/internal/cli/root.go new file mode 100644 index 0000000000..551b8bd31c --- /dev/null +++ b/library/productivity/devonthink/internal/cli/root.go @@ -0,0 +1,327 @@ +// Copyright 2026 rowdy and contributors. Licensed under Apache-2.0. See LICENSE. +// Generated by CLI Printing Press (https://github.com/mvanhorn/cli-printing-press). DO NOT EDIT. + +package cli + +import ( + "bytes" + "fmt" + "io" + "os" + "strings" + "text/tabwriter" + "time" + + "github.com/mvanhorn/printing-press-library/library/productivity/devonthink/internal/client" + "github.com/mvanhorn/printing-press-library/library/productivity/devonthink/internal/config" + "github.com/spf13/cobra" +) + +type rootFlags struct { + asJSON bool + compact bool + csv bool + plain bool + quiet bool + dryRun bool + noCache bool + noInput bool + idempotent bool + yes bool + agent bool + // allowPartialFailure downgrades a detected response-body partial-failure + // (e.g. Google Ads `partialFailureError`) from a non-zero exit to a + // stderr warning. Default false so silent partial successes surface as + // failures by default. + allowPartialFailure bool + selectFields string + configPath string + profileName string + deliverSpec string + timeout time.Duration + rateLimit float64 + maxAge time.Duration + dataSource string + freshnessMeta any + + // deliverBuf captures command output when --deliver is set to a + // non-stdout sink. Flushed to the sink after Execute returns. + deliverBuf *bytes.Buffer + deliverSink DeliverSink +} + +// RootCmd returns the Cobra command tree without executing it. The MCP server +// uses this to mirror every user-facing command as an agent tool. +func RootCmd() *cobra.Command { + var flags rootFlags + return newRootCmd(&flags) +} + +// Execute runs the CLI in non-interactive mode: never prompts, all values via flags or stdin. +func Execute() error { + var flags rootFlags + rootCmd := newRootCmd(&flags) + + err := rootCmd.Execute() + if err != nil && strings.Contains(err.Error(), "unknown flag") { + msg := err.Error() + // Extract the flag name from the error message (e.g., "unknown flag: --foob") + if idx := strings.Index(msg, "unknown flag: "); idx >= 0 { + flagStr := strings.TrimSpace(msg[idx+len("unknown flag: "):]) + if suggestion := suggestFlag(flagStr, rootCmd); suggestion != "" { + // Cobra already printed `Error: unknown flag: --foob` before + // returning; the wrap below attaches the hint to err.Error() + // for downstream consumers and exit-code classification, but + // would never reach stderr now that main.go no longer prints + // err. Emit the hint explicitly so the suggestion still + // shows up under Cobra's error line. + fmt.Fprintf(os.Stderr, "hint: did you mean --%s?\n", suggestion) + err = fmt.Errorf("%w\nhint: did you mean --%s?", err, suggestion) + } + } + } + if err == nil && flags.deliverBuf != nil { + if derr := Deliver(flags.deliverSink, flags.deliverBuf.Bytes(), flags.compact); derr != nil { + fmt.Fprintf(os.Stderr, "warning: deliver to %s:%s failed: %v\n", flags.deliverSink.Scheme, flags.deliverSink.Target, derr) + return derr + } + } + if err != nil && isCobraUsageError(err) { + // Cobra/pflag pre-RunE errors (unknown flag, unknown command, + // missing required, etc.) never flow through usageErr() because + // they originate inside rootCmd.Execute() before any user RunE + // runs. Without this wrap, ExitCode() falls through to the + // default and emits 1 — clobbering the conventional code-2 for + // usage errors that the helpers.go contract already promises. + return usageErr(err) + } + return err +} + +// isCobraUsageError reports whether err matches one of Cobra/pflag's +// pre-RunE usage-error shapes. Detection is by message prefix to match +// the same approach the unknown-flag hint path uses above; neither +// Cobra nor pflag exports typed sentinels for these. +// +// Patterns are anchored to the literal punctuation Cobra and pflag +// emit so an application's own RunE error that happens to contain the +// substring "required flag" or "invalid argument" doesn't get +// misclassified as a usage error. +// +// Patterns covered (Cobra v1.x + pflag v1.x as of 2026-05): +// - "unknown flag: --foo" (pflag) +// - "unknown shorthand flag: 'x' in -x" (pflag) +// - "unknown command \"foo\" for ..." (Cobra) +// - "required flag \"foo\" not set" (Cobra, single missing) +// - "required flag(s) \"foo\" not set" (Cobra, multiple missing) +// - "flag needs an argument: --foo" (pflag, missing value) +// - "invalid argument \"x\" for \"--y\" flag: ..." (pflag, parse failure) +// +// Cobra emits the singular form ("required flag") when exactly one +// MarkFlagRequired flag is missing, and the plural form ("required +// flag(s)") only when multiple are missing on the same command. Both +// shapes must be anchored to avoid matching app-level errors that +// happen to mention "required flag" as prose; the trailing space + quote +// (`required flag "`) is the literal punctuation cobra emits. +// +// Returns false for nil err. +func isCobraUsageError(err error) bool { + if err == nil { + return false + } + msg := err.Error() + return strings.HasPrefix(msg, "unknown flag") || + strings.HasPrefix(msg, "unknown shorthand flag") || + strings.HasPrefix(msg, "unknown command") || + strings.HasPrefix(msg, `required flag "`) || + strings.HasPrefix(msg, `required flag(s) "`) || + strings.HasPrefix(msg, "flag needs an argument:") || + strings.HasPrefix(msg, `invalid argument "`) +} + +func newRootCmd(flags *rootFlags) *cobra.Command { + rootCmd := &cobra.Command{ + Use: "devonthink-pp-cli", + Short: `Devonthink CLI — Local-first DEVONthink automation with safer shell workflows than raw AppleScript or MCP alone.`, + Long: `Devonthink CLI — Local-first DEVONthink automation with safer shell workflows than raw AppleScript or MCP alone. + +Highlights (not in the official API docs): + • context pack Build a compact evidence packet from records, selections, highlights, links, and related items. + • privacy audit Preview what a workflow may expose before content leaves the local machine. + • batch plan Stage multi-record edits as validated dry-run plans before applying them. + • inventory export Export DEVONthink databases, groups, tags, and document metadata for maintenance plugins. + • graph audit Detect orphans, broken links, unresolved wiki links, weak hubs, and tag-only clusters. + • mirror search Query a local SQLite mirror for repeatable fast analysis without repeated app calls. + • mcp call Call DEVONthink's official local MCP tools from scripts when the local MCP server is enabled. + • ledger list Review CLI-driven mutation plans, applies, target proofs, and rollback hints. + • selection snapshot Turn the current GUI selection into a reusable JSON workflow seed. + • agent-context Emit an agent contract that enforces local-machine and own-LAN DEVONthink access only. + +Agent mode: add --agent to any command for JSON output + non-interactive mode. +Health check: run 'devonthink-pp-cli doctor' to verify auth and connectivity. +See README.md or the bundled SKILL.md for recipes.`, + SilenceUsage: true, + Version: version, + } + rootCmd.SetVersionTemplate("devonthink-pp-cli {{ .Version }}\n") + + rootCmd.PersistentFlags().BoolVar(&flags.asJSON, "json", false, "Output as JSON") + rootCmd.PersistentFlags().BoolVar(&flags.compact, "compact", false, "Return only key fields (id, name, status, timestamps) for minimal token usage") + rootCmd.PersistentFlags().BoolVar(&flags.csv, "csv", false, "Output as CSV (table and array responses)") + rootCmd.PersistentFlags().BoolVar(&flags.plain, "plain", false, "Output as plain tab-separated text") + rootCmd.PersistentFlags().BoolVar(&flags.quiet, "quiet", false, "Bare output, one value per line") + rootCmd.PersistentFlags().StringVar(&flags.configPath, "config", "", "Config file path") + rootCmd.PersistentFlags().DurationVar(&flags.timeout, "timeout", 60*time.Second, "Request timeout") + rootCmd.PersistentFlags().BoolVar(&flags.dryRun, "dry-run", false, "Show request without sending") + rootCmd.PersistentFlags().BoolVar(&flags.noCache, "no-cache", false, "Bypass response cache") + rootCmd.PersistentFlags().BoolVar(&flags.noInput, "no-input", false, "Disable all interactive prompts (for CI/agents)") + rootCmd.PersistentFlags().BoolVar(&flags.idempotent, "idempotent", false, "Treat already-existing create results as a successful no-op") + rootCmd.PersistentFlags().StringVar(&flags.selectFields, "select", "", "Comma-separated fields to include in output (e.g. --select id,name,status)") + rootCmd.PersistentFlags().BoolVar(&flags.yes, "yes", false, "Skip confirmation prompts (for agents and scripts)") + rootCmd.PersistentFlags().BoolVar(&noColor, "no-color", false, "Disable colored output") + rootCmd.PersistentFlags().BoolVar(&humanFriendly, "human-friendly", false, "Enable colored output and rich formatting") + rootCmd.PersistentFlags().BoolVar(&flags.agent, "agent", false, "Set all agent-friendly defaults (--json --compact --no-input --no-color --yes)") + rootCmd.PersistentFlags().BoolVar(&flags.allowPartialFailure, "allow-partial-failure", false, "Downgrade response-body partial-failure (e.g. partialFailureError) to a warning instead of a non-zero exit") + rootCmd.PersistentFlags().StringVar(&flags.dataSource, "data-source", "auto", "Data source for read commands: auto (live with local fallback), live (API only), local (synced data only)") + rootCmd.PersistentFlags().DurationVar(&flags.maxAge, "max-age", 30*time.Minute, "Maximum acceptable age of local-store data before a stderr hint suggests sync; 0 disables") + rootCmd.PersistentFlags().StringVar(&flags.profileName, "profile", "", "Apply values from a saved profile (see 'devonthink-pp-cli profile list')") + rootCmd.PersistentFlags().StringVar(&flags.deliverSpec, "deliver", "", "Route output to a sink: stdout (default), file:<path>, webhook:<url>") + rootCmd.PersistentFlags().Float64Var(&flags.rateLimit, "rate-limit", 0, "Max requests per second (0 to disable)") + + rootCmd.PersistentPreRunE = func(cmd *cobra.Command, args []string) error { + if flags.deliverSpec != "" { + sink, err := ParseDeliverSink(flags.deliverSpec) + if err != nil { + return err + } + flags.deliverSink = sink + if sink.Scheme != "stdout" && sink.Scheme != "" { + flags.deliverBuf = &bytes.Buffer{} + cmd.SetOut(io.MultiWriter(os.Stdout, flags.deliverBuf)) + } + } + if flags.profileName != "" { + profile, err := GetProfile(flags.profileName) + if err != nil { + return err + } + if profile == nil { + available := ListProfileNames() + if len(available) == 0 { + return fmt.Errorf("profile %q not found (no profiles saved yet; run '%s profile save <name> --<flag> <value>')", flags.profileName, cmd.Root().Name()) + } + return fmt.Errorf("profile %q not found; available: %s", flags.profileName, strings.Join(available, ", ")) + } + if err := ApplyProfileToFlags(cmd, profile); err != nil { + return err + } + } + if flags.agent { + if !cmd.Flags().Changed("json") { + flags.asJSON = true + } + if !cmd.Flags().Changed("compact") { + flags.compact = true + } + if !cmd.Flags().Changed("no-input") { + flags.noInput = true + } + if !cmd.Flags().Changed("yes") { + flags.yes = true + } + if !cmd.Flags().Changed("no-color") { + noColor = true + } + } + switch flags.dataSource { + case "auto", "live", "local": + // valid + default: + return fmt.Errorf("invalid --data-source value %q: must be auto, live, or local", flags.dataSource) + } + return nil + } + rootCmd.AddCommand(newAiCmd(flags)) + rootCmd.AddCommand(newBatchCmd(flags)) + rootCmd.AddCommand(newGraphCmd(flags)) + rootCmd.AddCommand(newIngestCmd(flags)) + rootCmd.AddCommand(newLedgerCmd(flags)) + rootCmd.AddCommand(newMcpCmd(flags)) + rootCmd.AddCommand(newMediaCmd(flags)) + rootCmd.AddCommand(newMirrorCmd(flags)) + rootCmd.AddCommand(newRecordsCmd(flags)) + rootCmd.AddCommand(newSelectionCmd(flags)) + rootCmd.AddCommand(newDoctorCmd(flags)) + rootCmd.AddCommand(newAgentContextCmd(rootCmd)) + rootCmd.AddCommand(newProfileCmd(flags)) + rootCmd.AddCommand(newFeedbackCmd(flags)) + rootCmd.AddCommand(newWhichCmd(flags)) + rootCmd.AddCommand(newExportCmd(flags)) + rootCmd.AddCommand(newImportCmd(flags)) + rootCmd.AddCommand(newSearchCmd(flags)) + rootCmd.AddCommand(newSyncCmd(flags)) + rootCmd.AddCommand(newTailCmd(flags)) + rootCmd.AddCommand(newAnalyticsCmd(flags)) + rootCmd.AddCommand(newWorkflowCmd(flags)) + rootCmd.AddCommand(newAPICmd(flags)) + rootCmd.AddCommand(newContextPromotedCmd(flags)) + rootCmd.AddCommand(newDatabasesPromotedCmd(flags)) + rootCmd.AddCommand(newGroupsPromotedCmd(flags)) + rootCmd.AddCommand(newInventoryPromotedCmd(flags)) + rootCmd.AddCommand(newPrivacyPromotedCmd(flags)) + rootCmd.AddCommand(newRuntimePromotedCmd(flags)) + rootCmd.AddCommand(newSheetsPromotedCmd(flags)) + rootCmd.AddCommand(newTagsPromotedCmd(flags)) + rootCmd.AddCommand(newVersionCmd()) + + return rootCmd +} + +func ExitCode(err error) int { + var codeErr *cliError + if As(err, &codeErr) { + return codeErr.code + } + return 1 +} + +func (f *rootFlags) newClient() (*client.Client, error) { + cfg, err := config.Load(f.configPath) + if err != nil { + return nil, configErr(err) + } + c := client.New(cfg, f.timeout, f.rateLimit) + c.DryRun = f.dryRun + c.NoCache = f.noCache + return c, nil +} + +func (f *rootFlags) printJSON(w *cobra.Command, v any) error { + return printJSONFiltered(w.OutOrStdout(), v, f) +} + +func (f *rootFlags) printTable(w *cobra.Command, headers []string, rows [][]string) error { + if f.asJSON { + return fmt.Errorf("use printJSON for JSON output") + } + tw := tabwriter.NewWriter(w.OutOrStdout(), 2, 4, 2, ' ', 0) + header := "" + for i, h := range headers { + if i > 0 { + header += "\t" + } + header += h + } + fmt.Fprintln(tw, header) + for _, row := range rows { + line := "" + for i, cell := range row { + if i > 0 { + line += "\t" + } + line += cell + } + fmt.Fprintln(tw, line) + } + return tw.Flush() +} diff --git a/library/productivity/devonthink/internal/cli/root_test.go b/library/productivity/devonthink/internal/cli/root_test.go new file mode 100644 index 0000000000..2435029836 --- /dev/null +++ b/library/productivity/devonthink/internal/cli/root_test.go @@ -0,0 +1,244 @@ +// Copyright 2026 rowdy and contributors. Licensed under Apache-2.0. See LICENSE. +// Generated by CLI Printing Press (https://github.com/mvanhorn/cli-printing-press). DO NOT EDIT. + +package cli + +import ( + "encoding/json" + "errors" + "fmt" + "testing" +) + +// TestIsCobraUsageError covers the six pre-RunE error shapes Cobra and +// pflag can produce before any user RunE runs. Each must be detected so +// the caller can wrap in usageErr() and yield exit code 2. +func TestIsCobraUsageError(t *testing.T) { + t.Parallel() + cases := []struct { + name string + errIn error + want bool + }{ + {"nil", nil, false}, + {"unknown flag", errors.New("unknown flag: --foob"), true}, + {"unknown shorthand flag", errors.New("unknown shorthand flag: 'x' in -x"), true}, + {"unknown command", errors.New("unknown command \"foo\" for \"cli\""), true}, + {"required flag (singular)", errors.New("required flag \"query\" not set"), true}, + {"required flag(s) (plural)", errors.New("required flag(s) \"query\", \"vault\" not set"), true}, + {"flag needs argument", errors.New("flag needs an argument: --query"), true}, + {"invalid argument", errors.New("invalid argument \"abc\" for \"--limit\" flag: strconv.ParseInt: parsing \"abc\": invalid syntax"), true}, + // Non-usage errors must NOT be flagged — they should retain their + // original (or wrapped *cliError) exit code. + {"API error pass-through", errors.New("API returned 500"), false}, + {"network error pass-through", errors.New("dial tcp: connection refused"), false}, + // Pattern-tightness regressions: application RunE errors that + // happen to mention "required flag" / "flag needs an argument" / + // "invalid argument" as prose must NOT classify as usage errors. + // The anchored prefixes guard against this; if the patterns are + // ever loosened back to Contains() these tests catch the drift. + {"app msg containing 'required flag' as prose", errors.New("the required flag config is missing from the manifest"), false}, + {"app msg containing 'flag needs an argument' as prose", errors.New("this flag needs an argument upstream before retry"), false}, + {"app msg containing 'invalid argument' as prose", errors.New("invalid argument provided to handler at /foo"), false}, + } + for _, tc := range cases { + tc := tc + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + if got := isCobraUsageError(tc.errIn); got != tc.want { + t.Errorf("isCobraUsageError(%v) = %v, want %v", tc.errIn, got, tc.want) + } + }) + } +} + +// TestIsCobraUsageError_SurvivesHintRewrap covers the hint-suggestion +// path in Execute(): after rewrapping an "unknown flag" error with +// fmt.Errorf("%w\nhint: ...", err, ...), the rewrapped error must still +// classify as a usage error so the exit code stays at 2 rather than +// silently falling through to 1. +func TestIsCobraUsageError_SurvivesHintRewrap(t *testing.T) { + t.Parallel() + original := errors.New("unknown flag: --foob") + wrapped := fmt.Errorf("%w\nhint: did you mean --foo?", original) + if !isCobraUsageError(wrapped) { + t.Errorf("hint-rewrapped unknown-flag error must still classify as usage error, got: %v", wrapped) + } +} + +// TestExitCode_UsageError_WrappedAsCode2 covers the end-to-end contract: +// after Execute() wraps a Cobra usage error in usageErr(), ExitCode() +// must return 2, not 1. This is the user-visible promise — a bad flag +// exits 2 (POSIX usage convention), not the generic-error-1 fallback. +func TestExitCode_UsageError_WrappedAsCode2(t *testing.T) { + t.Parallel() + wrapped := usageErr(errors.New("unknown flag: --foob")) + if got := ExitCode(wrapped); got != 2 { + t.Errorf("ExitCode(usageErr(...)) = %d, want 2 (POSIX usage convention)", got) + } +} + +func TestWrapWithProvenanceIncludesScope(t *testing.T) { + t.Parallel() + got, err := wrapWithProvenance(json.RawMessage(`[{"uuid":"rec-1","name":"Invoice"}]`), DataProvenance{ + Source: "live", + Scope: map[string]any{ + "type": "smart_group", + "input": "Offene Rückerstattungen", + "uuid": "sg-1", + }, + }) + if err != nil { + t.Fatal(err) + } + var envelope struct { + Meta struct { + Source string `json:"source"` + Scope struct { + Type string `json:"type"` + Input string `json:"input"` + UUID string `json:"uuid"` + } `json:"scope"` + } `json:"meta"` + Results []map[string]any `json:"results"` + } + if err := json.Unmarshal(got, &envelope); err != nil { + t.Fatalf("unmarshal: %v\n%s", err, got) + } + if envelope.Meta.Source != "live" { + t.Fatalf("source = %q, want live", envelope.Meta.Source) + } + if envelope.Meta.Scope.Type != "smart_group" || envelope.Meta.Scope.Input != "Offene Rückerstattungen" || envelope.Meta.Scope.UUID != "sg-1" { + t.Fatalf("scope = %+v", envelope.Meta.Scope) + } + if len(envelope.Results) != 1 || envelope.Results[0]["uuid"] != "rec-1" { + t.Fatalf("results = %#v", envelope.Results) + } +} + +// TestFilterFields covers --select projection against the four payload +// shapes printed CLIs see in practice: bare arrays, direct objects, +// list envelopes (Stripe/GitHub/Notion-style wrapper + array), and +// flat objects. The envelope cases guard against a regression where +// wrapper-key + array responses returned `{}` because the selector +// heads matched the inner record fields, not the wrapper key. +func TestFilterFields(t *testing.T) { + t.Parallel() + cases := []struct { + name string + input string + fields string + want string + }{ + { + name: "bare array element-wise", + input: `[{"id":"a","name":"x","other":"y"},{"id":"b","name":"z","other":"w"}]`, + fields: "id,name", + want: `[{"id":"a","name":"x"},{"id":"b","name":"z"}]`, + }, + { + name: "direct object top-level match", + input: `{"id":"a","name":"b","other":"c"}`, + fields: "id,name", + want: `{"id":"a","name":"b"}`, + }, + { + name: "envelope single array sibling (orgo {projects:[...]})", + input: `{"projects":[{"id":"a","name":"x","other":"y"}]}`, + fields: "id,name", + want: `{"projects":[{"id":"a","name":"x"}]}`, + }, + { + name: "envelope with metadata sibling (github {total_count,items:[...]})", + input: `{"total_count":2,"items":[{"id":"a","name":"x","other":"y"},{"id":"b","name":"z","other":"w"}]}`, + fields: "id,name", + want: `{"items":[{"id":"a","name":"x"},{"id":"b","name":"z"}],"total_count":2}`, + }, + { + name: "envelope with metadata sibling (stripe {object,data:[...]})", + input: `{"object":"list","data":[{"id":"a","name":"x","other":"y"}]}`, + fields: "id,name", + want: `{"data":[{"id":"a","name":"x"}],"object":"list"}`, + }, + { + name: "selector matches envelope key (no descent into array)", + input: `{"projects":[{"id":"a","name":"x","other":"y"}]}`, + fields: "projects", + want: `{"projects":[{"id":"a","name":"x","other":"y"}]}`, + }, + { + name: "dotted-path through envelope key still descends", + input: `{"projects":[{"id":"a","name":"x","other":"y"}]}`, + fields: "projects.id", + want: `{"projects":[{"id":"a"}]}`, + }, + { + name: "flat object no match returns empty (no array fallback)", + input: `{"a":1,"b":2}`, + fields: "c", + want: `{}`, + }, + { + // Null pagination cursors are common envelope metadata. + // json.Unmarshal accepts JSON null into []json.RawMessage as + // a nil slice without error, so the array check must reject + // nil explicitly or null siblings would be coerced to `[]`. + name: "envelope preserves null sibling verbatim", + input: `{"items":[{"id":"a","name":"x"}],"next_cursor":null}`, + fields: "id,name", + want: `{"items":[{"id":"a","name":"x"}],"next_cursor":null}`, + }, + { + // Without a real array sibling the envelope fallback does not + // fire, so a flat object whose only "extra" key is null still + // returns {} for a non-matching selector. + name: "flat object with null sibling no match returns empty", + input: `{"a":1,"b":null}`, + fields: "c", + want: `{}`, + }, + { + // Multiple array siblings at the same level each receive the + // selector independently. Documents that the projection fans + // out across every array, not just the first one found. + name: "envelope with two array siblings filters both", + input: `{"events":[{"id":"e1","other":"x"}],"speakers":[{"id":"s1","other":"y"}]}`, + fields: "id", + want: `{"events":[{"id":"e1"}],"speakers":[{"id":"s1"}]}`, + }, + { + // Envelope fallback is intentionally one level deep. A nested + // object envelope like {"data":{"items":[...]}} surfaces no + // array at the outer level, so the fallback does not fire and + // the result is the empty-object that flat-no-match would + // produce. Pins the boundary so a future deeper-walk change + // is an explicit decision, not an accident. + name: "nested object envelope returns empty (one-level only)", + input: `{"data":{"items":[{"id":"a","other":"y"}]}}`, + fields: "id", + want: `{}`, + }, + } + for _, tc := range cases { + tc := tc + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + got := filterFields(json.RawMessage(tc.input), tc.fields) + // Normalize both sides through json.Unmarshal+Marshal so + // map-iteration order does not produce false negatives. + var gotV, wantV interface{} + if err := json.Unmarshal(got, &gotV); err != nil { + t.Fatalf("got is invalid json: %v (raw=%s)", err, string(got)) + } + if err := json.Unmarshal([]byte(tc.want), &wantV); err != nil { + t.Fatalf("want is invalid json: %v (raw=%s)", err, tc.want) + } + gotBytes, _ := json.Marshal(gotV) + wantBytes, _ := json.Marshal(wantV) + if string(gotBytes) != string(wantBytes) { + t.Errorf("filterFields(%q, %q) = %s, want %s", + tc.input, tc.fields, string(gotBytes), string(wantBytes)) + } + }) + } +} diff --git a/library/productivity/devonthink/internal/cli/search.go b/library/productivity/devonthink/internal/cli/search.go new file mode 100644 index 0000000000..7984406f81 --- /dev/null +++ b/library/productivity/devonthink/internal/cli/search.go @@ -0,0 +1,304 @@ +// Copyright 2026 rowdy and contributors. Licensed under Apache-2.0. See LICENSE. +// Generated by CLI Printing Press (https://github.com/mvanhorn/cli-printing-press). DO NOT EDIT. + +package cli + +import ( + "encoding/json" + "fmt" + "strings" + + "github.com/mvanhorn/printing-press-library/library/productivity/devonthink/internal/store" + "github.com/spf13/cobra" +) + +// isNilOrEmpty checks whether a JSON object has nil or empty values for +// common identifier fields (title, name, identifier, id). +// Also checks nested "document" objects for search result wrappers. +func isNilOrEmpty(raw json.RawMessage) bool { + var obj map[string]interface{} + if err := json.Unmarshal(raw, &obj); err != nil { + return true + } + // Check top-level fields + for _, key := range []string{"title", "name", "identifier", "id"} { + if v, ok := obj[key]; ok { + if v == nil { + continue + } + if s, ok := v.(string); ok && strings.TrimSpace(s) != "" { + return false + } + // Non-string, non-nil value (e.g. numeric ID) — keep it + if _, ok := v.(string); !ok { + return false + } + } + } + // Check nested "document" for search result wrappers like {score, document: {name, ...}} + if doc, ok := obj["document"]; ok { + if docMap, ok := doc.(map[string]interface{}); ok { + for _, key := range []string{"title", "name", "identifier", "id", "slug"} { + if v, ok := docMap[key]; ok && v != nil { + if s, ok := v.(string); ok && strings.TrimSpace(s) != "" { + return false + } + if _, ok := v.(string); !ok { + return false + } + } + } + } + } + // If the object has a "score" field, it's likely a search result — keep it + if _, ok := obj["score"]; ok { + return false + } + return true +} + +// extractSearchResults unwraps API search responses by checking common envelope paths. +func extractSearchResults(data json.RawMessage) []json.RawMessage { + // Try direct array first + var items []json.RawMessage + if json.Unmarshal(data, &items) == nil { + return items + } + // Try common wrapper paths: data, results, items + var wrapped map[string]json.RawMessage + if json.Unmarshal(data, &wrapped) == nil { + for _, key := range []string{"data", "results", "items", "records", "entries"} { + if inner, ok := wrapped[key]; ok { + if json.Unmarshal(inner, &items) == nil { + return items + } + } + } + } + // Return as single-item array + return []json.RawMessage{data} +} + +func newSearchCmd(flags *rootFlags) *cobra.Command { + var resourceType string + var limit int + var dbPath string + + cmd := &cobra.Command{ + Use: "search <query>", + Short: "Full-text search across synced data or live API", + Long: `Search data using FTS5 full-text search on locally synced data, +or hit the API's search endpoint when available. + +In auto mode (default): uses the API search endpoint if the API has one, +otherwise searches local data. Falls back to local on network failure. +In live mode: uses the API search endpoint only. +In local mode: searches locally synced data only.`, + Example: ` # Search (uses API endpoint if available, local FTS otherwise) + devonthink-pp-cli search "error timeout" + + # Force local search only + devonthink-pp-cli search "payment failed" --data-source local + + # Search a specific resource type locally + devonthink-pp-cli search "critical" --type transactions --data-source local + + # JSON output for piping + devonthink-pp-cli search "critical" --json --limit 20`, + Annotations: map[string]string{"mcp:hidden": "true"}, + RunE: func(cmd *cobra.Command, args []string) error { + if len(args) == 0 { + return cmd.Help() + } + query := args[0] + // This API has a search endpoint: GET /mirror/search + if flags.dataSource != "local" { + c, err := flags.newClient() + if err != nil { + return err + } + data, getErr := c.Get(cmd.Context(), "/mirror/search", map[string]string{ + "query": query, + }) + if getErr == nil { + // Live search succeeded + results := extractSearchResults(data) + prov := DataProvenance{Source: "live"} + return outputSearchResults(cmd, flags, results, limit, prov) + } + // Check if it's a network error for auto-mode fallback + if flags.dataSource == "live" || !isNetworkError(getErr) { + return classifyAPIError(getErr, flags) + } + // auto mode + network error: fall through to local FTS + fmt.Fprintf(cmd.ErrOrStderr(), "API unreachable, falling back to local search.\n") + } + + // Local FTS search + if dbPath == "" { + dbPath = defaultDBPath("devonthink-pp-cli") + } + + db, err := store.OpenWithContext(cmd.Context(), dbPath) + if err != nil { + return fmt.Errorf("opening local database: %w\nRun 'devonthink-pp-cli sync' first to populate the local database.", err) + } + defer db.Close() + + maybeEmitSyncHints(cmd, db, resourceType, flags.maxAge) + + var results []json.RawMessage + switch resourceType { + case "mcp": + results, err = db.SearchMcp(query, limit) + case "media": + results, err = db.SearchMedia(query, limit) + case "records": + results, err = db.SearchRecords(query, limit) + case "selection": + results, err = db.SearchSelection(query, limit) + case "": + // Search every FTS-enabled source — typed per-resource tables + // AND the generic resources_fts — and dedup by raw JSON so a + // row indexed in multiple FTS sources appears once. Without + // the generic-search call, rows that landed in resources_fts + // but not in any typed FTS table (e.g., a resource whose sync + // populated only the generic index) silently return zero. + seen := make(map[string]bool) + _ = seen // prevent unused error when no FTS tables exist + { + partial, searchErr := db.SearchMcp(query, limit) + if searchErr != nil { + return fmt.Errorf("search mcp failed: %w", searchErr) + } + for _, r := range partial { + key := string(r) + if !seen[key] { + seen[key] = true + results = append(results, r) + } + } + } + { + partial, searchErr := db.SearchMedia(query, limit) + if searchErr != nil { + return fmt.Errorf("search media failed: %w", searchErr) + } + for _, r := range partial { + key := string(r) + if !seen[key] { + seen[key] = true + results = append(results, r) + } + } + } + { + partial, searchErr := db.SearchRecords(query, limit) + if searchErr != nil { + return fmt.Errorf("search records failed: %w", searchErr) + } + for _, r := range partial { + key := string(r) + if !seen[key] { + seen[key] = true + results = append(results, r) + } + } + } + { + partial, searchErr := db.SearchSelection(query, limit) + if searchErr != nil { + return fmt.Errorf("search selection failed: %w", searchErr) + } + for _, r := range partial { + key := string(r) + if !seen[key] { + seen[key] = true + results = append(results, r) + } + } + } + { + partial, searchErr := db.Search(query, limit) + if searchErr != nil { + return fmt.Errorf("search resources_fts failed: %w", searchErr) + } + for _, r := range partial { + key := string(r) + if !seen[key] { + seen[key] = true + results = append(results, r) + } + } + } + default: + // Unrecognized type -- filter generic resources by type. + results, err = db.Search(query, limit, resourceType) + } + if err != nil { + return fmt.Errorf("search failed: %w", err) + } + + reason := "user_requested" + if flags.dataSource == "auto" { + reason = "api_unreachable" + } + prov := localProvenance(db, "search", reason) + + return outputSearchResults(cmd, flags, results, limit, prov) + }, + } + + cmd.Flags().StringVar(&resourceType, "type", "", "Filter by resource type") + cmd.Flags().IntVar(&limit, "limit", 50, "Maximum results to return") + cmd.Flags().StringVar(&dbPath, "db", "", "Database path (default: ~/.local/share/devonthink-pp-cli/data.db)") + + return cmd +} + +// outputSearchResults filters, counts, and outputs search results with provenance. +func outputSearchResults(cmd *cobra.Command, flags *rootFlags, results []json.RawMessage, limit int, prov DataProvenance) error { + // Filter out entries with nil or empty identifier fields. + filtered := make([]json.RawMessage, 0, len(results)) + for _, r := range results { + if !isNilOrEmpty(r) { + filtered = append(filtered, r) + } + } + results = filtered + + // Enforce limit across aggregated results. + if len(results) > limit { + results = results[:limit] + } + + jsonMode := flags.asJSON || !isTerminal(cmd.OutOrStdout()) + + // JSON mode always emits a valid envelope, including on no matches — + // agents pipe stdout through json.loads / jq and need parseable output + // regardless of result count. The filtered slice is built via make + // above, so it's non-nil even when empty; json.Marshal renders that + // as `[]` rather than `null`. + if jsonMode { + data, err := json.Marshal(results) + if err != nil { + return err + } + wrapped, err := wrapWithProvenance(data, prov) + if err != nil { + return err + } + return printOutput(cmd.OutOrStdout(), wrapped, true) + } + + if len(results) == 0 { + fmt.Fprintf(cmd.ErrOrStderr(), "No results (source: %s)\n", prov.Source) + return nil + } + + printProvenance(cmd, len(results), prov) + for _, r := range results { + fmt.Fprintln(cmd.OutOrStdout(), string(r)) + } + return nil +} diff --git a/library/productivity/devonthink/internal/cli/selection.go b/library/productivity/devonthink/internal/cli/selection.go new file mode 100644 index 0000000000..f6920a2579 --- /dev/null +++ b/library/productivity/devonthink/internal/cli/selection.go @@ -0,0 +1,22 @@ +// Copyright 2026 rowdy and contributors. Licensed under Apache-2.0. See LICENSE. +// Generated by CLI Printing Press (https://github.com/mvanhorn/cli-printing-press). DO NOT EDIT. + +package cli + +import ( + "github.com/spf13/cobra" +) + +func newSelectionCmd(flags *rootFlags) *cobra.Command { + cmd := &cobra.Command{ + Use: "selection", + Short: "Current DEVONthink GUI selection", + Hidden: true, + Annotations: map[string]string{"mcp:read-only": "true"}, + RunE: parentNoSubcommandRunE(flags), + } + + cmd.AddCommand(newSelectionGetCmd(flags)) + cmd.AddCommand(newSelectionSnapshotCmd(flags)) + return cmd +} diff --git a/library/productivity/devonthink/internal/cli/selection_get.go b/library/productivity/devonthink/internal/cli/selection_get.go new file mode 100644 index 0000000000..99bda40e14 --- /dev/null +++ b/library/productivity/devonthink/internal/cli/selection_get.go @@ -0,0 +1,79 @@ +// Copyright 2026 rowdy and contributors. Licensed under Apache-2.0. See LICENSE. +// Generated by CLI Printing Press (https://github.com/mvanhorn/cli-printing-press). DO NOT EDIT. + +package cli + +import ( + "encoding/json" + "fmt" + "os" + + "github.com/spf13/cobra" +) + +func newSelectionGetCmd(flags *rootFlags) *cobra.Command { + + cmd := &cobra.Command{ + Use: "get", + Short: "Return currently selected records", + Example: " devonthink-pp-cli selection get", + Annotations: map[string]string{"pp:endpoint": "selection.get", "pp:method": "GET", "pp:path": "/selection", "mcp:read-only": "true"}, + RunE: func(cmd *cobra.Command, args []string) error { + c, err := flags.newClient() + if err != nil { + return err + } + + path := "/selection" + params := map[string]string{} + data, prov, err := resolveReadWithStrategy(cmd.Context(), c, flags, "auto", "selection", false, path, params, nil, cmd.ErrOrStderr()) + if err != nil { + return classifyAPIError(err, flags) + } + // Print provenance to stderr for human-facing output only. + // Machine-format flags (--json, --csv, --compact, --quiet, --plain, + // --select) and piped stdout suppress this line; the JSON envelope + // already carries meta.source for those consumers. + // SYNC: keep this gate aligned with command_promoted.go.tmpl. + if wantsHumanTable(cmd.OutOrStdout(), flags) { + var countItems []json.RawMessage + _ = json.Unmarshal(data, &countItems) + printProvenance(cmd, len(countItems), prov) + } + // For JSON output, wrap with provenance envelope before passing through flags. + // --select wins over --compact when both are set; --compact only runs when + // no explicit fields were requested. Explicit format flags (--csv, --quiet, + // --plain) opt out of the auto-JSON path so piped consumers that asked for + // a non-JSON format reach the standard pipeline below. + if flags.asJSON || (!isTerminal(cmd.OutOrStdout()) && !flags.csv && !flags.quiet && !flags.plain) { + filtered := data + if flags.selectFields != "" { + filtered = filterFields(filtered, flags.selectFields) + } else if flags.compact { + filtered = compactFields(filtered) + } + wrapped, wrapErr := wrapWithProvenance(filtered, prov) + if wrapErr != nil { + return wrapErr + } + return printOutput(cmd.OutOrStdout(), wrapped, true) + } + // For all other output modes (table, csv, plain, quiet), use the standard pipeline + if wantsHumanTable(cmd.OutOrStdout(), flags) { + var items []map[string]any + if json.Unmarshal(data, &items) == nil && len(items) > 0 { + if err := printAutoTable(cmd.OutOrStdout(), items); err != nil { + return err + } + if len(items) >= 25 { + fmt.Fprintf(os.Stderr, "\nShowing %d results. To narrow: add --limit, --json --select, or filter flags.\n", len(items)) + } + return nil + } + } + return printOutputWithFlags(cmd.OutOrStdout(), data, flags) + }, + } + + return cmd +} diff --git a/library/productivity/devonthink/internal/cli/selection_snapshot.go b/library/productivity/devonthink/internal/cli/selection_snapshot.go new file mode 100644 index 0000000000..322691b02a --- /dev/null +++ b/library/productivity/devonthink/internal/cli/selection_snapshot.go @@ -0,0 +1,89 @@ +// Copyright 2026 rowdy and contributors. Licensed under Apache-2.0. See LICENSE. +// Generated by CLI Printing Press (https://github.com/mvanhorn/cli-printing-press). DO NOT EDIT. + +package cli + +import ( + "encoding/json" + "fmt" + "os" + + "github.com/spf13/cobra" +) + +func newSelectionSnapshotCmd(flags *rootFlags) *cobra.Command { + var flagNote string + var flagOutput string + + cmd := &cobra.Command{ + Use: "snapshot", + Short: "Capture the current selection as a reusable workflow seed", + Example: " devonthink-pp-cli selection snapshot --agent", + Annotations: map[string]string{"pp:endpoint": "selection.snapshot", "pp:method": "GET", "pp:path": "/selection/snapshot", "mcp:read-only": "true"}, + RunE: func(cmd *cobra.Command, args []string) error { + c, err := flags.newClient() + if err != nil { + return err + } + + path := "/selection/snapshot" + params := map[string]string{} + if flagNote != "" { + params["note"] = formatCLIParamValue(flagNote) + } + if flagOutput != "" { + params["output"] = formatCLIParamValue(flagOutput) + } + data, prov, err := resolveReadWithStrategy(cmd.Context(), c, flags, "auto", "selection", false, path, params, nil, cmd.ErrOrStderr()) + if err != nil { + return classifyAPIError(err, flags) + } + // Print provenance to stderr for human-facing output only. + // Machine-format flags (--json, --csv, --compact, --quiet, --plain, + // --select) and piped stdout suppress this line; the JSON envelope + // already carries meta.source for those consumers. + // SYNC: keep this gate aligned with command_promoted.go.tmpl. + if wantsHumanTable(cmd.OutOrStdout(), flags) { + var countItems []json.RawMessage + _ = json.Unmarshal(data, &countItems) + printProvenance(cmd, len(countItems), prov) + } + // For JSON output, wrap with provenance envelope before passing through flags. + // --select wins over --compact when both are set; --compact only runs when + // no explicit fields were requested. Explicit format flags (--csv, --quiet, + // --plain) opt out of the auto-JSON path so piped consumers that asked for + // a non-JSON format reach the standard pipeline below. + if flags.asJSON || (!isTerminal(cmd.OutOrStdout()) && !flags.csv && !flags.quiet && !flags.plain) { + filtered := data + if flags.selectFields != "" { + filtered = filterFields(filtered, flags.selectFields) + } else if flags.compact { + filtered = compactFields(filtered) + } + wrapped, wrapErr := wrapWithProvenance(filtered, prov) + if wrapErr != nil { + return wrapErr + } + return printOutput(cmd.OutOrStdout(), wrapped, true) + } + // For all other output modes (table, csv, plain, quiet), use the standard pipeline + if wantsHumanTable(cmd.OutOrStdout(), flags) { + var items []map[string]any + if json.Unmarshal(data, &items) == nil && len(items) > 0 { + if err := printAutoTable(cmd.OutOrStdout(), items); err != nil { + return err + } + if len(items) >= 25 { + fmt.Fprintf(os.Stderr, "\nShowing %d results. To narrow: add --limit, --json --select, or filter flags.\n", len(items)) + } + return nil + } + } + return printOutputWithFlags(cmd.OutOrStdout(), data, flags) + }, + } + cmd.Flags().StringVar(&flagNote, "note", "", "Operator note to include in the snapshot") + cmd.Flags().StringVar(&flagOutput, "output", "", "Optional output JSON file") + + return cmd +} diff --git a/library/productivity/devonthink/internal/cli/sync.go b/library/productivity/devonthink/internal/cli/sync.go new file mode 100644 index 0000000000..94453ff4ff --- /dev/null +++ b/library/productivity/devonthink/internal/cli/sync.go @@ -0,0 +1,1655 @@ +// Copyright 2026 rowdy and contributors. Licensed under Apache-2.0. See LICENSE. +// Generated by CLI Printing Press (https://github.com/mvanhorn/cli-printing-press). DO NOT EDIT. + +package cli + +import ( + "context" + "encoding/json" + "fmt" + "github.com/mvanhorn/printing-press-library/library/productivity/devonthink/internal/cliutil" + "github.com/mvanhorn/printing-press-library/library/productivity/devonthink/internal/store" + "github.com/spf13/cobra" + "io" + "net/url" + "os" + "regexp" + "strconv" + "strings" + "sync" + "sync/atomic" + "time" +) + +// unresolvedPathKeyRE matches `{key}` placeholders left in a sync path +// after syncResourcePath() resolution. Hierarchical APIs (Yahoo Fantasy, +// Reddit pre-2024, YouTube Data v3, MLB Stats, etc.) declare paths like +// "/league/{league_key}/players" that can only be filled from parent +// context — flat-list sync cannot fill them. Resources with unresolved +// keys emit sync_warning and are skipped without aborting the run, so +// sync still completes for resources that DO have resolvable paths. +var unresolvedPathKeyRE = regexp.MustCompile(`\{[a-zA-Z_][a-zA-Z0-9_]*\}`) + +// syncResult holds the outcome of syncing a single resource. +type syncResult struct { + Resource string + Count int + Err error + Warn error + Duration time.Duration +} + +func newSyncCmd(flags *rootFlags) *cobra.Command { + var resources []string + var full bool + var since string + var concurrency int + var dbPath string + var maxPages int + var latestOnly bool + var strict bool + var paramFlags []string + var resourceParamFlags []string + var globalParamFlags []string + + cmd := &cobra.Command{ + Use: "sync", + Short: "Sync API data to local SQLite for offline search and analysis", + Long: `Sync data from the API into a local SQLite database. Supports resumable +incremental sync (only fetches new data since last sync) and full resync. +Once synced, use the 'search' command for instant full-text search. + +Exit codes & warnings: + Resources the API denies access to (HTTP 403, or HTTP 400 with an + access-policy body) are reported as warnings rather than failing the + run. In --json mode each is emitted as a {"event":"sync_warning",...} + line carrying status, reason, and message fields, and a final + {"event":"sync_summary",...} aggregates the run. + + Exit 0 when at least one resource synced and no resource flagged in + the spec as critical (x-critical: true) failed; non-critical failures + emit {"event":"sync_warning","reason":"exit_policy_default_changed", + ...} so callers can detect that a partial failure was tolerated. Pass + --strict to exit non-zero on any per-resource failure. Exit is always + non-zero when every selected resource failed, regardless of --strict. + +Resource scoping: + --resources runs the named top-level resources, plus any parent-keyed + dependent whose own name is listed OR whose parent table is listed. + Naming a parent therefore cascades to its dependents (so "sync this + parent and its children" works without listing every nested resource + by hand). There is no flag today to suppress the cascade for a named + parent. To run a dependent without re-syncing its parent, list only + the dependent by name; the parent table must already be populated + from a prior sync.`, + Example: ` # Sync all resources + devonthink-pp-cli sync + + # Sync specific resources only + devonthink-pp-cli sync --resources channels,messages + + # Full resync (ignore previous checkpoint) + devonthink-pp-cli sync --full + + # Incremental sync: only records from the last 7 days + devonthink-pp-cli sync --since 7d + + # Parallel sync with 8 workers + devonthink-pp-cli sync --concurrency 8 + + # Latest-only: refresh head of each resource, no historical backfill + devonthink-pp-cli sync --latest-only`, + RunE: func(cmd *cobra.Command, args []string) error { + userParams, err := parseSyncUserParams(paramFlags, resourceParamFlags, globalParamFlags) + if err != nil { + return usageErr(err) + } + + c, err := flags.newClient() + if err != nil { + return err + } + c.NoCache = true + + if dbPath == "" { + dbPath = defaultDBPath("devonthink-pp-cli") + } + + db, err := store.OpenWithContext(cmd.Context(), dbPath) + if err != nil { + return fmt.Errorf("opening local database: %w", err) + } + defer db.Close() + + syncEventWriter := cmd.OutOrStdout() + + // If no specific resources, sync top-level resources + if len(resources) == 0 { + resources = defaultSyncResources() + } + + // Reject --resource-param keys that don't match a known resource. + // Validates against the full top-level + dependent set, not the + // user-filtered `resources` slice, so legitimate cases like + // "filter to A, but apply param to B if it gets synced" still + // catch typos without false positives. + if err := userParams.validateResourceNames(knownSyncResourceNames()); err != nil { + return usageErr(err) + } + + // --full: clear all sync cursors before starting + if full { + for _, resource := range resources { + _ = db.SaveSyncState(resource, "", 0) + } + } + + if cliutil.IsDogfoodEnv() && !cmd.Flags().Changed("max-pages") { + maxPages = 10 + } + + // --latest-only narrows to the first page of each resource + // ignoring the historical resume cursor. We cap maxPages at 1 + // here rather than re-interpreting it downstream so the rest + // of the sync loop stays oblivious. Mutually-useful with + // --since: if the user set --since, that threshold still wins + // and we don't short-circuit historical context they asked for. + if latestOnly { + if since == "" { + maxPages = 1 + // Clear the cursor so we start from the head each time; + // the goal of --latest-only is "refresh the top" not + // "resume from wherever I left off". + for _, resource := range resources { + existing, _, _, _ := db.GetSyncState(resource) + if existing != "" { + _ = db.SaveSyncState(resource, "", 0) + } + } + } else if humanFriendly { + fmt.Fprintln(os.Stderr, "warning: --latest-only ignored because --since is set; --since takes precedence") + } + } + // effectiveLatestOnly drives the max_pages_cap_hit suppression + // below. It must reflect whether --latest-only is actually the + // cap source, i.e. only when --since is empty. If --since wins + // (block above), --latest-only is a no-op for maxPages and any + // cap hit reflects an explicit operator limit or the dogfood + // safety limit, which is a real anomaly worth surfacing. + effectiveLatestOnly := latestOnly && since == "" + + // Resolve --since into an RFC3339 timestamp + sinceTS := "" + if since != "" { + ts, err := parseSinceDuration(since) + if err != nil { + return fmt.Errorf("invalid --since value %q: %w", since, err) + } + sinceTS = ts.Format(time.RFC3339) + } + + // Worker pool: produce resources, N workers consume + if concurrency < 1 { + concurrency = 4 + } + // Under PRINTING_PRESS_VERIFY=1 (mock/dry-run), all goroutines + // reach SQLite without the natural serialization that network + // latency provides in real syncs, so the worker pool races on + // the writer and trips SQLITE_BUSY despite _busy_timeout. + if cliutil.IsVerifyEnv() { + concurrency = 1 + } + + started := time.Now() + work := make(chan string, len(resources)) + results := make(chan syncResult, len(resources)) + + var wg sync.WaitGroup + for i := 0; i < concurrency; i++ { + wg.Add(1) + go func() { + defer wg.Done() + for resource := range work { + res := syncResource(cmd.Context(), c, db, resource, sinceTS, full, maxPages, effectiveLatestOnly, userParams, syncEventWriter) + results <- res + } + }() + } + + // Enqueue all resources + for _, resource := range resources { + work <- resource + } + close(work) + + // Collect results in a separate goroutine + go func() { + wg.Wait() + close(results) + }() + + var totalSynced int + var errCount int + var criticalErrCount int + var warnCount int + var successCount int + for res := range results { + if res.Err != nil { + if humanFriendly { + fmt.Fprintf(os.Stderr, " %s: error: %v\n", res.Resource, res.Err) + } + errCount++ + if criticalResources[res.Resource] { + criticalErrCount++ + } + } else if res.Warn != nil { + if humanFriendly { + fmt.Fprintf(os.Stderr, " %s: warning: %v\n", res.Resource, res.Warn) + } + warnCount++ + } else { + if humanFriendly { + fmt.Fprintf(os.Stderr, " %s: %d synced (done)\n", res.Resource, res.Count) + } + totalSynced += res.Count + successCount++ + } + } + + elapsed := time.Since(started) + totalResources := successCount + warnCount + errCount + if humanFriendly { + if warnCount > 0 { + fmt.Fprintf(os.Stderr, "Sync complete: %d records across %d resources (%d warned, %.1fs)\n", + totalSynced, totalResources, warnCount, elapsed.Seconds()) + } else { + fmt.Fprintf(os.Stderr, "Sync complete: %d records across %d resources (%.1fs)\n", + totalSynced, totalResources, elapsed.Seconds()) + } + } else { + fmt.Fprintf(syncEventWriter, `{"event":"sync_summary","total_records":%d,"resources":%d,"success":%d,"warned":%d,"errored":%d,"duration_ms":%d}`+"\n", + totalSynced, totalResources, successCount, warnCount, errCount, elapsed.Milliseconds()) + } + + // Exit-code policy: + // 1. --strict + any error -> non-zero (legacy contract) + // 2. any critical failure -> non-zero regardless of --strict + // 3. nothing synced -> non-zero (preserves "all-warned" / "all-errored" exit) + // 4. otherwise -> exit 0 (any data synced + no critical failed) + // When branch 4 suppresses what branch 1 would have rejected, emit a + // one-shot sync_warning with reason "exit_policy_default_changed" so + // CI scripts that depend on $? != 0 can discover the contract change + // without reading the CHANGELOG. + if strict && errCount > 0 { + return fmt.Errorf("%d resource(s) failed to sync", errCount) + } + if criticalErrCount > 0 { + return fmt.Errorf("%d critical resource(s) failed to sync", criticalErrCount) + } + if successCount == 0 { + if warnCount > 0 && errCount == 0 { + return fmt.Errorf("%d resource(s) skipped due to insufficient access", warnCount) + } + if errCount > 0 { + return fmt.Errorf("%d resource(s) failed to sync", errCount) + } + } + if errCount > 0 && !strict && criticalErrCount == 0 && successCount > 0 { + if !humanFriendly { + msg := fmt.Sprintf("%d resource(s) failed but exit code is 0 because the new default treats non-critical failures as warnings. Pass --strict to restore the old behavior, or annotate critical resources with x-critical: true. See CHANGELOG.", errCount) + fmt.Fprintf(syncEventWriter, `{"event":"sync_warning","reason":"exit_policy_default_changed","errored":%d,"message":"%s"}`+"\n", + errCount, strings.ReplaceAll(msg, `"`, `\"`)) + } else { + fmt.Fprintf(os.Stderr, "warning: %d resource(s) failed but exit code is 0 because the new default treats non-critical failures as warnings. Pass --strict to restore the old behavior, or annotate critical resources with x-critical: true.\n", errCount) + } + } + return nil + }, + } + + cmd.Flags().StringSliceVar(&resources, "resources", nil, "Comma-separated resource types to sync. Naming a parent also runs its parent-keyed dependents (see Long help for scoping).") + cmd.Flags().BoolVar(&full, "full", false, "Full resync (ignore previous checkpoint)") + cmd.Flags().StringVar(&since, "since", "", "Incremental sync duration (e.g. 7d, 24h, 1w, 30m)") + cmd.Flags().IntVar(&concurrency, "concurrency", 4, "Number of parallel sync workers") + cmd.Flags().StringVar(&dbPath, "db", "", "Database path (default: ~/.local/share/devonthink-pp-cli/data.db)") + cmd.Flags().IntVar(&maxPages, "max-pages", 0, "Maximum pages to fetch per resource (0 = unlimited; cap-hit emits a sync_warning event)") + cmd.Flags().BoolVar(&latestOnly, "latest-only", false, "Refresh head of each resource only; clears resume cursor and caps pages at 1. Mutually exclusive with --since (--since wins).") + cmd.Flags().BoolVar(&strict, "strict", false, "Exit non-zero on any per-resource failure (default: only critical failures or all-resource failure exit non-zero).") + cmd.Flags().StringArrayVar(¶mFlags, "param", nil, "Extra query param to inject into flat-list sync requests (repeatable, key=value). Skipped on path-scoped dependent requests so a top-level scope like workspace=<id> does not double up on /parents/<id>/children calls. Use --global-param to inject everywhere. Avoid pagination keys (limit/since/cursor) — overriding them corrupts resume state.") + cmd.Flags().StringArrayVar(&resourceParamFlags, "resource-param", nil, "Per-resource extra query param (repeatable, resource:key=value). Wins over --param and --global-param when keys conflict.") + cmd.Flags().StringArrayVar(&globalParamFlags, "global-param", nil, "Extra query param to inject into every sync request including dependent path-scoped calls (repeatable, key=value). Use when an API requires a scope on every call regardless of path nesting.") + + return cmd +} + +// syncResource handles the full paginated sync of a single resource. +// It resumes from the last cursor unless sinceTS or full mode overrides it. +// channel_workflow.go.tmpl mirrors the trailing dates arg conditional; +// keep both call sites in sync if this signature changes. +func syncResource(ctx context.Context, c interface { + Get(context.Context, string, map[string]string) (json.RawMessage, error) + RateLimit() float64 +}, db *store.Store, resource, sinceTS string, full bool, maxPages int, latestOnly bool, userParams *syncUserParams, syncEvents io.Writer) syncResult { + started := time.Now() + if syncEvents == nil { + syncEvents = io.Discard + } + + if !humanFriendly { + fmt.Fprintf(syncEvents, `{"event":"sync_start","resource":"%s"}`+"\n", resource) + } + + path, err := syncResourcePath(resource) + if err != nil { + return syncResult{Resource: resource, Err: err, Duration: time.Since(started)} + } + + // Skip resources whose path template still contains unresolved `{key}` + // placeholders after syncResourcePath() resolution. These paths require + // parent context (league_key, team_key, channel_id, etc.) that flat-list + // sync cannot fill. Emit a sync_warning describing the missing keys and + // continue — sync exits 0 if any resource succeeded, so this keeps + // hierarchical-API CLIs functional for the resources they CAN sync flat. + if missingKeys := unresolvedPathKeyRE.FindAllString(path, -1); len(missingKeys) > 0 { + if !humanFriendly { + payload := struct { + Event string `json:"event"` + Resource string `json:"resource"` + Reason string `json:"reason"` + Keys []string `json:"keys"` + Path string `json:"path"` + Message string `json:"message"` + }{ + Event: "sync_warning", + Resource: resource, + Reason: "unfilled_path_key", + Keys: missingKeys, + Path: path, + Message: fmt.Sprintf("path %s requires parent context (%s); resource skipped", path, strings.Join(missingKeys, ", ")), + } + payloadJSON, _ := json.Marshal(payload) + fmt.Fprintf(syncEvents, "%s\n", payloadJSON) + } else { + fmt.Fprintf(os.Stderr, " %s skipped (requires parent context: %s)\n", + resource, strings.Join(missingKeys, ", ")) + } + return syncResult{ + Resource: resource, + Warn: fmt.Errorf("skipped %s: unresolved path keys %v", resource, missingKeys), + Duration: time.Since(started), + } + } + + var totalCount int + + // Resume cursor from sync_state (unless --full cleared it) + existingCursor, lastSynced, _, _ := db.GetSyncState(resource) + if !full { + if storedCount, err := db.Count(resource); err == nil && storedCount == 0 { + existingCursor = "" + lastSynced = time.Time{} + } + } + + // Determine the since param value: + // 1. Explicit --since flag takes priority + // 2. Otherwise use last_synced_at from sync_state for incremental sync + sinceParam := syncResourceSinceParam(resource) + effectiveSince := sinceTS + if effectiveSince == "" && !lastSynced.IsZero() && !full { + effectiveSince = lastSynced.Format(time.RFC3339) + } + // Resources whose list endpoint declares no temporal-filter parameter + // fall back to plain pagination — sending a synthetic since=... would + // reach the API as an unknown query param and (for strict APIs like + // Notion) fail the whole resource with a 400. Warn once per resource + // when the user expected incremental behavior. + if effectiveSince != "" && sinceParam == "" { + if humanFriendly { + fmt.Fprintf(os.Stderr, " %s: incremental sync ignored (endpoint declares no temporal filter; falling back to full pagination)\n", resource) + } else { + fmt.Fprintf(syncEvents, `{"event":"sync_warning","resource":"%s","reason":"resource_not_incremental","message":"endpoint does not declare a temporal filter parameter; incremental sync has no effect for this resource"}`+"\n", resource) + } + effectiveSince = "" + } + if effectiveSince != "" { + effectiveSince = formatSyncSinceValue(effectiveSince, syncResourceSinceParamFormat(resource)) + } + + cursor := existingCursor + pageSize := determinePaginationDefaults() + var progressCount int64 + pagesFetched := 0 + lastNextCursor := "" + capExitHit := false + capExitCursor := "" + // extractFailureTotal accumulates per-item primary-key extraction + // misses across pages within this resource sync. Resource-level + // concurrency is 1 (one goroutine per resource via the work channel) + // so this counter cannot race. We emit one primary_key_unresolved + // sync_anomaly per resource per run when there's at least one miss + // (rate-limited via the anomalyEmitted flag) and a roll-up + // all_items_failed_id_extraction event when 100% of a single page + // failed extraction. + var extractFailureTotal int + var consumedTotal int + anomalyEmitted := false + + for { + params := map[string]string{} + + if resourceSupportsPagination(resource) { + params[pageSize.limitParam] = strconv.Itoa(pageSize.limit) + if cursor != "" { + params[pageSize.cursorParam] = cursor + } + } + + // Set since filter + if effectiveSince != "" { + params[sinceParam] = effectiveSince + } + // Apply user-supplied --param / --resource-param overrides last so they + // win over spec-derived defaults (e.g. forcing mine=true on a list + // endpoint whose OpenAPI spec marks the filter optional). + userParams.applyTo(resource, params, false) + + data, err := c.Get(ctx, path, params) + if err != nil { + if w, ok := isSyncAccessWarning(err); ok { + if !humanFriendly { + fmt.Fprintln(syncEvents, syncWarningJSON(resource, "", w.Status, w.Reason, w.Message)) + } + return syncResult{Resource: resource, Count: totalCount, Warn: fmt.Errorf("skipped %s: %s", resource, w.Reason), Duration: time.Since(started)} + } + if !humanFriendly { + fmt.Fprintln(syncEvents, syncErrorJSON(resource, "", err)) + } + return syncResult{Resource: resource, Count: totalCount, Err: fmt.Errorf("fetching %s: %w", resource, err), Duration: time.Since(started)} + } + + // Dry-run sentinel: client.dryRun returns `{"dry_run": true}` instead + // of a real response when --dry-run is set. The upsert path below + // would otherwise fail with "missing id for <resource>" because the + // sentinel has no items and no id; emit a synthetic success event so + // validate-narrative --full-examples (which auto-appends --dry-run) + // sees a clean exit. + if isDryRunResponse(data) { + if !humanFriendly { + fmt.Fprintf(syncEvents, `{"event":"sync_dryrun","resource":"%s"}`+"\n", resource) + } + return syncResult{Resource: resource, Count: 0, Duration: time.Since(started)} + } + + // Try to extract items from the response. + // Strategy: try array first, then common wrapper keys. + items, nextCursor, hasMore := extractPageItems(data, pageSize.cursorParam) + + // Page-int paginator fallback: when the API paginates by integer + // ?page=N and emits no body cursor, treat a full page as a signal + // to advance numerically. Without this the loop breaks after page + // 1 even though more pages exist (the original symptom in #1296). + // Guard on cursorType, not cursorParam name, so all canonical + // spellings (page / page_number / pageNumber / page[number]) work. + if pageSize.cursorType == "page" && nextCursor == "" && len(items) >= pageSize.limit && pageAllowsPageIntFallback(data) { + currentPage, _ := strconv.Atoi(cursor) + if currentPage < 1 { + currentPage = 1 + } + nextCursor = strconv.Itoa(currentPage + 1) + hasMore = true + } + if pageSize.cursorType == "offset" && nextCursor == "" && len(items) >= pageSize.limit && pageAllowsPageIntFallback(data) { + currentOffset, _ := strconv.Atoi(cursor) + nextCursor = strconv.Itoa(currentOffset + pageSize.limit) + hasMore = true + } + + if len(items) == 0 && len(data) > 0 && !isJSONResponse(data) { + if humanFriendly { + fmt.Fprintf(os.Stderr, "\nwarning: %s returned a 200 response with a non-JSON body; no rows were stored.\n", resource) + } else { + fmt.Fprintf(syncEvents, `{"event":"sync_anomaly","resource":"%s","reason":"non_json_200_body"}`+"\n", resource) + } + break + } + + if len(items) == 0 { + if isEmptyPageResponse(data) { + break + } + // Single object response - try to store as-is + if err := upsertSingleObject(db, resource, data); err != nil { + if !humanFriendly { + fmt.Fprintln(syncEvents, syncErrorJSON(resource, "", err)) + } + return syncResult{Resource: resource, Err: err, Duration: time.Since(started)} + } + totalCount++ + break + } + + // Batch upsert all items from this page. UpsertBatch returns + // (stored, extractFailures, err): stored counts rows actually + // landed; extractFailures counts items that survived JSON + // unmarshal but had no extractable primary key (templated + // IDField AND generic fallback both missed). Tracking these + // separately lets us emit precise sync_anomaly events: a + // roll-up "all_items_failed_id_extraction" when an entire + // page yields zero stored, a per-resource + // "primary_key_unresolved" the first time any single item + // fails, and the F4b "stored_count_zero_after_extraction" + // probe when extraction succeeded but rows still didn't land. + stored, extractFailures, err := upsertResourceBatch(db, resource, items) + if err != nil { + if !humanFriendly { + fmt.Fprintln(syncEvents, syncErrorJSON(resource, "", err)) + } + return syncResult{Resource: resource, Count: totalCount, Err: fmt.Errorf("upserting batch for %s: %w", resource, err), Duration: time.Since(started)} + } + + consumedTotal += len(items) + extractFailureTotal += extractFailures + + // When a non-empty page yielded zero stored rows, the API + // returned items in a shape we couldn't extract IDs from — + // likely scalar IDs (Firebase /topstories.json, GitHub user- + // repo lists) where the spec author should declare a hydration + // pattern, or an unrecognized primary-key field name. + if len(items) > 0 && stored == 0 { + if humanFriendly { + fmt.Fprintf(os.Stderr, "warning: %s returned %d items but stored 0 — the local store will be empty for this resource. Likely cause: scalar item shape rather than objects with extractable IDs.\n", resource, len(items)) + } else { + fmt.Fprintf(syncEvents, `{"event":"sync_anomaly","resource":"%s","consumed":%d,"stored":0,"reason":"all_items_failed_id_extraction"}`+"\n", resource, len(items)) + } + anomalyEmitted = true + } else if extractFailures > 0 && !anomalyEmitted { + // Per-item primary-key resolution failure but at least one + // item landed — emit one structured warning per resource per + // sync run so users see the first occurrence of silent drops + // instead of waiting for total failure. + if humanFriendly { + fmt.Fprintf(os.Stderr, "\nwarning: %s had %d item(s) on this page with no extractable primary key — those rows were dropped silently. Annotate the spec with x-resource-id to fix.\n", resource, extractFailures) + } else { + fmt.Fprintf(syncEvents, `{"event":"sync_anomaly","resource":"%s","consumed":%d,"stored":%d,"count":%d,"reason":"primary_key_unresolved"}`+"\n", resource, len(items), stored, extractFailures) + } + anomalyEmitted = true + } + + totalCount += stored + atomic.AddInt64(&progressCount, int64(stored)) + if resourceSupportsPagination(resource) && nextCursor == "" && pageSize.cursorParam != "offset" && len(items) >= pageSize.limit && pageMayHaveMore(data) { + emitSyncMissingPaginationCursorWarning(syncEvents, humanFriendly, resource, "") + } + + // Progress reporting (include rate limit info when active) + currentRate := c.RateLimit() + if humanFriendly { + if currentRate > 0 { + fmt.Fprintf(os.Stderr, "\r %s: %d synced [%.1f req/s]", resource, atomic.LoadInt64(&progressCount), currentRate) + } else { + fmt.Fprintf(os.Stderr, "\r %s: %d synced", resource, atomic.LoadInt64(&progressCount)) + } + } else { + if currentRate > 0 { + fmt.Fprintf(syncEvents, `{"event":"sync_progress","resource":"%s","fetched":%d,"rate_rps":%.1f}`+"\n", resource, atomic.LoadInt64(&progressCount), currentRate) + } else { + fmt.Fprintf(syncEvents, `{"event":"sync_progress","resource":"%s","fetched":%d}`+"\n", resource, atomic.LoadInt64(&progressCount)) + } + } + + pagesFetched++ + + // Enforce page ceiling to prevent runaway syncs on large-catalog APIs. + // Suppress the cap-hit warning when --latest-only is the cap source: + // the template pinned maxPages=1 by user intent, and emitting one + // warning per paginated resource would mask real sync_anomaly / + // sync_error output in the same stream. + if maxPages > 0 && pagesFetched >= maxPages { + truncatedByCap := resourceSupportsPagination(resource) && hasMore + truncatedByCap = truncatedByCap && len(items) >= pageSize.limit + if truncatedByCap { + capExitCursor = nextCursor + } + if truncatedByCap && capExitCursor == "" { + if pageSize.cursorType == "offset" { + currentOffset, _ := strconv.Atoi(cursor) + capExitCursor = strconv.Itoa(currentOffset + pageSize.limit) + } else { + truncatedByCap = false + } + } + if truncatedByCap && capExitCursor != cursor { + if !latestOnly { + capExitHit = true + if humanFriendly { + fmt.Fprintf(os.Stderr, "\n %s: reached --max-pages limit (%d pages, %d items)\n", resource, maxPages, totalCount) + } else { + fmt.Fprintf(syncEvents, `{"event":"sync_warning","resource":"%s","reason":"max_pages_cap_hit","message":"reached --max-pages cap of %d; data may be truncated. Re-run with --max-pages 0 (unlimited) or higher to verify."}`+"\n", resource, maxPages) + } + } + } + break + } + + // Sticky-cursor detector: if the API echoes the same next cursor across + // consecutive pages without advancing, abort to prevent burning the + // --max-pages budget on a non-terminating loop. Checked AFTER the cap + // guard so cap-hit takes precedence; checked BEFORE the natural-end + // check below because the natural-end check would not catch a sticky + // non-empty cursor on its own. + if nextCursor != "" && nextCursor == lastNextCursor { + if humanFriendly { + fmt.Fprintf(os.Stderr, "\n %s: API returned the same next cursor across two pages; aborting to prevent budget waste.\n", resource) + } else { + fmt.Fprintf(syncEvents, `{"event":"sync_warning","resource":"%s","reason":"stuck_pagination","message":"API returned the same next cursor across two pages for resource %s; aborting to prevent budget waste."}`+"\n", resource, resource) + } + break + } + lastNextCursor = nextCursor + + // Determine if there are more pages. + if !resourceSupportsPagination(resource) { + break + } + if !hasMore || len(items) < pageSize.limit { + break + } + if nextCursor == "" { + if pageSize.cursorType == "offset" { + // Cursor-based APIs return the next cursor in the envelope. + // Offset-based APIs carry their pagination position client-side. + currentOffset, _ := strconv.Atoi(cursor) + nextCursor = strconv.Itoa(currentOffset + pageSize.limit) + } else { + // A cursor-based API reporting has_more without a next cursor + // cannot advance safely; stop instead of looping silently. + break + } + } + + // Save cursor after each page for resumability + if err := db.SaveSyncState(resource, nextCursor, totalCount); err != nil { + // Non-fatal: log and continue + fmt.Fprintf(os.Stderr, "\nwarning: failed to save sync state for %s: %v\n", resource, err) + } + + cursor = nextCursor + } + + // Final sync state: clear cursor on natural completion, but preserve the + // resume cursor when an operator intentionally capped the page budget. + finalCursor := "" + if capExitHit { + finalCursor = capExitCursor + } + _ = db.SaveSyncState(resource, finalCursor, totalCount) + + // F4b symptom probe: if items were consumed and successfully + // extracted (extractFailures < consumed) but nothing landed in + // the store, something downstream of extraction silently dropped + // rows — FTS5 trigger error, transaction rollback, character + // encoding. Emit a sync_anomaly so the symptom is visible the + // next time it recurs; the underlying root cause is held out for + // controlled repro. + if consumedTotal > 0 && totalCount == 0 && extractFailureTotal < consumedTotal { + if humanFriendly { + fmt.Fprintf(os.Stderr, "\nwarning: %s consumed %d items, extracted %d primary keys, but stored 0 rows — extraction succeeded yet nothing landed. Investigate FTS triggers / transaction rollback / encoding.\n", resource, consumedTotal, consumedTotal-extractFailureTotal) + } else { + fmt.Fprintf(syncEvents, `{"event":"sync_anomaly","resource":"%s","consumed":%d,"stored":0,"extract_failures":%d,"reason":"stored_count_zero_after_extraction"}`+"\n", resource, consumedTotal, extractFailureTotal) + } + } + + if !humanFriendly { + fmt.Fprintf(syncEvents, `{"event":"sync_complete","resource":"%s","total":%d,"duration_ms":%d}`+"\n", resource, totalCount, time.Since(started).Milliseconds()) + } + + return syncResult{Resource: resource, Count: totalCount, Duration: time.Since(started)} +} + +// paginationDefaults holds the resolved pagination parameter names and page size. +type paginationDefaults struct { + cursorParam string + cursorType string // paginator class: "", "cursor", "page_token", "offset", "page" + limitParam string + limit int +} + +// determinePaginationDefaults returns the pagination parameter names to use. +// Values are detected from the API spec by the profiler at generation time. +func determinePaginationDefaults() paginationDefaults { + return paginationDefaults{ + cursorParam: "after", + cursorType: "", + limitParam: "limit", + limit: 100, + } +} + +func resourceSupportsPagination(resource string) bool { + switch resource { + case "ledger": + return true + case "mirror": + return true + } + return false +} + +// syncResourceSinceParam returns the query parameter name this resource's +// list endpoint declares for incremental temporal filtering, or "" when the +// endpoint declares none. Skipping the param for "" resources avoids +// validation-error 400s on APIs that reject unknown query keys. +func syncResourceSinceParam(resource string) string { + switch resource { + case "ledger": + return "since" + } + return "" +} + +func syncResourceSinceParamFormat(resource string) string { + switch resource { + } + return "" +} + +func formatSyncSinceValue(value string, paramFormat string) string { + if strings.EqualFold(paramFormat, "date") { + if ts, err := time.Parse(time.RFC3339, value); err == nil { + return ts.Format("2006-01-02") + } + if ts, err := time.Parse(time.RFC3339Nano, value); err == nil { + return ts.Format("2006-01-02") + } + if _, err := time.Parse("2006-01-02", value); err == nil { + return value + } + } + return value +} + +// extractPageItems attempts to extract an array of items and pagination cursor from a response. +// It tries multiple strategies: +// 1. Direct JSON array +// 2. Common wrapper keys: "data", "results", "items", "records", "nodes", "entries" +// 3. JSend-style nested data envelopes: {"data":{"<resource>":[...]}} +// It also extracts the next cursor from common response fields. +func extractPageItems(data json.RawMessage, cursorParam string) ([]json.RawMessage, string, bool) { + // Strategy 1: direct array + var items []json.RawMessage + if err := json.Unmarshal(data, &items); err == nil { + return items, "", false + } + + // Strategy 2: object with known wrapper keys + var envelope map[string]json.RawMessage + if err := json.Unmarshal(data, &envelope); err != nil { + return nil, "", false + } + + if items, ok := extractItemsByKnownKeys(envelope); ok { + nextCursor, hasMore := extractPaginationFromEnvelope(envelope, cursorParam) + return items, nextCursor, hasMore + } + + for _, key := range dataEnvelopeKeys { + raw, ok := envelope[key] + if !ok { + continue + } + var inner map[string]json.RawMessage + if json.Unmarshal(raw, &inner) != nil { + continue + } + if items, ok := extractItemsFromEnvelope(inner); ok { + nextCursor, hasMore := extractPaginationFromEnvelope(inner, cursorParam) + outerCursor, outerHasMore := extractPaginationFromEnvelope(envelope, cursorParam) + if nextCursor == "" { + nextCursor = outerCursor + } + hasMore = hasMore || outerHasMore + return items, nextCursor, hasMore + } + } + + if items, ok := extractSingleObjectArraySibling(envelope); ok { + nextCursor, hasMore := extractPaginationFromEnvelope(envelope, cursorParam) + return items, nextCursor, hasMore + } + + return nil, "", false +} + +func extractItemsFromEnvelope(envelope map[string]json.RawMessage) ([]json.RawMessage, bool) { + if items, ok := extractItemsByKnownKeys(envelope); ok { + return items, true + } + return extractSingleObjectArraySibling(envelope) +} + +func extractItemsByKnownKeys(envelope map[string]json.RawMessage) ([]json.RawMessage, bool) { + for _, key := range pageItemKeys { + if raw, ok := envelope[key]; ok { + if items, ok := extractObjectArray(raw); ok { + return items, true + } + } + } + return nil, false +} + +func extractSingleObjectArraySibling(envelope map[string]json.RawMessage) ([]json.RawMessage, bool) { + // Fallback: try every key in the envelope. If exactly one maps to a JSON + // array with items, use it. This handles APIs that wrap responses with the + // resource name alongside arbitrary scalar metadata + // (e.g., {"markets": [...], "request_id": "..."}). + var arrayItems []json.RawMessage + arrayCount := 0 + for key, raw := range envelope { + if pageMetadataArrayKeys[key] { + continue + } + if candidate, ok := extractObjectArray(raw); ok { + arrayItems = candidate + arrayCount++ + continue + } + var rawArray []json.RawMessage + if json.Unmarshal(raw, &rawArray) == nil && !isJSONNull(raw) { + continue + } + } + if arrayCount == 1 { + return arrayItems, true + } + return nil, false +} + +func extractObjectArray(raw json.RawMessage) ([]json.RawMessage, bool) { + var items []json.RawMessage + if err := json.Unmarshal(raw, &items); err != nil || len(items) == 0 { + return nil, false + } + var obj map[string]json.RawMessage + if err := json.Unmarshal(items[0], &obj); err != nil { + return nil, false + } + return items, true +} + +// isDryRunResponse detects the `{"dry_run": true}` sentinel that +// client.dryRun returns instead of a real API response. The sync loop +// uses this to short-circuit before the upsert path, which would +// otherwise fail with "missing id for <resource>" against a sentinel +// that has no items and no id. The check requires exactly one key +// (dry_run) so a live API response that happens to include a top-level +// dry_run field alongside real data isn't misclassified as the +// sentinel and silently zero out the sync count. +func isDryRunResponse(data json.RawMessage) bool { + var envelope map[string]json.RawMessage + if err := json.Unmarshal(data, &envelope); err != nil { + return false + } + if len(envelope) != 1 { + return false + } + raw, ok := envelope["dry_run"] + if !ok { + return false + } + var v bool + return json.Unmarshal(raw, &v) == nil && v +} + +func isEmptyPageResponse(data json.RawMessage) bool { + var direct []json.RawMessage + if err := json.Unmarshal(data, &direct); err == nil && !isJSONNull(data) { + return len(direct) == 0 + } + + var envelope map[string]json.RawMessage + if err := json.Unmarshal(data, &envelope); err != nil { + return false + } + + if isEmptyPageEnvelope(envelope) { + return true + } + + for _, key := range dataEnvelopeKeys { + raw, ok := envelope[key] + if !ok { + continue + } + if isJSONNull(raw) { + if envelopeReportsFailure(envelope) { + return true + } + continue + } + var inner map[string]json.RawMessage + if json.Unmarshal(raw, &inner) == nil && isEmptyPageEnvelope(inner) { + return true + } + } + + return false +} + +func envelopeReportsFailure(envelope map[string]json.RawMessage) bool { + for _, key := range []string{"success", "Success"} { + if raw, ok := envelope[key]; ok { + var success bool + if json.Unmarshal(raw, &success) == nil { + return !success + } + } + } + for _, key := range []string{"status", "Status"} { + if raw, ok := envelope[key]; ok { + var status string + if json.Unmarshal(raw, &status) == nil { + switch strings.ToLower(status) { + case "error", "fail", "failed": + return true + } + return false + } + } + } + return false +} + +func isEmptyPageEnvelope(envelope map[string]json.RawMessage) bool { + for _, key := range pageItemKeys { + if raw, ok := envelope[key]; ok { + var items []json.RawMessage + if err := json.Unmarshal(raw, &items); err == nil && !isJSONNull(raw) { + if len(items) > 0 { + var obj map[string]json.RawMessage + if err := json.Unmarshal(items[0], &obj); err != nil { + continue + } + } + return len(items) == 0 + } + } + } + if hasExactlyOneNullArrayWithZeroCount(envelope) { + return true + } + return hasExactlyOneEmptyArray(envelope) +} + +func hasExactlyOneNullArrayWithZeroCount(envelope map[string]json.RawMessage) bool { + nullArrayCount := 0 + hasZeroCount := false + for key, raw := range envelope { + if isZeroCountField(key, raw) { + hasZeroCount = true + continue + } + if isJSONNull(raw) && isNullPageItemCandidateKey(key) { + nullArrayCount++ + continue + } + if pageEnvelopeMetadataKeys[key] { + continue + } + return false + } + return nullArrayCount == 1 && hasZeroCount +} + +func isZeroCountField(key string, raw json.RawMessage) bool { + switch key { + case "total", "Total", "count", "Count", "total_count", "totalCount", "TotalCount": + default: + return false + } + // A JSON null count is not a numeric zero-count signal: json.Unmarshal of + // "null" into a float64 is a no-op (leaves n at 0, returns nil), so without + // this guard a null count would falsely qualify as zero — masking a + // possibly-malformed {"items":null,"total":null} as an empty page. + if isJSONNull(raw) { + return false + } + var n float64 + return json.Unmarshal(raw, &n) == nil && n == 0 +} + +func isNullPageItemCandidateKey(key string) bool { + for _, itemKey := range pageItemKeys { + if key == itemKey { + return true + } + } + return !pageEnvelopeMetadataKeys[key] +} + +func hasExactlyOneEmptyArray(envelope map[string]json.RawMessage) bool { + arrayCount := 0 + for _, raw := range envelope { + var candidate []json.RawMessage + if err := json.Unmarshal(raw, &candidate); err == nil && !isJSONNull(raw) { + // Skip candidate arrays that contain primitives (not objects). + if len(candidate) > 0 { + var obj map[string]json.RawMessage + if err := json.Unmarshal(candidate[0], &obj); err != nil { + continue + } + } + if len(candidate) == 0 { + arrayCount++ + } + } + } + return arrayCount == 1 +} + +func isJSONNull(raw json.RawMessage) bool { + return strings.TrimSpace(string(raw)) == "null" +} + +func isJSONResponse(data json.RawMessage) bool { + var probe any + return json.Unmarshal(data, &probe) == nil +} + +// extractPaginationFromEnvelope extracts cursor and has_more from a response envelope. +func extractPaginationFromEnvelope(envelope map[string]json.RawMessage, cursorParam string) (string, bool) { + var hasMore bool + + nextCursor := nextCursorFromLinks(envelope, cursorParam) + + // Try common cursor field names + cursorKeys := []string{ + "next_cursor", "nextCursor", "next_token", "nextToken", "cursor", + "next_page_token", "nextPageToken", "page_token", "after", "end_cursor", "endCursor", + } + if nextCursor == "" { + nextCursor = findCursorInMap(envelope, cursorKeys) + } + + // If no top-level cursor was found, look one level deeper into well-known + // pagination wrapper objects. Slack returns {"messages":[...], + // "response_metadata":{"next_cursor":"..."}}; Pipedrive uses + // "additional_data"; MongoDB Atlas uses "pagination"; many APIs use + // "meta" or "paging". Purely additive — only + // runs when the top-level scan returned empty — and uses the same + // cursorKeys set so wrapper contents go through the same name match. + if nextCursor == "" { + paginationWrapperKeys := []string{"response_metadata", "additional_data", "pagination", "meta", "paging"} + for _, wrapperKey := range paginationWrapperKeys { + rawWrapper, ok := envelope[wrapperKey] + if !ok { + continue + } + var inner map[string]json.RawMessage + if json.Unmarshal(rawWrapper, &inner) != nil { + continue + } + if c := findCursorInMap(inner, cursorKeys); c != "" { + nextCursor = c + break + } + } + } + + if nextCursor == "" { + nextCursor = nextCursorFromTopLevelURL(envelope, cursorParam) + } + + // Try common has_more field names + hasMoreKeys := []string{"has_more", "hasMore", "has_next", "hasNext", "next_page"} + for _, key := range hasMoreKeys { + if raw, ok := envelope[key]; ok { + if err := json.Unmarshal(raw, &hasMore); err == nil { + break + } + } + } + + // If we found a cursor, assume there are more pages even without explicit has_more + if nextCursor != "" && !hasMore { + hasMore = true + } + + return nextCursor, hasMore +} + +func pageAllowsPageIntFallback(data json.RawMessage) bool { + return pageMayHaveMore(data) +} + +func pageMayHaveMore(data json.RawMessage) bool { + hasMore, parsed := pageExplicitHasMore(data) + return !parsed || hasMore +} + +func pageExplicitHasMore(data json.RawMessage) (bool, bool) { + var envelope map[string]json.RawMessage + if err := json.Unmarshal(data, &envelope); err != nil { + return false, false + } + if hasMore, parsed := envelopeExplicitHasMore(envelope); parsed { + return hasMore, true + } + for _, key := range dataEnvelopeKeys { + raw, ok := envelope[key] + if !ok { + continue + } + var inner map[string]json.RawMessage + if json.Unmarshal(raw, &inner) == nil { + if hasMore, parsed := envelopeExplicitHasMore(inner); parsed { + return hasMore, true + } + } + } + return false, false +} + +func envelopeExplicitHasMore(envelope map[string]json.RawMessage) (bool, bool) { + for _, key := range []string{"has_more", "hasMore", "has_next", "hasNext", "next_page"} { + raw, ok := envelope[key] + if !ok { + continue + } + var hasMore bool + if json.Unmarshal(raw, &hasMore) == nil { + return hasMore, true + } + } + return false, false +} + +// nextCursorFromLinks extracts JSON:API-style pagination cursors from +// {"links":{"next":"https://example.com/items?page[cursor]=..."}}. +func nextCursorFromLinks(envelope map[string]json.RawMessage, cursorParam string) string { + rawLinks, ok := envelope["links"] + if !ok { + return "" + } + var links map[string]json.RawMessage + if json.Unmarshal(rawLinks, &links) != nil { + return "" + } + rawNext, ok := links["next"] + if !ok { + return "" + } + var nextURL string + if json.Unmarshal(rawNext, &nextURL) != nil || nextURL == "" { + return "" + } + + return cursorFromNextURL(nextURL, cursorParam) +} + +// nextCursorFromTopLevelURL extracts a cursor from top-level absolute or relative next URLs. +func nextCursorFromTopLevelURL(envelope map[string]json.RawMessage, cursorParam string) string { + for _, key := range []string{"next", "next_url"} { + rawNext, ok := envelope[key] + if !ok { + continue + } + var nextURL string + if json.Unmarshal(rawNext, &nextURL) != nil || nextURL == "" { + continue + } + if isFollowableNextURL(nextURL) { + return cursorFromNextURL(nextURL, cursorParam) + } + } + return "" +} + +// isFollowableNextURL reports whether a top-level "next" string is a URL we can +// pull a cursor query param from: an absolute http(s) URL, a root-relative path, +// or any value carrying a query string. Bare opaque cursor tokens (no query) +// are rejected so they aren't mis-parsed. +func isFollowableNextURL(nextURL string) bool { + lower := strings.ToLower(nextURL) + return strings.HasPrefix(lower, "http") || + strings.HasPrefix(nextURL, "/") || + strings.Contains(nextURL, "?") +} + +func cursorFromNextURL(nextURL string, cursorParam string) string { + cursorKeys := []string{cursorParam} + if cursorParam != "page[cursor]" { + cursorKeys = append(cursorKeys, "page[cursor]") + } + if cursorParam != "cursor" { + cursorKeys = append(cursorKeys, "cursor") + } + if cursorParam != "after" { + cursorKeys = append(cursorKeys, "after") + } + + parsed, err := url.Parse(nextURL) + if err != nil { + return "" + } + values := parsed.Query() + for _, key := range cursorKeys { + if key == "" { + continue + } + if cursor := values.Get(key); cursor != "" { + return cursor + } + } + return "" +} + +// findCursorInMap returns the first non-empty string-typed value in m +// whose key matches one of cursorKeys. Used by extractPaginationFromEnvelope +// to scan both the top-level envelope and well-known wrapper objects with +// the same name-match rules — extracted so the two scans can't drift. +func findCursorInMap(m map[string]json.RawMessage, cursorKeys []string) string { + for _, key := range cursorKeys { + raw, ok := m[key] + if !ok { + continue + } + var s string + if err := json.Unmarshal(raw, &s); err == nil && s != "" { + return s + } + } + return "" +} + +func emitSyncMissingPaginationCursorWarning(syncEvents io.Writer, humanFriendly bool, resource, parent string) { + if humanFriendly { + if parent != "" { + fmt.Fprintf(os.Stderr, "\nwarning: %s returned a full page for parent %s without a next cursor; data may be truncated.\n", resource, parent) + return + } + fmt.Fprintf(os.Stderr, "\nwarning: %s returned a full page without a next cursor; data may be truncated.\n", resource) + return + } + if parent != "" { + fmt.Fprintf(syncEvents, `{"event":"sync_warning","resource":"%s","parent":"%s","reason":"pagination_cursor_missing","message":"API returned a full page without a usable next cursor; data may be truncated."}`+"\n", resource, parent) + return + } + fmt.Fprintf(syncEvents, `{"event":"sync_warning","resource":"%s","reason":"pagination_cursor_missing","message":"API returned a full page without a usable next cursor; data may be truncated."}`+"\n", resource) +} + +type discriminatorDispatch struct { + Field string + Values map[string]string +} + +var discriminatorDispatchers = map[string]discriminatorDispatch{} + +func upsertResourceBatch(db *store.Store, resource string, items []json.RawMessage) (int, int, error) { + if _, ok := discriminatorDispatchers[resource]; !ok { + return db.UpsertBatch(resource, items) + } + + grouped := map[string][]json.RawMessage{} + order := []string{} + for _, item := range items { + target := resource + if obj, err := store.DecodeJSONObject(item); err == nil { + target = resolveDiscriminatedResource(resource, obj) + } + if _, ok := grouped[target]; !ok { + order = append(order, target) + } + grouped[target] = append(grouped[target], item) + } + + var stored, extractFailures int + for _, target := range order { + targetStored, targetExtractFailures, err := db.UpsertBatch(target, grouped[target]) + if err != nil { + return stored, extractFailures + targetExtractFailures, err + } + stored += targetStored + extractFailures += targetExtractFailures + } + return stored, extractFailures, nil +} + +func resolveDiscriminatedResource(resource string, obj map[string]any) string { + dispatcher, ok := discriminatorDispatchers[resource] + if !ok || dispatcher.Field == "" { + return resource + } + value := store.LookupFieldValue(obj, dispatcher.Field) + if value == nil { + return resource + } + if target, ok := dispatcher.Values[fmt.Sprintf("%v", value)]; ok && target != "" { + return target + } + return resource +} + +// upsertSingleObject stores a non-array API response as a single record. +func upsertSingleObject(db *store.Store, resource string, data json.RawMessage) error { + obj, err := store.DecodeJSONObject(data) + if err != nil { + // Not a JSON object either - store raw under resource name + return db.Upsert(resource, resource, data) + } + + resource = resolveDiscriminatedResource(resource, obj) + + id := extractID(resource, obj) + if id == "" { + id = resource + } + + switch resource { + case "ai": + return db.UpsertAi(data) + case "databases": + return db.UpsertDatabases(data) + case "graph": + return db.UpsertGraph(data) + case "groups": + return db.UpsertGroups(data) + case "ingest": + return db.UpsertIngest(data) + case "ledger": + return db.UpsertLedger(data) + case "mcp": + return db.UpsertMcp(data) + case "media": + return db.UpsertMedia(data) + case "mirror": + return db.UpsertMirror(data) + case "records": + return db.UpsertRecords(data) + case "runtime": + return db.UpsertRuntime(data) + case "selection": + return db.UpsertSelection(data) + case "tags": + return db.UpsertTags(data) + default: + return db.Upsert(resource, id, data) + } +} + +// parseSinceDuration converts human-friendly duration strings into a time.Time. +// Supported formats: "7d" (days), "24h" (hours), "30m" (minutes), "1w" (weeks). +func parseSinceDuration(s string) (time.Time, error) { + re := regexp.MustCompile(`^(\d+)([dhwm])$`) + matches := re.FindStringSubmatch(strings.TrimSpace(s)) + if matches == nil { + return time.Time{}, fmt.Errorf("expected format like 7d, 24h, 1w, or 30m") + } + + n, err := strconv.Atoi(matches[1]) + if err != nil { + return time.Time{}, err + } + + now := time.Now() + switch matches[2] { + case "d": + return now.Add(-time.Duration(n) * 24 * time.Hour), nil + case "h": + return now.Add(-time.Duration(n) * time.Hour), nil + case "w": + return now.Add(-time.Duration(n) * 7 * 24 * time.Hour), nil + case "m": + return now.Add(-time.Duration(n) * time.Minute), nil + default: + return time.Time{}, fmt.Errorf("unknown unit %q", matches[2]) + } +} + +func defaultSyncResources() []string { + return []string{ + "databases", + "ledger", + "mirror", + "records", + "selection", + } +} + +// knownSyncResourceNames returns every resource name sync will accept — +// flat resources plus any parent-child dependents. Used by --resource-param +// validation to reject misspellings before they become silent no-ops. +func knownSyncResourceNames() []string { + names := []string{ + "databases", + "ledger", + "mcp", + "mirror", + "records", + "selection", + "selection-snapshot", + } + return names +} + +// syncResourcePath maps resource names to their actual API endpoint paths. +// For REST APIs this is typically "/<resource>". For non-REST APIs (e.g., Steam) +// this preserves the actual endpoint path like "/ISteamApps/GetAppList/v2". +func syncResourcePath(resource string) (string, error) { + paths := map[string]string{ + "databases": "/databases", + "ledger": "/ledger", + "mcp": "/mcp/tools", + "mirror": "/mirror/search", + "records": "/records/lookup", + "selection": "/selection", + "selection-snapshot": "/selection/snapshot", + } + if p, ok := paths[resource]; ok { + return p, nil + } + return "", fmt.Errorf("unknown sync resource %q", resource) +} + +// resourceIDFieldOverrides projects per-resource IDField (set by the profiler +// from x-resource-id or the response-schema fallback chain) into a runtime +// lookup map. extractID consults this first so the templated path wins over +// the generic fallback list; the generic list applies only when the override +// is empty or the override field is absent on a given item. +// +// Includes both flat resources and dependent (parent-child) resources so +// annotations on a child path-item are honored at runtime, not just on +// flat paths. +var resourceIDFieldOverrides = map[string]string{} + +// genericIDFieldFallbacks is the runtime safety net for resources that did +// NOT receive a templated IDField. API-specific names belong in spec +// annotations (x-resource-id), not this list. Order matters: vendor +// identifier names (gid, sid, uid, uuid, guid) take precedence over `name` +// so APIs like Asana (gid) and Twilio (sid) don't fall through to a display +// field and upsert on names — see #1394. +var genericIDFieldFallbacks = []string{"id", "ID", "gid", "sid", "uid", "uuid", "guid", "name", "slug", "key", "code"} + +// pageItemKeys is scanned in priority order; lowercase REST-convention keys +// come first, PascalCase .NET variants second. Without the PascalCase row, +// {"Items": [...]} envelopes fall through to the ambiguity scan and a +// single-array sibling miscount silently truncates sync. +var pageItemKeys = []string{ + "data", "results", "items", "records", "nodes", "entries", "features", + "Data", "Results", "Items", "Records", "Nodes", "Entries", "Features", +} + +var dataEnvelopeKeys = []string{"data", "Data", "result", "Result"} + +var pageMetadataArrayKeys = map[string]bool{ + "errors": true, "Errors": true, + "warnings": true, "Warnings": true, +} + +var pageEnvelopeMetadataKeys = map[string]bool{ + // list wrappers themselves + "data": true, "results": true, "items": true, + "Data": true, "Results": true, "Items": true, + // pagination cursors / tokens + "next_cursor": true, "nextCursor": true, "NextCursor": true, + "next_token": true, "nextToken": true, "NextToken": true, + "next_page_token": true, "nextPageToken": true, "NextPageToken": true, + "page_token": true, "pageToken": true, "PageToken": true, + "end_cursor": true, "endCursor": true, "EndCursor": true, + "start_cursor": true, "startCursor": true, "StartCursor": true, + "cursor": true, "Cursor": true, "after": true, "After": true, "before": true, "Before": true, + // has-more flags and page numbers + "has_more": true, "hasMore": true, "HasMore": true, + "has_next": true, "hasNext": true, "HasNext": true, + "next_page": true, "nextPage": true, "NextPage": true, + "previous_page": true, "previousPage": true, "PreviousPage": true, + "page": true, "Page": true, "page_size": true, "pageSize": true, "PageSize": true, + "per_page": true, "perPage": true, "PerPage": true, + // counts / totals + "total": true, "Total": true, "count": true, "Count": true, "size": true, "Size": true, + "total_count": true, "totalCount": true, "TotalCount": true, + // JSend / common status envelopes + "success": true, "status": true, "message": true, "error": true, "errors": true, + "warnings": true, "Warnings": true, "ok": true, "Ok": true, + // wrapper objects + "links": true, "meta": true, "pagination": true, + "response_metadata": true, "paging": true, + // links shape + "next": true, "prev": true, "previous": true, "first": true, "last": true, +} + +// criticalResources is the template-time projection of per-resource Critical +// (set by the profiler from the spec's path-item x-critical extension). It +// is consulted at error-aggregation time so a non-critical failure can be +// downgraded to a sync_warning + exit 0 unless --strict was passed. +// +// Includes both flat resources and dependent (parent-child) resources so a +// failed child sync flagged x-critical: true exits non-zero just like a +// flat-resource critical failure. +var criticalResources = map[string]bool{} + +// extractID resolves an item's primary-key field. It consults the +// per-resource templated override first; on miss, it falls through to the +// generic fallback list. resource may be empty for callers that don't have +// a resource context (only the generic list applies in that case). +// +// Field lookups go through store.LookupFieldValue so snake_case overrides +// match camelCase JSON renderings. UpsertBatch resolves fields the same +// way — divergence between the two paths produces silent drops on +// heterogeneous payloads. +func extractID(resource string, obj map[string]any) string { + if override, ok := resourceIDFieldOverrides[resource]; ok && override != "" { + if v := store.LookupFieldValue(obj, override); v != nil { + s := store.ResourceIDString(v) + if s != "" && s != "<nil>" { + return s + } + } + } + for _, key := range genericIDFieldFallbacks { + if v := store.LookupFieldValue(obj, key); v != nil { + s := store.ResourceIDString(v) + if s != "" && s != "<nil>" { + return s + } + } + } + if s := suffixIDFieldFallback(resource, obj); s != "" { + return s + } + return "" +} + +// suffixIDFieldFallback resolves an id-less resource that keys on its own +// "<name>_code" / "<name>_id" / "<name>_key" / "<name>_slug" field (e.g. the +// "currencies" resource keying on "currency_code" — see #2327). It is scoped to +// the resource's OWN name so a foreign key like account_id/parent_id is never +// promoted to the primary key, and it uses direct map lookups in a fixed suffix +// order so the chosen id is deterministic. +func suffixIDFieldFallback(resourceType string, obj map[string]any) string { + for _, base := range resourceIDBaseNames(resourceType) { + for _, suffix := range []string{"_id", "_code", "_key", "_slug"} { + if v, ok := obj[base+suffix]; ok { + if s := scalarIDString(v); s != "" && s != "<nil>" { + return s + } + } + } + } + return "" +} + +// resourceIDBaseNames returns lowercase candidate singular/plural stems of a +// resource name to build "<base>_id"-style key probes from (e.g. "currencies" +// -> ["currencies","currency"]). OpenAPI-/path-derived names can carry a +// leading verb token ("get-currencies"), so the same probes are also attempted +// on the de-verbed stem. Minimal English depluralization; the raw name is +// always included so already-singular names work too. +func resourceIDBaseNames(resourceType string) []string { + r := strings.ToLower(strings.TrimSpace(resourceType)) + if r == "" { + return nil + } + stems := []string{r} + if d := stripLeadingResourceVerb(r); d != "" && d != r { + stems = append(stems, d) + } + var bases []string + seen := map[string]bool{} + add := func(s string) { + if s != "" && !seen[s] { + seen[s] = true + bases = append(bases, s) + } + } + for _, stem := range stems { + add(stem) + add(depluralizeResourceStem(stem)) + } + return bases +} + +func stripLeadingResourceVerb(r string) string { + for _, verb := range []string{"get", "list", "fetch", "find", "retrieve", "read", "show", "all"} { + for _, sep := range []string{"-", "_"} { + prefix := verb + sep + if strings.HasPrefix(r, prefix) && len(r) > len(prefix) { + return r[len(prefix):] + } + } + } + return "" +} + +func depluralizeResourceStem(r string) string { + switch { + case strings.HasSuffix(r, "ies") && len(r) > 3: + return strings.TrimSuffix(r, "ies") + "y" // currencies -> currency + // Plurals formed by adding "es" to a base ending in s/x/z/ch/sh. The + // double-s "sses" guard (not bare "ses") keeps soft-e plurals — where the + // singular already ends in a silent "e" (cases, databases, licenses, + // purchases) — out of this branch; they fall through to the "-s" case below + // (cases -> case, not cas). Trade-off: a genuine "-es" plural of an s-ending + // singular (buses, statuses) depluralizes imperfectly, but those are rare as + // resource names and this stem only feeds best-effort id-field probing. + case strings.HasSuffix(r, "sses") || strings.HasSuffix(r, "xes") || + strings.HasSuffix(r, "zes") || strings.HasSuffix(r, "ches") || + strings.HasSuffix(r, "shes"): + return strings.TrimSuffix(r, "es") // classes -> class, boxes -> box, dishes -> dish + case strings.HasSuffix(r, "s") && !strings.HasSuffix(r, "ss") && len(r) > 1: + return strings.TrimSuffix(r, "s") // languages -> language, cases -> case + } + return r +} + +func scalarIDString(value any) string { + switch value.(type) { + case string, bool, int, int8, int16, int32, int64, + uint, uint8, uint16, uint32, uint64, + float32, float64, json.Number, []byte: + return store.ResourceIDString(value) + default: + return "" + } +} diff --git a/library/productivity/devonthink/internal/cli/sync_hint.go b/library/productivity/devonthink/internal/cli/sync_hint.go new file mode 100644 index 0000000000..c192adf7fe --- /dev/null +++ b/library/productivity/devonthink/internal/cli/sync_hint.go @@ -0,0 +1,123 @@ +// Copyright 2026 rowdy and contributors. Licensed under Apache-2.0. See LICENSE. +// Generated by CLI Printing Press (https://github.com/mvanhorn/cli-printing-press). DO NOT EDIT. + +package cli + +import ( + "database/sql" + "errors" + "fmt" + "io" + "strings" + "time" + + "github.com/mvanhorn/printing-press-library/library/productivity/devonthink/internal/store" + "github.com/spf13/cobra" +) + +type syncHintState struct { + hasState bool + lastSynced time.Time +} + +func maybeEmitSyncHints(cmd *cobra.Command, db *store.Store, resourceType string, maxAge time.Duration) { + if cmd == nil { + return + } + emitSyncHints(cmd.ErrOrStderr(), db, resourceType, maxAge) +} + +func emitSyncHints(w io.Writer, db *store.Store, resourceType string, maxAge time.Duration) { + state, err := readSyncHintState(db, resourceType) + if err != nil || w == nil { + return + } + if !state.hasState { + fmt.Fprintf(w, "hint: local store has not been synced yet. Run 'devonthink-pp-cli sync' before trusting local results.\n") + return + } + if maxAge <= 0 { + return + } + age := time.Since(state.lastSynced) + if age <= maxAge { + return + } + fmt.Fprintf(w, "hint: local store data is %s old, older than --max-age=%s. Run 'devonthink-pp-cli sync' to refresh.\n", syncHintRoundAge(age), maxAge) +} + +func hintIfUnsynced(cmd *cobra.Command, db *store.Store, resourceType string) bool { + if cmd == nil || db == nil { + return false + } + state, err := readSyncHintState(db, resourceType) + if err != nil || state.hasState { + return false + } + fmt.Fprintf(cmd.ErrOrStderr(), "hint: local store has not been synced yet. Run 'devonthink-pp-cli sync' before trusting local results.\n") + return true +} + +func hintIfStale(cmd *cobra.Command, db *store.Store, resourceType string, maxAge time.Duration) bool { + if cmd == nil || db == nil || maxAge <= 0 { + return false + } + state, err := readSyncHintState(db, resourceType) + if err != nil || !state.hasState { + return false + } + age := time.Since(state.lastSynced) + if age <= maxAge { + return false + } + fmt.Fprintf(cmd.ErrOrStderr(), "hint: local store data is %s old, older than --max-age=%s. Run 'devonthink-pp-cli sync' to refresh.\n", syncHintRoundAge(age), maxAge) + return true +} + +func readSyncHintState(db *store.Store, resourceType string) (syncHintState, error) { + if db == nil { + return syncHintState{}, nil + } + query := `SELECT last_synced_at FROM sync_state` + args := []any{} + if strings.TrimSpace(resourceType) != "" { + query += ` WHERE resource_type = ?` + args = append(args, resourceType) + } else { + query += ` WHERE last_synced_at IS NOT NULL` + } + query += ` ORDER BY last_synced_at ASC LIMIT 1` + + var lastSynced sql.NullTime + err := db.DB().QueryRow(query, args...).Scan(&lastSynced) + if err == nil { + if !lastSynced.Valid { + return syncHintState{}, nil + } + return syncHintState{ + hasState: true, + lastSynced: lastSynced.Time, + }, nil + } + if errors.Is(err, sql.ErrNoRows) || syncHintMissingTable(err) { + return syncHintState{}, nil + } + return syncHintState{}, err +} + +func syncHintMissingTable(err error) bool { + for err != nil { + if strings.Contains(err.Error(), "no such table") { + return true + } + err = errors.Unwrap(err) + } + return false +} + +func syncHintRoundAge(age time.Duration) time.Duration { + if age < time.Minute { + return age.Round(time.Second) + } + return age.Round(time.Minute) +} diff --git a/library/productivity/devonthink/internal/cli/sync_hint_test.go b/library/productivity/devonthink/internal/cli/sync_hint_test.go new file mode 100644 index 0000000000..ebabc2cc84 --- /dev/null +++ b/library/productivity/devonthink/internal/cli/sync_hint_test.go @@ -0,0 +1,175 @@ +// Copyright 2026 rowdy and contributors. Licensed under Apache-2.0. See LICENSE. +// Generated by CLI Printing Press (https://github.com/mvanhorn/cli-printing-press). DO NOT EDIT. + +package cli + +import ( + "bytes" + "context" + "path/filepath" + "strings" + "testing" + "time" + + "github.com/mvanhorn/printing-press-library/library/productivity/devonthink/internal/store" + "github.com/spf13/cobra" +) + +func newSyncHintTestStore(t *testing.T) *store.Store { + t.Helper() + db, err := store.OpenWithContext(context.Background(), filepath.Join(t.TempDir(), "data.db")) + if err != nil { + t.Fatalf("open store: %v", err) + } + t.Cleanup(func() { db.Close() }) + return db +} + +func newSyncHintTestCmd() (*cobra.Command, *bytes.Buffer) { + var stderr bytes.Buffer + cmd := &cobra.Command{Use: "devonthink-pp-cli"} + cmd.SetErr(&stderr) + return cmd, &stderr +} + +func TestHintIfUnsynced_EmptySyncStateWritesHintToStderr(t *testing.T) { + db := newSyncHintTestStore(t) + cmd, stderr := newSyncHintTestCmd() + + if !hintIfUnsynced(cmd, db, "") { + t.Fatalf("hintIfUnsynced returned false for empty sync_state") + } + if got := stderr.String(); !strings.Contains(got, "Run 'devonthink-pp-cli sync'") { + t.Fatalf("stderr = %q, want sync hint", got) + } +} + +func TestHintIfStale_BackdatedSyncStateWritesHintToStderr(t *testing.T) { + db := newSyncHintTestStore(t) + if _, err := db.DB().Exec( + `INSERT INTO sync_state(resource_type, last_synced_at, total_count) VALUES (?, ?, ?)`, + "issues", time.Now().Add(-2*time.Hour), 1, + ); err != nil { + t.Fatalf("seed sync_state: %v", err) + } + cmd, stderr := newSyncHintTestCmd() + + if hintIfUnsynced(cmd, db, "") { + t.Fatalf("hintIfUnsynced returned true after sync_state was seeded") + } + if !hintIfStale(cmd, db, "", 30*time.Minute) { + t.Fatalf("hintIfStale returned false for stale sync_state") + } + got := stderr.String() + if !strings.Contains(got, "older than --max-age=30m0s") || !strings.Contains(got, "Run 'devonthink-pp-cli sync'") { + t.Fatalf("stderr = %q, want stale sync hint", got) + } +} + +func TestHintIfStale_MaxAgeZeroDisablesHint(t *testing.T) { + db := newSyncHintTestStore(t) + if _, err := db.DB().Exec( + `INSERT INTO sync_state(resource_type, last_synced_at, total_count) VALUES (?, ?, ?)`, + "issues", time.Now().Add(-2*time.Hour), 1, + ); err != nil { + t.Fatalf("seed sync_state: %v", err) + } + cmd, stderr := newSyncHintTestCmd() + + if hintIfStale(cmd, db, "", 0) { + t.Fatalf("hintIfStale returned true when maxAge is zero") + } + if stderr.Len() != 0 { + t.Fatalf("stderr = %q, want no hint", stderr.String()) + } +} + +func TestHintIfUnsynced_NullTimestampWritesHint(t *testing.T) { + db := newSyncHintTestStore(t) + if _, err := db.DB().Exec( + `INSERT INTO sync_state(resource_type, last_synced_at, total_count) VALUES (?, ?, ?)`, + "issues", nil, 1, + ); err != nil { + t.Fatalf("seed sync_state: %v", err) + } + cmd, stderr := newSyncHintTestCmd() + + if !hintIfUnsynced(cmd, db, "issues") { + t.Fatalf("hintIfUnsynced returned false for null last_synced_at") + } + if got := stderr.String(); !strings.Contains(got, "has not been synced yet") { + t.Fatalf("stderr = %q, want unsynced hint", got) + } +} + +func TestHintIfStale_AllResourcesIgnoresNullTimestampRows(t *testing.T) { + db := newSyncHintTestStore(t) + now := time.Now() + for _, row := range []struct { + resource string + syncedAt any + }{ + {"users", nil}, + {"issues", now.Add(-2 * time.Hour)}, + } { + if _, err := db.DB().Exec( + `INSERT INTO sync_state(resource_type, last_synced_at, total_count) VALUES (?, ?, ?)`, + row.resource, row.syncedAt, 1, + ); err != nil { + t.Fatalf("seed %s sync_state: %v", row.resource, err) + } + } + + cmd, stderr := newSyncHintTestCmd() + if hintIfUnsynced(cmd, db, "") { + t.Fatalf("hintIfUnsynced returned true when a valid sync timestamp exists") + } + if !hintIfStale(cmd, db, "", 30*time.Minute) { + t.Fatalf("hintIfStale returned false for oldest valid all-resource timestamp") + } + if got := stderr.String(); !strings.Contains(got, "older than --max-age=30m0s") { + t.Fatalf("stderr = %q, want stale hint from valid timestamp", got) + } +} + +func TestHintIfStale_ResourceFilterUsesRequestedResource(t *testing.T) { + db := newSyncHintTestStore(t) + now := time.Now() + for _, row := range []struct { + resource string + syncedAt time.Time + }{ + {"users", now.Add(-5 * time.Minute)}, + {"issues", now.Add(-2 * time.Hour)}, + } { + if _, err := db.DB().Exec( + `INSERT INTO sync_state(resource_type, last_synced_at, total_count) VALUES (?, ?, ?)`, + row.resource, row.syncedAt, 1, + ); err != nil { + t.Fatalf("seed %s sync_state: %v", row.resource, err) + } + } + + cmd, stderr := newSyncHintTestCmd() + if hintIfStale(cmd, db, "users", 30*time.Minute) { + t.Fatalf("hintIfStale returned true for fresh users resource") + } + if stderr.Len() != 0 { + t.Fatalf("stderr = %q, want no hint for fresh users resource", stderr.String()) + } + + if !hintIfStale(cmd, db, "issues", 30*time.Minute) { + t.Fatalf("hintIfStale returned false for stale issues resource") + } + if got := stderr.String(); !strings.Contains(got, "older than --max-age=30m0s") { + t.Fatalf("stderr = %q, want stale issues hint", got) + } + + cmd, stderr = newSyncHintTestCmd() + if !hintIfUnsynced(cmd, db, "comments") { + t.Fatalf("hintIfUnsynced returned false for unsynced comments resource") + } + if got := stderr.String(); !strings.Contains(got, "has not been synced yet") { + t.Fatalf("stderr = %q, want unsynced comments hint", got) + } +} diff --git a/library/productivity/devonthink/internal/cli/sync_numeric_id_test.go b/library/productivity/devonthink/internal/cli/sync_numeric_id_test.go new file mode 100644 index 0000000000..714dfc995f --- /dev/null +++ b/library/productivity/devonthink/internal/cli/sync_numeric_id_test.go @@ -0,0 +1,47 @@ +// Copyright 2026 rowdy and contributors. Licensed under Apache-2.0. See LICENSE. +// Generated by CLI Printing Press (https://github.com/mvanhorn/cli-printing-press). DO NOT EDIT. + +package cli + +import ( + "encoding/json" + "path/filepath" + "testing" + + "github.com/mvanhorn/printing-press-library/library/productivity/devonthink/internal/store" +) + +func TestSyncSingleObject_PreservesLargeIntegerResourceIDs(t *testing.T) { + dbPath := filepath.Join(t.TempDir(), "data.db") + db, err := store.Open(dbPath) + if err != nil { + t.Fatalf("open: %v", err) + } + defer db.Close() + + if err := upsertSingleObject(db, "numeric_ids", json.RawMessage(`{"id": 55043301, "name": "large"}`)); err != nil { + t.Fatalf("upsertSingleObject: %v", err) + } + + var got string + if err := db.DB().QueryRow( + `SELECT id FROM resources WHERE resource_type = ?`, + "numeric_ids", + ).Scan(&got); err != nil { + t.Fatalf("query resource id: %v", err) + } + if got != "55043301" { + t.Fatalf("resource id = %q, want %q", got, "55043301") + } + + var literalMatches int + if err := db.DB().QueryRow( + `SELECT COUNT(*) FROM resources WHERE resource_type = ? AND id = '55043301'`, + "numeric_ids", + ).Scan(&literalMatches); err != nil { + t.Fatalf("count literal id matches: %v", err) + } + if literalMatches != 1 { + t.Fatalf("literal id matches = %d, want 1", literalMatches) + } +} diff --git a/library/productivity/devonthink/internal/cli/tail.go b/library/productivity/devonthink/internal/cli/tail.go new file mode 100644 index 0000000000..2d610a6879 --- /dev/null +++ b/library/productivity/devonthink/internal/cli/tail.go @@ -0,0 +1,167 @@ +// Copyright 2026 rowdy and contributors. Licensed under Apache-2.0. See LICENSE. +// Generated by CLI Printing Press (https://github.com/mvanhorn/cli-printing-press). DO NOT EDIT. + +package cli + +import ( + "context" + "encoding/json" + "fmt" + "os" + "os/signal" + "syscall" + "time" + + "github.com/mvanhorn/printing-press-library/library/productivity/devonthink/internal/cliutil" + "github.com/spf13/cobra" +) + +func newTailCmd(flags *rootFlags) *cobra.Command { + var resource string + var interval time.Duration + var follow bool + + cmd := &cobra.Command{ + Use: "tail [resource]", + Short: "Stream live changes by polling the API at regular intervals", + Annotations: map[string]string{"mcp:read-only": "true"}, + Long: `Tail streams live data changes by polling the API at configurable intervals. +Events are emitted as NDJSON to stdout for piping to other tools. +Gracefully shuts down on SIGTERM/SIGINT. + +Note: For APIs with WebSocket or SSE support, a future version will use +native streaming instead of polling.`, + Example: ` # Tail all changes every 10 seconds + devonthink-pp-cli tail --interval 10s + + # Tail a specific resource + devonthink-pp-cli tail messages --interval 5s + + # Pipe to jq for filtering + devonthink-pp-cli tail events --interval 30s | jq 'select(.type == "error")'`, + RunE: func(cmd *cobra.Command, args []string) error { + if len(args) > 0 { + resource = args[0] + } + // JSON help envelope: when called with no resource AND --json, + // surface the list of known resources so agents can discover + // what to pass without parsing a usage error message. + // Envelope: {resources: [...], note}. + if resource == "" && flags.asJSON { + return printJSONFiltered(cmd.OutOrStdout(), map[string]any{ + "resources": tailKnownResources(), + "note": "tail requires a resource name; pass one of the listed names", + }, flags) + } + if resource == "" { + return fmt.Errorf("resource name required (e.g., 'tail messages')") + } + + c, err := flags.newClient() + if err != nil { + return err + } + c.NoCache = true + if cliutil.IsDogfoodEnv() { + follow = false + } + + path := "/" + resource + + sig := make(chan os.Signal, 1) + signal.Notify(sig, syscall.SIGTERM, syscall.SIGINT) + + enc := json.NewEncoder(os.Stdout) + + if follow { + fmt.Fprintf(os.Stderr, "Tailing %s every %s (Ctrl+C to stop)\n", resource, interval) + } + + // Initial fetch + if err := fetchAndEmit(cmd.Context(), c, path, enc); err != nil { + fmt.Fprintf(os.Stderr, "warning: initial fetch failed: %v\n", err) + } + if !follow { + return nil + } + + ticker := time.NewTicker(interval) + defer ticker.Stop() + + for { + select { + case <-sig: + fmt.Fprintln(os.Stderr, "\nShutting down gracefully...") + return nil + case <-ticker.C: + if err := fetchAndEmit(cmd.Context(), c, path, enc); err != nil { + fmt.Fprintf(os.Stderr, "warning: poll failed: %v\n", err) + } + } + } + }, + } + + cmd.Flags().StringVar(&resource, "resource", "", "Resource type to tail") + cmd.Flags().DurationVar(&interval, "interval", 10*time.Second, "Poll interval") + cmd.Flags().BoolVar(&follow, "follow", true, "Keep running (set --follow=false for single poll)") + + return cmd +} + +// tailKnownResources returns the resource names this CLI exposes, so the +// no-arg JSON help envelope can list them without depending on sync's +// defaultSyncResources (which only exists when sync is generated). +func tailKnownResources() []string { + return []string{ + "ai", + "batch", + "context", + "databases", + "graph", + "groups", + "ingest", + "inventory", + "ledger", + "mcp", + "media", + "mirror", + "privacy", + "records", + "runtime", + "selection", + "sheets", + "tags", + } +} + +func fetchAndEmit(ctx context.Context, c interface { + Get(context.Context, string, map[string]string) (json.RawMessage, error) +}, path string, enc *json.Encoder) error { + data, err := c.Get(ctx, path, nil) + if err != nil { + return err + } + + var items []json.RawMessage + if err := json.Unmarshal(data, &items); err != nil { + event := map[string]any{ + "event": "data", + "timestamp": time.Now().UTC().Format(time.RFC3339), + "data": json.RawMessage(data), + } + return enc.Encode(event) + } + + for _, item := range items { + event := map[string]any{ + "event": "data", + "timestamp": time.Now().UTC().Format(time.RFC3339), + "data": item, + } + if err := enc.Encode(event); err != nil { + return err + } + } + return nil +} diff --git a/library/productivity/devonthink/internal/cli/version.go b/library/productivity/devonthink/internal/cli/version.go new file mode 100644 index 0000000000..26cf042168 --- /dev/null +++ b/library/productivity/devonthink/internal/cli/version.go @@ -0,0 +1,25 @@ +// Copyright 2026 rowdy and contributors. Licensed under Apache-2.0. See LICENSE. +// Generated by CLI Printing Press (https://github.com/mvanhorn/cli-printing-press). DO NOT EDIT. + +package cli + +import ( + "fmt" + + "github.com/spf13/cobra" +) + +// version is the printed CLI's version, overridable at build time via ldflags. +var version = "1.0.0" + +// newVersionCmd prints the CLI name and version. Shared by the HTTP and device +// generators so both printed-CLI shapes carry an identical version command. +func newVersionCmd() *cobra.Command { + return &cobra.Command{ + Use: "version", + Short: "Print version", + Run: func(cmd *cobra.Command, args []string) { + fmt.Printf("%s %s\n", cmd.Root().Name(), version) + }, + } +} diff --git a/library/productivity/devonthink/internal/cli/which.go b/library/productivity/devonthink/internal/cli/which.go new file mode 100644 index 0000000000..547edc73bd --- /dev/null +++ b/library/productivity/devonthink/internal/cli/which.go @@ -0,0 +1,222 @@ +// Copyright 2026 rowdy and contributors. Licensed under Apache-2.0. See LICENSE. +// Generated by CLI Printing Press (https://github.com/mvanhorn/cli-printing-press). DO NOT EDIT. + +package cli + +import ( + "fmt" + "sort" + "strings" + + "github.com/spf13/cobra" +) + +// whichEntry is one row of the curated capability index. The index is +// seeded at generation time from the same NovelFeature list that drives +// the SKILL.md feature section, so the command a `which` query returns +// is guaranteed to exist and to match what the skill advertises. +type whichEntry struct { + Command string `json:"command"` + Description string `json:"description"` + Group string `json:"group,omitempty"` + WhyItMatters string `json:"why_it_matters,omitempty"` +} + +// whichIndex is the curated list of capabilities this CLI advertises as +// its hero features. Endpoint-level commands are discoverable via +// `--help`; `which` exists to resolve a natural-language capability +// query to one of the commands the skill says matter most. +var whichIndex = []whichEntry{ + {Command: "context pack", Description: "Build a compact evidence packet from records, selections, highlights, links, and related items.", Group: "Local state that compounds", WhyItMatters: "Use this when an agent needs enough DEVONthink context to reason without dumping whole documents."}, + {Command: "privacy audit", Description: "Preview what a workflow may expose before content leaves the local machine.", Group: "Local-first safety", WhyItMatters: "Use this before sending DEVONthink-derived context to an external model or shared MCP endpoint."}, + {Command: "batch plan", Description: "Stage multi-record edits as validated dry-run plans before applying them.", Group: "Safe automation", WhyItMatters: "Use this for multi-record writes where each target must be checked before mutation."}, + {Command: "inventory export", Description: "Export DEVONthink databases, groups, tags, and document metadata for maintenance plugins.", Group: "Agent-native plumbing", WhyItMatters: "Use this when structure-audit or inbox-triage tooling needs a stable local inventory contract."}, + {Command: "graph audit", Description: "Detect orphans, broken links, unresolved wiki links, weak hubs, and tag-only clusters.", Group: "Local state that compounds", WhyItMatters: "Use this when DEVONthink should behave like a maintained knowledge graph instead of a folder pile."}, + {Command: "mirror search", Description: "Query a local SQLite mirror for repeatable fast analysis without repeated app calls.", Group: "Local state that compounds", WhyItMatters: "Use this for repeated analysis, dashboards, and low-token agent workflows."}, + {Command: "mcp call", Description: "Call DEVONthink's official local MCP tools from scripts when the local MCP server is enabled.", Group: "Agent-native plumbing", WhyItMatters: "Use this when the official MCP exposes a new tool before the CLI has a promoted command."}, + {Command: "ledger list", Description: "Review CLI-driven mutation plans, applies, target proofs, and rollback hints.", Group: "Safe automation", WhyItMatters: "Use this to audit or explain what recent automation did to DEVONthink."}, + {Command: "selection snapshot", Description: "Turn the current GUI selection into a reusable JSON workflow seed.", Group: "Safe automation", WhyItMatters: "Use this when the human has curated records in the GUI and wants an agent-safe handoff."}, + {Command: "agent-context", Description: "Emit an agent contract that enforces local-machine and own-LAN DEVONthink access only.", Group: "Local-first safety", WhyItMatters: "Use this before handing DEVONthink access to an agent that must avoid remote control paths."}, +} + +// whichMatch pairs an index entry with its ranking score for a query. +// Higher score means stronger match. The ranker is naive (exact token +// then substring then group tag) because 20-40 entries do not need +// semantic retrieval - a ranker upgrade is a future change that would +// not break this contract. +type whichMatch struct { + Entry whichEntry `json:"entry"` + Score int `json:"score"` +} + +// rankWhich returns up to `limit` best matches for `query` against the +// index, sorted by descending score. Score breakdown: +// +// +3 exact token match on the command's leaf or full path +// +2 substring match on the command (any part) +// +2 substring match on the description +// +1 group tag contains the query as a word +// +// Ties break on declaration order in the index. An empty query returns +// every entry at score 0 in declaration order - this is the "list all" +// behavior the skill documents for broad agent discovery. +func rankWhich(index []whichEntry, query string, limit int) []whichMatch { + if limit <= 0 { + limit = 3 + } + q := strings.ToLower(strings.TrimSpace(query)) + if q == "" { + out := make([]whichMatch, 0, len(index)) + for _, e := range index { + out = append(out, whichMatch{Entry: e, Score: 0}) + } + return out + } + qTokens := strings.Fields(q) + + scored := make([]whichMatch, 0, len(index)) + for i, e := range index { + score := whichScoreEntry(e, q, qTokens) + scored = append(scored, whichMatch{Entry: e, Score: score}) + _ = i + } + + sort.SliceStable(scored, func(i, j int) bool { + return scored[i].Score > scored[j].Score + }) + // Drop zero-score matches when the query was non-empty; agents + // branching on exit code rely on "no match" meaning no confidence. + filtered := scored[:0] + for _, m := range scored { + if m.Score > 0 { + filtered = append(filtered, m) + } + } + if len(filtered) > limit { + filtered = filtered[:limit] + } + return filtered +} + +func whichScoreEntry(e whichEntry, query string, qTokens []string) int { + score := 0 + cmd := strings.ToLower(e.Command) + cmdTokens := strings.Fields(cmd) + desc := strings.ToLower(e.Description) + group := strings.ToLower(e.Group) + + // Exact token match on the command path (any token). + for _, qt := range qTokens { + for _, ct := range cmdTokens { + if qt == ct { + score += 3 + break + } + } + } + // Substring match on the full command (covers hyphenated leaves). + if strings.Contains(cmd, query) { + score += 2 + } + // Substring match on the description. + if strings.Contains(desc, query) { + score += 2 + } + // Group tag match. + if group != "" { + for _, qt := range qTokens { + if strings.Contains(group, qt) { + score += 1 + break + } + } + } + return score +} + +func newWhichCmd(flags *rootFlags) *cobra.Command { + var limit int + cmd := &cobra.Command{ + Use: "which [query]", + Short: "Find the command that implements a capability", + Annotations: map[string]string{ + "mcp:read-only": "true", + "pp:typed-exit-codes": "0,2", + }, + Long: `which resolves a natural-language capability query (for example, "search messages" or "stale tickets") to the best matching command from this CLI's curated feature index. + +Exit codes: + 0 at least one match found + 2 no confident match - the query did not score against any indexed capability; fall back to '--help' or 'search' if this CLI has one`, + Example: ` devonthink-pp-cli which "stale tickets" + devonthink-pp-cli which "bottleneck" + devonthink-pp-cli which --limit 1 "send message" + devonthink-pp-cli which # list the full capability index`, + RunE: func(cmd *cobra.Command, args []string) error { + if len(whichIndex) == 0 { + return usageErr(fmt.Errorf("this CLI has no curated capability index; run '--help' to see every command")) + } + query := strings.Join(args, " ") + matches := rankWhich(whichIndex, query, limit) + + // Empty query returns the whole index at score 0 (listing mode). + if strings.TrimSpace(query) == "" { + return renderWhich(cmd, flags, rankWhichAll(whichIndex)) + } + + if len(matches) == 0 { + // Under --json, return an empty matches envelope at exit 0 + // so agents can branch on `matches.length == 0` instead of + // parsing a usage error message. Non-JSON keeps the typed + // exit-2 path so terminal users see the help hint. + if flags.asJSON { + return printJSONFiltered(cmd.OutOrStdout(), map[string]any{ + "matches": []whichMatch{}, + }, flags) + } + return usageErr(fmt.Errorf("no match for %q; try '%s --help' for the full command list", query, cmd.Root().Name())) + } + return renderWhich(cmd, flags, matches) + }, + } + cmd.Flags().IntVar(&limit, "limit", 3, "Maximum number of matches to return") + return cmd +} + +// rankWhichAll is a narrow helper used by the "empty query lists the +// index" path. It returns every entry in declaration order at score 0 +// so the render path treats them uniformly. +func rankWhichAll(index []whichEntry) []whichMatch { + out := make([]whichMatch, 0, len(index)) + for _, e := range index { + out = append(out, whichMatch{Entry: e, Score: 0}) + } + return out +} + +func renderWhich(cmd *cobra.Command, flags *rootFlags, matches []whichMatch) error { + w := cmd.OutOrStdout() + // Output shape follows the same rule as every other generated + // command: JSON when the caller asked for it OR when stdout is not + // a terminal; table when a human is looking. + asJSON := flags.asJSON + if !asJSON && !isTerminal(w) { + asJSON = true + } + if asJSON { + // JSON envelope: {matches: [...]}. The wrap is critical: + // printJSONFiltered's --compact path uses compactListFields + // (allowlist) for top-level arrays, which would strip + // entry/score keys; routing through compactObjectFields + // (blocklist) via an object envelope preserves them. + if matches == nil { + matches = []whichMatch{} + } + return printJSONFiltered(w, map[string]any{"matches": matches}, flags) + } + fmt.Fprintf(w, "%-24s %-8s %s\n", "COMMAND", "SCORE", "DESCRIPTION") + for _, m := range matches { + fmt.Fprintf(w, "%-24s %-8d %s\n", m.Entry.Command, m.Score, m.Entry.Description) + } + return nil +} diff --git a/library/productivity/devonthink/internal/cli/which_test.go b/library/productivity/devonthink/internal/cli/which_test.go new file mode 100644 index 0000000000..10a5911dbd --- /dev/null +++ b/library/productivity/devonthink/internal/cli/which_test.go @@ -0,0 +1,98 @@ +// Copyright 2026 rowdy and contributors. Licensed under Apache-2.0. See LICENSE. +// Generated by CLI Printing Press (https://github.com/mvanhorn/cli-printing-press). DO NOT EDIT. + +package cli + +import ( + "strings" + "testing" +) + +// Fixture index used across which-ranking tests. Covers a typical mix +// of single-word commands, multi-word commands, and grouped entries so +// the ranker is exercised against shapes a generated CLI actually +// produces. +var whichTestIndex = []whichEntry{ + {Command: "search", Description: "Full-text search across synced resources", Group: "Local state"}, + {Command: "stale", Description: "Find tickets that have not moved in a while", Group: "Local state"}, + {Command: "bottleneck", Description: "Identify pipeline bottlenecks", Group: "Local state"}, + {Command: "send", Description: "Send a message", Group: "Write operations"}, + {Command: "sync", Description: "Sync resources to local SQLite", Group: "Local state"}, +} + +// Happy path: a query that matches a command by keyword returns that +// command first. This is the load-bearing promise of `which`. +func TestRankWhich_ExactTokenMatchWins(t *testing.T) { + got := rankWhich(whichTestIndex, "search", 3) + if len(got) == 0 { + t.Fatalf("expected at least one match, got zero") + } + if got[0].Entry.Command != "search" { + t.Errorf("top match: want search, got %s", got[0].Entry.Command) + } +} + +// Happy path: a query matching the description wins when the command +// itself does not contain the query tokens. +func TestRankWhich_DescriptionMatch(t *testing.T) { + got := rankWhich(whichTestIndex, "bottlenecks", 3) + if len(got) == 0 || got[0].Entry.Command != "bottleneck" { + t.Errorf("expected bottleneck command as top match for bottlenecks query, got %+v", got) + } +} + +// Happy path: a multi-word query resolves to the best single match by +// summing per-token scores. +func TestRankWhich_MultiTokenQuery(t *testing.T) { + got := rankWhich(whichTestIndex, "send a message", 3) + if len(got) == 0 || got[0].Entry.Command != "send" { + t.Errorf("expected send as top match for 'send a message', got %+v", got) + } +} + +// Edge case: empty query should surface the full index (listing mode) +// rather than treating as no-match. Agents use this for broad discovery. +func TestRankWhich_EmptyQueryListsIndex(t *testing.T) { + got := rankWhich(whichTestIndex, "", 3) + if len(got) != len(whichTestIndex) { + t.Errorf("empty query should return all %d entries, got %d", len(whichTestIndex), len(got)) + } + for i, m := range got { + if m.Score != 0 { + t.Errorf("empty query entry %d: score should be 0, got %d", i, m.Score) + } + } +} + +// Edge case: the limit flag caps the result set so agents can ask for +// a single top answer when they want a deterministic branch. +func TestRankWhich_LimitCapsResults(t *testing.T) { + got := rankWhich(whichTestIndex, "local", 1) + if len(got) > 1 { + t.Errorf("limit=1 should return at most 1 match, got %d", len(got)) + } +} + +// No-match path: a query that hits nothing in the index returns an +// empty slice so the caller can exit with the no-match code (2) rather +// than printing a misleading best-effort result. +func TestRankWhich_NoMatchReturnsEmpty(t *testing.T) { + got := rankWhich(whichTestIndex, "nonexistentxyz", 3) + if len(got) != 0 { + t.Errorf("nonsense query should return zero matches, got %d (%+v)", len(got), got) + } +} + +// Sanity: whichIndex compiles and is well-formed. Generated CLIs with +// zero NovelFeatures ship an empty index, and that is still a valid +// state (which returns the "no curated index" error at runtime). +func TestWhichIndex_ExistsAndIsWellFormed(t *testing.T) { + for i, e := range whichIndex { + if e.Command == "" { + t.Errorf("whichIndex[%d] has empty Command - template rendered bad data", i) + } + if strings.TrimSpace(e.Description) == "" { + t.Errorf("whichIndex[%d] (%s) has empty Description - template rendered bad data", i, e.Command) + } + } +} diff --git a/library/productivity/devonthink/internal/client/client.go b/library/productivity/devonthink/internal/client/client.go new file mode 100644 index 0000000000..83839e2fb3 --- /dev/null +++ b/library/productivity/devonthink/internal/client/client.go @@ -0,0 +1,845 @@ +// Copyright 2026 rowdy and contributors. Licensed under Apache-2.0. See LICENSE. +// Generated by CLI Printing Press (https://github.com/mvanhorn/cli-printing-press). DO NOT EDIT. + +package client + +import ( + "bytes" + "context" + "crypto/sha256" + "encoding/base64" + "encoding/hex" + "encoding/json" + "errors" + "fmt" + "github.com/mvanhorn/printing-press-library/library/productivity/devonthink/internal/cliutil" + "github.com/mvanhorn/printing-press-library/library/productivity/devonthink/internal/config" + "io" + "math" + "net/http" + "net/url" + "os" + "path/filepath" + "sort" + "strings" + "time" +) + +const BinaryResponseHeader = "X-Printing-Press-Binary-Response" + +type Client struct { + BaseURL string + Config *config.Config + HTTPClient *http.Client + DryRun bool + NoCache bool + cacheDir string + limiter *cliutil.AdaptiveLimiter +} + +// RequestBaseURL returns the base URL used for requests. +// Novel commands that build request URLs by hand should use this instead of +// concatenating c.BaseURL directly. +func (c *Client) RequestBaseURL() string { + return c.BaseURL +} + +// APIError carries HTTP status information for structured exit codes. +type APIError struct { + Method string + Path string + StatusCode int + Body string +} + +func (e *APIError) Error() string { + return fmt.Sprintf("%s %s returned HTTP %d: %s", e.Method, e.Path, e.StatusCode, e.Body) +} + +func newHTTPClient(timeout time.Duration, jar http.CookieJar) *http.Client { + return &http.Client{Timeout: timeout, Jar: jar} +} + +func New(cfg *config.Config, timeout time.Duration, rateLimit float64) *Client { + homeDir, _ := os.UserHomeDir() + cacheDir := filepath.Join(homeDir, ".cache", "devonthink-pp-cli", "http") + httpClient := newHTTPClient(timeout, nil) + c := &Client{ + BaseURL: strings.TrimRight(cfg.BaseURL, "/"), + Config: cfg, + HTTPClient: httpClient, + cacheDir: cacheDir, + limiter: cliutil.NewAdaptiveLimiter(rateLimit), + } + // CheckRedirect re-derives auth on each hop. Go's default replays the + // original Authorization header verbatim, which breaks nonce-bound + // schemes (OAuth 1.0a PLAINTEXT, SigV4, Hawk): the duplicate nonce + // trips the server's replay detector with a 401. c.authHeader() + // returns a fresh value for those schemes and the same static value + // for Bearer/api_key, so post-redirect headers are byte-identical for + // static auth and freshly-signed for nonce-bound auth. + httpClient.CheckRedirect = func(req *http.Request, via []*http.Request) error { + if len(via) >= 10 { + // Match Go's defaultCheckRedirect: a plain error so Client.Do + // returns it through do()'s err != nil branch. ErrUseLastResponse + // would cause Do to return the 3xx with nil error, which do() + // would then classify as a successful response and hand the HTML + // "Moved Permanently" body back to the caller. + return errors.New("stopped after 10 redirects") + } + // Same-host gate mirrors Go's shouldCopyHeaderOnRedirect: a + // cross-domain 3xx (open redirect or partner handoff) must not + // receive the auth credential, even though we are inside + // CheckRedirect where Go's automatic stripping has already run. + if req.URL.Host == via[0].URL.Host { + if h, err := c.authHeader(req.Context()); err == nil && h != "" { + req.Header.Set("Authorization", h) // #nosec G119 -- authorization is re-added only for same-host redirects. + } + } else { + // Cross-host hop: Go strips standard auth headers (Authorization, + // Cookie) but not custom ones, so a custom API-key header would be + // forwarded verbatim to the redirect target. Delete it explicitly. + req.Header.Del("Authorization") + } + return nil + } + return c +} + +// RateLimit returns the current effective rate limit in req/s. Returns 0 if disabled. +func (c *Client) RateLimit() float64 { + return c.limiter.Rate() +} + +func (c *Client) Get(ctx context.Context, path string, params map[string]string) (json.RawMessage, error) { + return c.GetWithHeaders(ctx, path, params, nil) +} + +func (c *Client) GetWithHeaders(ctx context.Context, path string, params map[string]string, headers map[string]string) (json.RawMessage, error) { + if err := c.validateCachedRequestAuth(ctx); err != nil { + return nil, err + } + binaryResponse := c.wantsBinaryResponse(headers) + cacheEnabled := c.responseCacheEnabled(binaryResponse) + // Check cache for GET requests + if cacheEnabled { + if cached, ok := c.readCache(path, params); ok { + return cached, nil + } + } + result, _, err := c.do(ctx, "GET", path, params, nil, headers) + if err == nil && cacheEnabled { + c.writeCache(path, params, result) + } + return result, err +} + +// GetNoCache issues a GET that bypasses the cache read for this call only, +// then refreshes the cache with the fresh response on success. Use for +// polling-until-terminal patterns where every call must reflect current +// server state; the same (path, params) pair returning a stale +// "in-progress" snapshot from cache would lock the poll loop on the +// initial response. Writing-back on success means subsequent c.Get calls +// (e.g. a follow-up `... get <id>` after WaitForJob returns) see the +// terminal value, not the stale non-terminal snapshot left behind by the +// first poll. +func (c *Client) GetNoCache(ctx context.Context, path string, params map[string]string) (json.RawMessage, error) { + return c.GetWithHeadersNoCache(ctx, path, params, nil) +} + +// GetWithHeadersNoCache is GetWithHeaders that skips the cache read but still +// writes cacheable fresh responses on success. See GetNoCache for when to +// prefer this over Get/GetWithHeaders. +func (c *Client) GetWithHeadersNoCache(ctx context.Context, path string, params map[string]string, headers map[string]string) (json.RawMessage, error) { + binaryResponse := c.wantsBinaryResponse(headers) + result, _, err := c.do(ctx, "GET", path, params, nil, headers) + if err == nil && c.responseCacheEnabled(binaryResponse) { + c.writeCache(path, params, result) + } + return result, err +} + +func (c *Client) responseCacheEnabled(binaryResponse bool) bool { + return !binaryResponse && !c.NoCache && !c.DryRun && c.cacheDir != "" +} + +// The response cache stores text/JSON bodies as <hash>.json. Binary callers +// opt out so opaque blobs do not land under a misleading extension. +func (c *Client) wantsBinaryResponse(headers map[string]string) bool { + binaryResponse := false + if c != nil && c.Config != nil { + if value, ok := binaryResponseHeaderValue(c.Config.Headers); ok { + binaryResponse = value + } + } + if value, ok := binaryResponseHeaderValue(headers); ok { + binaryResponse = value + } + return binaryResponse +} + +func binaryResponseHeaderValue(headers map[string]string) (bool, bool) { + found := false + for k, v := range headers { + if strings.EqualFold(k, BinaryResponseHeader) { + found = true + if strings.EqualFold(v, "true") { + return true, true + } + } + } + return false, found +} + +func (c *Client) validateCachedRequestAuth(ctx context.Context) error { + return nil +} + +func (c *Client) ProbeGet(ctx context.Context, path string) (int, error) { + _, status, err := c.do(ctx, "GET", path, nil, nil, nil) + return status, err +} + +func (c *Client) cacheKey(path string, params map[string]string) string { + key := path + key += "|base_url=" + c.BaseURL + if c.Config != nil { + key += "|auth_source=" + c.Config.AuthSource + if authHeader := c.Config.AuthHeader(); authHeader != "" { + authHash := sha256.Sum256([]byte(authHeader)) + key += "|auth=" + hex.EncodeToString(authHash[:8]) + } + if c.Config.Path != "" { + key += "|config_path=" + c.Config.Path + } + } + paramKeys := make([]string, 0, len(params)) + for k := range params { + paramKeys = append(paramKeys, k) + } + sort.Strings(paramKeys) + for _, k := range paramKeys { + key += k + "=" + params[k] + } + h := sha256.Sum256([]byte(key)) + return hex.EncodeToString(h[:8]) +} + +func (c *Client) readCache(path string, params map[string]string) (json.RawMessage, bool) { + cacheFile := filepath.Join(c.cacheDir, c.cacheKey(path, params)+".json") + info, err := os.Stat(cacheFile) + if err != nil || time.Since(info.ModTime()) > 5*time.Minute { + return nil, false + } + data, err := os.ReadFile(cacheFile) // #nosec G304 -- cacheFile is derived from the CLI-owned cache dir and a sha256 key. + if err != nil { + return nil, false + } + return json.RawMessage(data), true +} + +func (c *Client) writeCache(path string, params map[string]string, data json.RawMessage) { + _ = os.MkdirAll(c.cacheDir, 0o700) + cacheFile := filepath.Join(c.cacheDir, c.cacheKey(path, params)+".json") + _ = os.WriteFile(cacheFile, []byte(data), 0o600) +} + +// invalidateCache wholesale-removes the cache directory so the next read +// after a mutation cannot return a stale snapshot. Selective per-resource +// invalidation rejected: cache keys are opaque sha256 hashes. +func (c *Client) invalidateCache() { + if c.cacheDir == "" { + return + } + _ = os.RemoveAll(c.cacheDir) +} + +func (c *Client) Post(ctx context.Context, path string, body any) (json.RawMessage, int, error) { + return c.do(ctx, "POST", path, nil, body, nil) +} + +func (c *Client) PostWithParams(ctx context.Context, path string, params map[string]string, body any) (json.RawMessage, int, error) { + return c.do(ctx, "POST", path, params, body, nil) +} + +func (c *Client) PostWithHeaders(ctx context.Context, path string, body any, headers map[string]string) (json.RawMessage, int, error) { + return c.do(ctx, "POST", path, nil, body, headers) +} + +func (c *Client) PostWithParamsAndHeaders(ctx context.Context, path string, params map[string]string, body any, headers map[string]string) (json.RawMessage, int, error) { + return c.do(ctx, "POST", path, params, body, headers) +} + +// PostQueryWithParams is a POST that does not mutate remote state — used +// by read-only operations that ride a mutating verb on the wire (GraphQL +// queries, JSON-RPC reads, POST-based search endpoints). The verify-mode +// short-circuit does not fire for these calls; the request reaches the +// real transport even under PRINTING_PRESS_VERIFY=1 without LIVE_HTTP=1. +func (c *Client) PostQueryWithParams(ctx context.Context, path string, params map[string]string, body any) (json.RawMessage, int, error) { + return c.doRead(ctx, "POST", path, params, body, nil) +} + +// PostQueryWithParamsAndHeaders is the headers-aware counterpart to +// PostQueryWithParams. See PostQueryWithParams for the verify-mode rationale. +func (c *Client) PostQueryWithParamsAndHeaders(ctx context.Context, path string, params map[string]string, body any, headers map[string]string) (json.RawMessage, int, error) { + return c.doRead(ctx, "POST", path, params, body, headers) +} + +func (c *Client) Delete(ctx context.Context, path string) (json.RawMessage, int, error) { + return c.do(ctx, "DELETE", path, nil, nil, nil) +} + +func (c *Client) DeleteWithParams(ctx context.Context, path string, params map[string]string) (json.RawMessage, int, error) { + return c.do(ctx, "DELETE", path, params, nil, nil) +} + +func (c *Client) DeleteWithBody(ctx context.Context, path string, body any) (json.RawMessage, int, error) { + return c.do(ctx, "DELETE", path, nil, body, nil) +} + +func (c *Client) DeleteWithParamsAndBody(ctx context.Context, path string, params map[string]string, body any) (json.RawMessage, int, error) { + return c.do(ctx, "DELETE", path, params, body, nil) +} + +func (c *Client) DeleteWithHeaders(ctx context.Context, path string, headers map[string]string) (json.RawMessage, int, error) { + return c.do(ctx, "DELETE", path, nil, nil, headers) +} + +func (c *Client) DeleteWithParamsAndHeaders(ctx context.Context, path string, params map[string]string, headers map[string]string) (json.RawMessage, int, error) { + return c.do(ctx, "DELETE", path, params, nil, headers) +} + +func (c *Client) DeleteWithBodyAndHeaders(ctx context.Context, path string, body any, headers map[string]string) (json.RawMessage, int, error) { + return c.do(ctx, "DELETE", path, nil, body, headers) +} + +func (c *Client) DeleteWithParamsAndBodyAndHeaders(ctx context.Context, path string, params map[string]string, body any, headers map[string]string) (json.RawMessage, int, error) { + return c.do(ctx, "DELETE", path, params, body, headers) +} + +func (c *Client) Put(ctx context.Context, path string, body any) (json.RawMessage, int, error) { + return c.do(ctx, "PUT", path, nil, body, nil) +} + +func (c *Client) PutWithParams(ctx context.Context, path string, params map[string]string, body any) (json.RawMessage, int, error) { + return c.do(ctx, "PUT", path, params, body, nil) +} + +func (c *Client) PutWithHeaders(ctx context.Context, path string, body any, headers map[string]string) (json.RawMessage, int, error) { + return c.do(ctx, "PUT", path, nil, body, headers) +} + +func (c *Client) PutWithParamsAndHeaders(ctx context.Context, path string, params map[string]string, body any, headers map[string]string) (json.RawMessage, int, error) { + return c.do(ctx, "PUT", path, params, body, headers) +} + +func (c *Client) Patch(ctx context.Context, path string, body any) (json.RawMessage, int, error) { + return c.do(ctx, "PATCH", path, nil, body, nil) +} + +func (c *Client) PatchWithParams(ctx context.Context, path string, params map[string]string, body any) (json.RawMessage, int, error) { + return c.do(ctx, "PATCH", path, params, body, nil) +} + +func (c *Client) PatchWithHeaders(ctx context.Context, path string, body any, headers map[string]string) (json.RawMessage, int, error) { + return c.do(ctx, "PATCH", path, nil, body, headers) +} + +func (c *Client) PatchWithParamsAndHeaders(ctx context.Context, path string, params map[string]string, body any, headers map[string]string) (json.RawMessage, int, error) { + return c.do(ctx, "PATCH", path, params, body, headers) +} + +// isMutatingVerb reports whether the HTTP method writes server state. +// Used by do()'s verify-mode short-circuit to gate dial-out: under +// PRINTING_PRESS_VERIFY=1 (without LIVE_HTTP=1 opt-in), generated +// commands must not actually issue mutating requests, even if a +// handler-level dry-run check was missed. +func isMutatingVerb(method string) bool { + switch method { + case "DELETE", "POST", "PUT", "PATCH": + return true + } + return false +} + +// verifyShortCircuitEnvelope returns the synthetic JSON body that +// stands in for a real mutating response when do() short-circuits in +// verify mode. The __pp_verify_synthetic__ sentinel is namespace- +// reserved (no real API uses __pp_*) so downstream consumers +// (validate-narrative, agent inspections) can key on one obvious field +// instead of trying to disambiguate common literals like status:"noop". +// method and path are echoed back as diagnostic prose for human/agent +// inspection. +func verifyShortCircuitEnvelope(method, path string) json.RawMessage { + body, _ := json.Marshal(map[string]any{ + "__pp_verify_synthetic__": true, + "status": "noop", + "reason": "verify_short_circuit", + "method": method, + "path": path, + }) + return json.RawMessage(body) +} + +func sleepContext(ctx context.Context, wait time.Duration) error { + timer := time.NewTimer(wait) + defer timer.Stop() + select { + case <-timer.C: + return nil + case <-ctx.Done(): + return ctx.Err() + } +} + +// do executes an HTTP request. headerOverrides, when non-nil, override global +// RequiredHeaders for this specific request (used for per-endpoint API versioning). +func (c *Client) do(ctx context.Context, method, path string, params map[string]string, body any, headerOverrides map[string]string) (json.RawMessage, int, error) { + return c.doInternal(ctx, method, path, params, body, headerOverrides, false) +} + +// doRead is do() minus the verify-mode mutating-verb gate. Used by the +// PostQuery* family for read-only operations that ride a mutating verb on +// the wire (GraphQL queries, JSON-RPC reads, POST-based search endpoints). +// The wire verb is still POST/PUT/PATCH so the server sees a real request, +// but the verify-mode short-circuit does not fire because the operation +// does not mutate remote state. +func (c *Client) doRead(ctx context.Context, method, path string, params map[string]string, body any, headerOverrides map[string]string) (json.RawMessage, int, error) { + return c.doInternal(ctx, method, path, params, body, headerOverrides, true) +} + +// doInternal is the shared implementation behind do() and doRead(). The +// readOnlyIntent flag is set by doRead() callers (read-only POST/PUT/PATCH +// operations like GraphQL queries) to skip the mutating-verb verify-mode +// gate. Plain do() callers leave it false and get the usual short-circuit. +func (c *Client) doInternal(ctx context.Context, method, path string, params map[string]string, body any, headerOverrides map[string]string, readOnlyIntent bool) (json.RawMessage, int, error) { + // Verify-mode transport-layer gate. When the verifier (or any consumer + // that sets PRINTING_PRESS_VERIFY=1) drives a mutating verb without + // the LIVE_HTTP=1 opt-in, return a synthetic envelope without dialing, + // minting auth, or touching the cache. The verify pipeline itself + // sets both env vars in mock mode so its httptest server still sees + // real requests; every other consumer gets a safe no-op. + // + // readOnlyIntent suppresses the gate for read-only operations that + // happen to ride a mutating verb on the wire (GraphQL queries, JSON-RPC + // reads, POST-based search endpoints). The handler-level annotation + // `mcp:read-only: true` drives the codegen choice of doRead() vs do(). + // + // Placement note: this fires BEFORE URL building, auth header + // minting, and the success-branch invalidateCache() call below — so + // no cache invalidation runs (no remote state changed) and no + // client_credentials mint happens unnecessarily. + if !readOnlyIntent && isMutatingVerb(method) && cliutil.IsVerifyEnv() && !cliutil.IsVerifyLiveHTTPEnv() { + return verifyShortCircuitEnvelope(method, path), http.StatusOK, nil + } + if c.isLocalDEVONthink() { + return c.doLocalDEVONthink(ctx, method, path, params, body, headerOverrides, readOnlyIntent) + } + targetURL := c.BaseURL + path + + var bodyBytes []byte + if body != nil { + b, err := json.Marshal(body) + if err != nil { + return nil, 0, fmt.Errorf("marshaling body: %w", err) + } + bodyBytes = b + } + + // Resolve auth material before the dry-run branch so --dry-run can preview + // exactly what would be sent. Uses only cached credentials; a token that + // requires a network refresh will be re-fetched on the live request path, + // not during dry-run. + authHeader, err := c.authHeader(ctx) + if err != nil { + return nil, 0, err + } + + // Build the request for dry-run display or actual execution + if c.DryRun { + return c.dryRun(method, targetURL, path, params, bodyBytes, headerOverrides, authHeader) + } + + const maxRetries = 3 + var lastErr error + + for attempt := 0; attempt <= maxRetries; attempt++ { + // Proactive rate limiting — wait before sending + c.limiter.Wait() + var bodyReader io.Reader + if bodyBytes != nil { + bodyReader = strings.NewReader(string(bodyBytes)) + } + + req, err := http.NewRequestWithContext(ctx, method, targetURL, bodyReader) + if err != nil { + return nil, 0, fmt.Errorf("creating request: %w", err) + } + if bodyBytes != nil { + req.Header.Set("Content-Type", "application/json") + } + + if params != nil { + q := req.URL.Query() + for k, v := range params { + if v != "" { + q.Set(k, v) + } + } + req.URL.RawQuery = q.Encode() + } + + if authHeader != "" { + req.Header.Set("Authorization", authHeader) + } + if c.Config != nil { + for k, v := range c.Config.Headers { + req.Header.Set(k, v) + } + } + // Per-endpoint header overrides (e.g., different API version per resource) + for k, v := range headerOverrides { + req.Header.Set(k, v) + } + binaryResponse := strings.EqualFold(req.Header.Get(BinaryResponseHeader), "true") + if binaryResponse { + req.Header.Del(BinaryResponseHeader) + } + if req.Header.Get("User-Agent") == "" { + req.Header.Set("User-Agent", "devonthink-pp-cli/0.1.0") + } + // Go's net/http omits Accept by default; browsers, curl, and other + // stdlibs always send it. Fingerprint-checking WAFs (Imperva, Akamai, + // Cloudflare bot-mode, DataDome) flag the absence as a bot signal + // and answer with empty-body 5xx, 403, or a challenge redirect + // depending on vendor and rule tier. The value is application/json + // rather than */* because strict-JSON APIs (Zendesk, Atlassian REST, + // Salesforce) return 415 on anything that isn't literally + // application/json; specs that need a different content type + // (vendor mediatypes, XML, HTML) declare it via RequiredHeaders or + // per-endpoint headerOverrides, both of which run before this + // if-empty default. + if req.Header.Get("Accept") == "" { + if binaryResponse { + req.Header.Set("Accept", "*/*") + } else { + req.Header.Set("Accept", "application/json") + } + } + + resp, err := c.HTTPClient.Do(req) + if err != nil { + if ctxErr := ctx.Err(); ctxErr != nil { + return nil, 0, ctxErr + } + lastErr = fmt.Errorf("%s %s: %w", method, c.displayURL(path, authHeader), c.maskError(err, authHeader)) + continue + } + + respBody, err := io.ReadAll(resp.Body) + closeErr := resp.Body.Close() + if err != nil { + return nil, 0, fmt.Errorf("reading response: %w", err) + } + if closeErr != nil { + return nil, 0, fmt.Errorf("closing response body: %w", closeErr) + } + + // Success + if resp.StatusCode < 400 { + c.limiter.OnSuccess() + if method != http.MethodGet && !c.DryRun { + c.invalidateCache() + } + // Non-textual bodies (PDF, zip, image, octet-stream) must not be + // run through the JSON sanitizer or returned as raw json.RawMessage + // — return a self-describing base64 envelope instead. Textual and + // JSON responses fall through to the unchanged path. + if isBinaryResponseContentType(resp.Header.Get("Content-Type")) { + env, encErr := wrapBinaryResponse(resp.Header.Get("Content-Type"), respBody) + if encErr != nil { + return nil, 0, encErr + } + return env, resp.StatusCode, nil + } + return json.RawMessage(sanitizeJSONResponse(respBody)), resp.StatusCode, nil + } + + if !binaryResponse { + respBody = sanitizeJSONResponse(respBody) + } + + apiErr := &APIError{ + Method: method, + Path: c.displayURL(path, authHeader), + StatusCode: resp.StatusCode, + Body: c.maskCredentialText(truncateBody(respBody), authHeader), + } + + // Rate limited - adjust adaptive limiter and retry + if resp.StatusCode == 429 && attempt < maxRetries { + c.limiter.OnRateLimit() + wait := cliutil.RetryAfter(resp) + fmt.Fprintf(os.Stderr, "rate limited, waiting %s (attempt %d/%d, rate adjusted to %.1f req/s)\n", wait, attempt+1, maxRetries, c.limiter.Rate()) + if err := sleepContext(ctx, wait); err != nil { + return nil, 0, err + } + lastErr = apiErr + continue + } + + // Server error - retry with backoff + if resp.StatusCode >= 500 && attempt < maxRetries { + wait := time.Duration(math.Pow(2, float64(attempt))) * time.Second + fmt.Fprintf(os.Stderr, "server error %d, retrying in %s (attempt %d/%d)\n", resp.StatusCode, wait, attempt+1, maxRetries) + if err := sleepContext(ctx, wait); err != nil { + return nil, 0, err + } + lastErr = apiErr + continue + } + + // Client error or retries exhausted - return the error + return nil, resp.StatusCode, apiErr + } + + return nil, 0, lastErr +} + +// dryRun prints the outgoing request exactly as the live path would send it, +// using the auth material already resolved in `do()`. Never triggers a network +// call — the caller is responsible for passing cached auth material only. +func (c *Client) dryRun(method, targetURL, path string, params map[string]string, body []byte, headerOverrides map[string]string, authHeader string) (json.RawMessage, int, error) { + fmt.Fprintf(os.Stderr, "%s %s\n", method, c.displayURL(targetURL, authHeader)) + queryPrinted := false + if params != nil { + keys := make([]string, 0, len(params)) + for k := range params { + if params[k] != "" { + keys = append(keys, k) + } + } + sort.Strings(keys) + for _, k := range keys { + sep := "?" + if queryPrinted { + sep = "&" + } + fmt.Fprintf(os.Stderr, " %s%s=%s\n", sep, k, c.maskCredentialText(params[k], authHeader)) + queryPrinted = true + } + } + _ = queryPrinted + if body != nil { + var pretty json.RawMessage + if json.Unmarshal(body, &pretty) == nil { + enc := json.NewEncoder(os.Stderr) + enc.SetIndent(" ", " ") + fmt.Fprintf(os.Stderr, " Body:\n") + _ = enc.Encode(pretty) + } + } + if authHeader != "" { + fmt.Fprintf(os.Stderr, " %s: %s\n", "Authorization", maskToken(authHeader)) + } + fmt.Fprintf(os.Stderr, "\n(dry run - no request sent)\n") + return json.RawMessage(`{"dry_run": true}`), 0, nil +} + +func (c *Client) ConfiguredTimeout() time.Duration { + if c.HTTPClient != nil && c.HTTPClient.Timeout > 0 { + return c.HTTPClient.Timeout + } + return 60 * time.Second +} + +func (c *Client) authHeader(ctx context.Context) (string, error) { + if c.Config == nil { + return "", nil + } + authHeader := c.Config.AuthHeader() + return authHeader, nil +} + +// binaryResponseEnvelope wraps a non-textual success body so it survives the +// json.RawMessage contract every consumer (CLI output, --json, MCP tools) +// depends on. Without it, raw bytes (PDF, zip, image) are corrupted by +// sanitizeJSONResponse and emitted as invalid JSON. The _pp_binary +// discriminator lets callers and agents detect and base64-decode the payload. +type binaryResponseEnvelope struct { + PPBinary bool `json:"_pp_binary"` + ContentType string `json:"content_type"` + Encoding string `json:"encoding"` + Bytes int `json:"bytes"` + Data string `json:"data"` +} + +// isBinaryResponseContentType reports whether a successful response with this +// Content-Type must be base64-wrapped instead of treated as text/JSON. It is +// deliberately narrow: JSON, */*, XML, and every text/* type (including +// text/html, so response_format:html CLIs are untouched) pass through +// unchanged. Only genuinely binary payloads are wrapped. +func isBinaryResponseContentType(ct string) bool { + mt := strings.ToLower(strings.TrimSpace(ct)) + if i := strings.IndexByte(mt, ';'); i >= 0 { + mt = strings.TrimSpace(mt[:i]) + } + if mt == "" { + return false + } + switch { + case mt == "application/json", mt == "text/json", mt == "*/*": + return false + case strings.HasPrefix(mt, "text/"): + return false + case strings.HasSuffix(mt, "+json"), strings.HasSuffix(mt, "+xml"): + return false + case mt == "application/xml", mt == "application/xhtml+xml": + return false + case mt == "application/javascript", mt == "application/ecmascript", + mt == "application/x-www-form-urlencoded", mt == "application/graphql": + return false + } + return true +} + +// wrapBinaryResponse marshals body into a self-describing base64 envelope. +func wrapBinaryResponse(ct string, body []byte) (json.RawMessage, error) { + out, err := json.Marshal(binaryResponseEnvelope{ + PPBinary: true, + ContentType: ct, + Encoding: "base64", + Bytes: len(body), + Data: base64.StdEncoding.EncodeToString(body), + }) + if err != nil { + return nil, fmt.Errorf("encoding binary response: %w", err) + } + return json.RawMessage(out), nil +} + +// sanitizeJSONResponse strips known JSONP/XSSI prefixes and UTF-8 BOM from +// response bodies so that downstream JSON parsing succeeds. For clean JSON +// responses these checks are no-ops. +func sanitizeJSONResponse(body []byte) []byte { + // UTF-8 BOM + body = bytes.TrimPrefix(body, []byte("\xEF\xBB\xBF")) + + // JSONP/XSSI prefixes, ordered longest-first where prefixes overlap + prefixes := [][]byte{ + []byte(")]}'\n"), + []byte(")]}'"), + []byte("{}&&"), + []byte("for(;;);"), + []byte("while(1);"), + } + for _, p := range prefixes { + if bytes.HasPrefix(body, p) { + body = bytes.TrimPrefix(body, p) + body = bytes.TrimLeft(body, " \t\r\n") + break + } + } + return body +} + +// maskToken redacts all but the last 4 characters of a token for safe display. +func maskToken(token string) string { + if token == "" { + return "" + } + if len(token) <= 4 { + return "****" + } + return "****" + token[len(token)-4:] +} + +type maskedError struct { + msg string +} + +func (e maskedError) Error() string { + return e.msg +} + +func (c *Client) displayURL(rawURL string, extraCredentials ...string) string { + return c.maskCredentialText(rawURL, extraCredentials...) +} + +func (c *Client) maskError(err error, extraCredentials ...string) error { + if err == nil { + return nil + } + raw := err.Error() + msg := c.maskCredentialText(raw, extraCredentials...) + if msg == raw { + return err + } + return maskedError{msg: msg} +} + +func (c *Client) maskCredentialText(text string, extraCredentials ...string) string { + if text == "" { + return text + } + type credentialMask struct { + needle string + replacement string + } + var masks []credentialMask + seen := map[string]struct{}{} + addValue := func(value string) { + value = strings.TrimSpace(value) + if value == "" { + return + } + replacement := maskToken(value) + addMask := func(needle string) { + if needle == "" { + return + } + if _, ok := seen[needle]; ok { + return + } + seen[needle] = struct{}{} + masks = append(masks, credentialMask{needle: needle, replacement: replacement}) + } + addMask(value) + if escaped := url.QueryEscape(value); escaped != value { + addMask(escaped) + } + if escaped := url.PathEscape(value); escaped != value { + addMask(escaped) + } + } + addCredential := func(value string) { + value = strings.TrimSpace(value) + addValue(value) + if _, token, ok := strings.Cut(value, " "); ok { + addValue(token) + } + } + for _, value := range extraCredentials { + addCredential(value) + } + if c != nil && c.Config != nil { + addCredential(c.Config.AuthHeaderVal) + addCredential(c.Config.AuthHeader()) + } + sort.SliceStable(masks, func(i, j int) bool { + return len(masks[i].needle) > len(masks[j].needle) + }) + masked := text + for _, mask := range masks { + masked = strings.ReplaceAll(masked, mask.needle, mask.replacement) + } + return masked +} + +func truncateBody(b []byte) string { + const maxBytes = 4096 + if len(b) <= maxBytes { + return string(b) + } + return strings.ToValidUTF8(string(b[:maxBytes]), "") + "..." +} diff --git a/library/productivity/devonthink/internal/client/client_test.go b/library/productivity/devonthink/internal/client/client_test.go new file mode 100644 index 0000000000..33313b153e --- /dev/null +++ b/library/productivity/devonthink/internal/client/client_test.go @@ -0,0 +1,72 @@ +// Copyright 2026 rowdy and contributors. Licensed under Apache-2.0. See LICENSE. +// Generated by CLI Printing Press (https://github.com/mvanhorn/cli-printing-press). DO NOT EDIT. + +package client + +import ( + "bytes" + "strings" + "testing" + "unicode/utf8" +) + +func TestTruncateBody(t *testing.T) { + t.Parallel() + + const maxBytes = 4096 + + cases := []struct { + name string + input []byte + wantLen int + wantHasTail bool + }{ + {"empty", nil, 0, false}, + {"under cap", []byte("hello"), 5, false}, + {"at cap", bytes.Repeat([]byte("a"), maxBytes), maxBytes, false}, + {"one over cap", bytes.Repeat([]byte("a"), maxBytes+1), maxBytes + 3, true}, + {"huge body", bytes.Repeat([]byte("a"), maxBytes*8), maxBytes + 3, true}, + } + + for _, tc := range cases { + tc := tc + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + got := truncateBody(tc.input) + if len(got) != tc.wantLen { + t.Fatalf("len = %d, want %d", len(got), tc.wantLen) + } + if tc.wantHasTail && !strings.HasSuffix(got, "...") { + t.Fatalf("want trailing %q", "...") + } + if !tc.wantHasTail && strings.HasSuffix(got, "...") { + t.Fatalf("unexpected trailing %q in %q", "...", got) + } + }) + } +} + +func TestTruncateBody_UTF8RuneAtBoundary(t *testing.T) { + t.Parallel() + + // '€' is 3 bytes (0xE2 0x82 0xAC). Place it so the slice at byte 4096 cuts + // mid-rune; strings.ToValidUTF8 should drop the partial rune cleanly rather + // than emit U+FFFD or invalid UTF-8. + prefix := strings.Repeat("a", 4094) + body := []byte(prefix + "€" + strings.Repeat("b", 100)) + got := truncateBody(body) + + if !utf8.ValidString(got) { + t.Fatalf("output is not valid UTF-8") + } + if strings.ContainsRune(got, utf8.RuneError) { + t.Fatalf("output contains replacement rune U+FFFD") + } + if !strings.HasSuffix(got, "...") { + t.Fatalf("want trailing %q", "...") + } + // Partial rune must be dropped, not replaced: 4094 valid bytes + "...". + if want := 4094 + 3; len(got) != want { + t.Fatalf("len = %d, want %d (partial rune should be dropped, not replaced)", len(got), want) + } +} diff --git a/library/productivity/devonthink/internal/client/client_verify_short_circuit_test.go b/library/productivity/devonthink/internal/client/client_verify_short_circuit_test.go new file mode 100644 index 0000000000..4de14f2fa4 --- /dev/null +++ b/library/productivity/devonthink/internal/client/client_verify_short_circuit_test.go @@ -0,0 +1,169 @@ +// Copyright 2026 rowdy and contributors. Licensed under Apache-2.0. See LICENSE. +// Generated by CLI Printing Press (https://github.com/mvanhorn/cli-printing-press). DO NOT EDIT. + +package client + +import ( + "bytes" + "context" + "encoding/json" + "io" + "net/http" + "testing" + "time" + + "github.com/mvanhorn/printing-press-library/library/productivity/devonthink/internal/config" +) + +// recordingRoundTripper counts how many times its RoundTrip method is +// invoked and returns an empty 200 response. Used by the verify-mode +// short-circuit tests to assert that the transport layer never dials +// when the gate fires. +type recordingRoundTripper struct { + calls int +} + +func (r *recordingRoundTripper) RoundTrip(req *http.Request) (*http.Response, error) { + r.calls++ + return &http.Response{ + StatusCode: 200, + Body: io.NopCloser(bytes.NewReader([]byte("{}"))), + Header: http.Header{}, + }, nil +} + +// newClientWithRecorder builds a minimal *Client wired to a recording +// transport. The Client is constructed through New() so the unexported +// limiter and cacheDir fields are initialized, then HTTPClient is +// swapped out for one whose Transport records every call. +func newClientWithRecorder(t *testing.T) (*Client, *recordingRoundTripper) { + t.Helper() + rec := &recordingRoundTripper{} + cfg := &config.Config{BaseURL: "http://example.test"} + c := New(cfg, time.Second, 0) + c.HTTPClient = &http.Client{Transport: rec} + c.NoCache = true + return c, rec +} + +// TestClient_VerifyShortCircuit_MutatingVerbs pins the transport-layer +// short-circuit emitted by client.go.tmpl. Under PRINTING_PRESS_VERIFY=1 +// with no LIVE_HTTP opt-in, every mutating verb (DELETE/POST/PUT/PATCH) +// must return a synthetic envelope without dialing. +// +// A future template edit that drops the gate, narrows the verb list, or +// removes either env-var check would silently re-open the agent-readiness +// gap the gate was added to close. This test fails on any of those +// drifts as part of every printed CLI's own go-test pass. +func TestClient_VerifyShortCircuit_MutatingVerbs(t *testing.T) { + for _, verb := range []string{"DELETE", "POST", "PUT", "PATCH"} { + verb := verb + t.Run(verb, func(t *testing.T) { + t.Setenv("PRINTING_PRESS_VERIFY", "1") + // LIVE_HTTP must NOT be set for the short-circuit branch. + // Explicit unset defends against a stale value in the parent env. + t.Setenv("PRINTING_PRESS_VERIFY_LIVE_HTTP", "") + c, rec := newClientWithRecorder(t) + + body, status, err := c.do(context.Background(), verb, "/test", nil, nil, nil) + if err != nil { + t.Fatalf("do(%s) returned error: %v", verb, err) + } + if status != http.StatusOK { + t.Fatalf("do(%s) status = %d, want %d", verb, status, http.StatusOK) + } + if rec.calls != 0 { + t.Fatalf("do(%s) attempted %d HTTP calls; want 0 (short-circuit)", verb, rec.calls) + } + + var env map[string]any + if err := json.Unmarshal(body, &env); err != nil { + t.Fatalf("envelope is not valid JSON: %v", err) + } + if got, _ := env["__pp_verify_synthetic__"].(bool); !got { + t.Fatalf("envelope must include __pp_verify_synthetic__: true; got %v", env) + } + if got, _ := env["reason"].(string); got != "verify_short_circuit" { + t.Fatalf("envelope reason = %q, want %q", got, "verify_short_circuit") + } + if got, _ := env["method"].(string); got != verb { + t.Fatalf("envelope method = %q, want %q", got, verb) + } + if got, _ := env["path"].(string); got != "/test" { + t.Fatalf("envelope path = %q, want %q", got, "/test") + } + }) + } +} + +// TestClient_VerifyShortCircuit_LiveHTTPOptIn pins the verify mock-mode +// contract: when LIVE_HTTP=1 is set alongside VERIFY=1, the short-circuit +// does NOT fire and the transport dials. The verifier's httptest mock +// server depends on this opt-in path to receive mutating requests so its +// pass/fail assertions can run against real wire-format responses. +func TestClient_VerifyShortCircuit_LiveHTTPOptIn(t *testing.T) { + t.Setenv("PRINTING_PRESS_VERIFY", "1") + t.Setenv("PRINTING_PRESS_VERIFY_LIVE_HTTP", "1") + c, rec := newClientWithRecorder(t) + + _, _, _ = c.do(context.Background(), "DELETE", "/test", nil, nil, nil) + + if rec.calls < 1 { + t.Fatalf("LIVE_HTTP=1 should opt back in to real dial; recorder saw %d calls", rec.calls) + } +} + +// TestClient_VerifyShortCircuit_NoEnv pins the operator path: with no +// verify env vars set, mutating verbs dial normally. +func TestClient_VerifyShortCircuit_NoEnv(t *testing.T) { + // Explicitly unset to defend against test-runner inherited values. + t.Setenv("PRINTING_PRESS_VERIFY", "") + t.Setenv("PRINTING_PRESS_VERIFY_LIVE_HTTP", "") + c, rec := newClientWithRecorder(t) + + _, _, _ = c.do(context.Background(), "DELETE", "/test", nil, nil, nil) + + if rec.calls < 1 { + t.Fatalf("no verify env should dial normally; recorder saw %d calls", rec.calls) + } +} + +// TestClient_VerifyShortCircuit_GETControl pins that the gate is +// verb-specific: GET requests are never short-circuited, even under +// PRINTING_PRESS_VERIFY=1, because they cannot mutate remote state. +// A regression that broadens isMutatingVerb to include GET would break +// every CLI's cached-fallback and list/show paths under verify mode. +func TestClient_VerifyShortCircuit_GETControl(t *testing.T) { + t.Setenv("PRINTING_PRESS_VERIFY", "1") + t.Setenv("PRINTING_PRESS_VERIFY_LIVE_HTTP", "") + c, rec := newClientWithRecorder(t) + + _, _, _ = c.do(context.Background(), "GET", "/test", nil, nil, nil) + + if rec.calls < 1 { + t.Fatalf("GET must never short-circuit; recorder saw %d calls", rec.calls) + } +} + +// TestClient_VerifyShortCircuit_ReadOnlyPOST pins the doRead bypass: a +// POST routed through doRead (the path PostQuery* takes for GraphQL +// queries, JSON-RPC reads, and POST-based search) must NOT short-circuit +// under PRINTING_PRESS_VERIFY=1, because the operation does not mutate +// remote state. Without this, an inherited verify env silently breaks +// every read on a shared-endpoint API. +func TestClient_VerifyShortCircuit_ReadOnlyPOST(t *testing.T) { + for _, verb := range []string{"POST", "PUT", "PATCH"} { + verb := verb + t.Run(verb, func(t *testing.T) { + t.Setenv("PRINTING_PRESS_VERIFY", "1") + t.Setenv("PRINTING_PRESS_VERIFY_LIVE_HTTP", "") + c, rec := newClientWithRecorder(t) + + _, _, _ = c.doRead(context.Background(), verb, "/test", nil, nil, nil) + + if rec.calls < 1 { + t.Fatalf("doRead(%s) must dial through; recorder saw %d calls", verb, rec.calls) + } + }) + } +} diff --git a/library/productivity/devonthink/internal/client/local_devonthink.go b/library/productivity/devonthink/internal/client/local_devonthink.go new file mode 100644 index 0000000000..92cfbcabb3 --- /dev/null +++ b/library/productivity/devonthink/internal/client/local_devonthink.go @@ -0,0 +1,1076 @@ +// Copyright 2026 rowdy and contributors. Licensed under Apache-2.0. See LICENSE. + +package client + +import ( + "bytes" + "context" + "encoding/json" + "fmt" + "io" + "net/http" + "os" + "os/exec" + "path/filepath" + "strconv" + "strings" + "time" +) + +const devonthinkAppID = "DNtp" + +func (c *Client) isLocalDEVONthink() bool { + if c == nil { + return false + } + baseURL := strings.TrimSpace(c.BaseURL) + if baseURL == "" { + return true + } + switch strings.ToLower(baseURL) { + case "local", "devonthink", "devonthink://local": + return true + } + return false +} + +func (c *Client) doLocalDEVONthink(ctx context.Context, method, path string, params map[string]string, body any, _ map[string]string, _ bool) (json.RawMessage, int, error) { + if c.DryRun { + return localJSON(map[string]any{ + "status": "dry-run", + "dry_run": true, + "method": method, + "path": path, + "params": params, + "body": body, + "message": "Would execute against the local DEVONthink app on this Mac.", + }) + } + + if isMutatingVerb(method) && !(path == "/mcp/call" && localMCPToolLooksReadOnly(params["tool"])) { + return localJSON(map[string]any{ + "status": "blocked", + "method": method, + "path": path, + "message": "Local write support is intentionally gated in this build. Re-run with --dry-run to preview, or use batch plan/apply after reviewing the generated plan.", + }) + } + + switch { + case path == "/" || path == "/runtime/doctor": + return localJSON(c.localRuntimeDoctor(ctx)) + case path == "/databases": + return localJSON(c.localDatabases(ctx)) + case path == "/records/search": + return localJSON(c.localRecordSearch(ctx, params)) + case path == "/records/lookup": + return localJSON(c.localRecordLookup(ctx, params)) + case path == "/records/create" || path == "/records/update" || path == "/records/move": + return localJSON(c.localBlockedOperation(method, path)) + case path == "/selection": + return localJSON(c.localSelection(ctx)) + case path == "/selection/snapshot": + return localJSON(c.localSelectionSnapshot(ctx, params)) + case path == "/inventory/export": + return c.localInventoryExport(ctx, params) + case path == "/groups/tree": + return localJSON(c.localGroupsTree(ctx, params)) + case path == "/graph/links": + return localJSON(c.localGraphLinks(ctx, params)) + case path == "/graph/audit": + return localJSON(c.localGraphAudit(ctx, params)) + case path == "/tags/analyze": + return localJSON(c.localTagsAnalyze(ctx, params)) + case path == "/sheets/get": + return localJSON(c.localSheet(ctx, params)) + case path == "/ai/ask" || path == "/ai/summarize": + return localJSON(c.localAIRead(path, params)) + case path == "/mcp/tools": + return c.localMCPTools(ctx) + case path == "/mcp/schema": + return c.localMCPSchema(ctx) + case path == "/mcp/call": + return c.localMCPCall(ctx, params, body) + case path == "/batch/plan": + return localJSON(c.localBatchPlan(params, body)) + case path == "/batch/apply": + return localJSON(c.localBatchApply(params, body)) + case path == "/ledger": + return localJSON(c.localLedgerList(params)) + case strings.HasPrefix(path, "/ledger/"): + return localJSON(c.localLedgerShow(path)) + case path == "/privacy/audit": + return localJSON(c.localPrivacyAudit(ctx, params)) + case path == "/mirror/sync": + return localJSON(c.localMirrorSync(params)) + case path == "/mirror/search": + return localJSON(c.localMirrorSearch(ctx, params)) + case path == "/context/pack": + return localJSON(c.localContextPack(ctx, params)) + case strings.HasPrefix(path, "/records/"): + return localJSON(c.localRecordPath(ctx, path, params)) + case path == "/ingest/file" || path == "/ingest/url" || path == "/media/ocr" || path == "/media/transcribe": + return localJSON(c.localBlockedOperation(method, path)) + default: + return localJSON(map[string]any{ + "status": "unsupported", + "path": path, + "message": "No local DEVONthink adapter exists for this endpoint yet.", + }) + } +} + +func localJSON(v any) (json.RawMessage, int, error) { + data, err := json.MarshalIndent(v, "", " ") + if err != nil { + return nil, 0, err + } + return json.RawMessage(data), http.StatusOK, nil +} + +func (c *Client) localRuntimeDoctor(ctx context.Context) map[string]any { + version, versionErr := devonthinkVersion(ctx) + toolsPath, toolsOK := devonthinkMCPToolsPath() + mcpRunning := false + if raw, ok, err := c.localMCPToolJSON(ctx, "is_running", map[string]any{}); err == nil && ok { + var running map[string]any + if json.Unmarshal(raw, &running) == nil { + if value, _ := running["running"].(bool); value { + mcpRunning = true + } + } + } + return map[string]any{ + "app": map[string]any{ + "name": "DEVONthink", + "bundle_id": "com.devon-technologies.think3", + "running": versionErr == nil || mcpRunning, + "version": version, + "applescript": versionErr == nil, + "error": errorString(versionErr), + "local_only": true, + "network_scope": "this Mac or user-controlled LAN only", + }, + "mcp": map[string]any{ + "tools_file": toolsPath, + "tools_file_found": toolsOK, + "http_ok": mcpRunning, + "note": "The official DEVONthink MCP server is local. This CLI uses local automation by default and can expose MCP metadata when available.", + }, + "privacy": map[string]any{ + "default_transport": "local", + "cloud_required": false, + }, + } +} + +func (c *Client) localDatabases(ctx context.Context) []map[string]any { + if raw, ok, err := c.localMCPToolJSON(ctx, "get_databases", map[string]any{}); err == nil && ok { + var databases []map[string]any + if json.Unmarshal(raw, &databases) == nil { + return databases + } + } + names, err := devonthinkDatabaseNames(ctx) + if err != nil { + return []map[string]any{} + } + out := make([]map[string]any, 0, len(names)) + for _, name := range names { + out = append(out, map[string]any{ + "name": name, + "local_only": true, + }) + } + return out +} + +func (c *Client) localRecordSearch(ctx context.Context, params map[string]string) []map[string]any { + query := firstNonEmpty(params["query"], "kind:document") + args := map[string]any{"query": query} + if limit, ok := intParam(params, "limit"); ok { + args["limit"] = limit + } + if offset, ok := intParam(params, "offset"); ok { + args["offset"] = offset + } + if sort := strings.TrimSpace(params["sort"]); sort != "" { + args["sort"] = sort + } + if database := strings.TrimSpace(params["database"]); database != "" { + args["database_uuid"] = database + } + if group := strings.TrimSpace(params["group"]); group != "" { + args["group_uuid"] = group + } + if raw, ok, err := c.localMCPToolJSON(ctx, "search_records", args); err == nil && ok { + var envelope struct { + Results []map[string]any `json:"results"` + } + if json.Unmarshal(raw, &envelope) == nil { + return envelope.Results + } + var direct []map[string]any + if json.Unmarshal(raw, &direct) == nil { + return direct + } + } + return []map[string]any{} +} + +func (c *Client) localRecordLookup(ctx context.Context, params map[string]string) map[string]any { + args := map[string]any{} + for _, key := range []string{"name", "url", "path", "location", "filename", "comment"} { + if value := strings.TrimSpace(params[key]); value != "" { + args[key] = value + break + } + } + if len(args) > 0 { + if raw, ok, err := c.localMCPToolJSON(ctx, "lookup_records", args); err == nil && ok { + var matches []map[string]any + if json.Unmarshal(raw, &matches) == nil { + return map[string]any{"matches": matches, "query": args} + } + } + } + return map[string]any{ + "matches": []map[string]any{}, + "query": params, + "note": "Lookup is local-only. Live record enumeration will use DEVONthink automation or MCP as that bridge is expanded.", + } +} + +func (c *Client) localSelection(ctx context.Context) []map[string]any { + if raw, ok, err := c.localMCPToolJSON(ctx, "get_selected_records", map[string]any{}); err == nil && ok { + var records []map[string]any + if json.Unmarshal(raw, &records) == nil { + return records + } + } + records, _ := devonthinkSelectedRecordNames(ctx) + return records +} + +func (c *Client) localSelectionSnapshot(ctx context.Context, params map[string]string) map[string]any { + records := c.localSelection(ctx) + warnings := []string{} + return map[string]any{ + "created_at": time.Now().UTC().Format(time.RFC3339), + "name": firstNonEmpty(params["name"], "selection-snapshot"), + "records": records, + "count": len(records), + "warnings": warnings, + } +} + +func (c *Client) localInventoryExport(ctx context.Context, params map[string]string) (json.RawMessage, int, error) { + format := firstNonEmpty(params["format"], "maintenance") + warnings := []string{} + databases := c.localDatabases(ctx) + if len(databases) == 0 { + warnings = append(warnings, "No databases were returned by local MCP/automation.") + } + query := firstNonEmpty(params["query"], "kind:document") + limit := 100 + if parsed, ok := intParam(params, "limit"); ok && parsed > 0 { + limit = parsed + } + documents := c.localRecordSearch(ctx, map[string]string{ + "query": query, + "limit": strconv.Itoa(limit), + }) + for i := range documents { + delete(documents[i], "path") + } + payload := map[string]any{ + "schema": map[string]any{ + "name": "devonthink-pp-cli.inventory", + "version": 1, + "format": format, + }, + "generated_at": time.Now().UTC().Format(time.RFC3339), + "source": "devonthink-pp-cli", + "local_only": true, + "databases": databases, + "groups": []map[string]any{}, + "documents": documents, + "tags": []map[string]any{}, + "query": query, + "limit": limit, + "warnings": warnings, + "maintenance_contract": map[string]any{ + "consumer": "Rowdy/DEVONthink-ai-maintenance", + "role": "mechanical inventory export; filing and triage policy stay outside the core CLI", + }, + } + data, status, err := localJSON(payload) + if err != nil { + return data, status, err + } + if output := strings.TrimSpace(params["output"]); output != "" { + if err := writeLocalOutput(output, data); err != nil { + return nil, 0, err + } + } + return data, status, nil +} + +func (c *Client) localGroupsTree(ctx context.Context, params map[string]string) map[string]any { + args := map[string]any{} + if uuid := firstNonEmpty(params["uuid"], params["root"], params["root_uuid"]); uuid != "" { + args["uuid"] = uuid + } + if depth, ok := intParam(params, "max_depth"); ok { + args["max_depth"] = depth + } + if include := strings.TrimSpace(params["include_documents"]); include != "" { + args["include_documents"] = strings.EqualFold(include, "true") || include == "1" + } + if len(args) > 0 { + if raw, ok, err := c.localMCPToolJSON(ctx, "get_group_tree", args); err == nil && ok { + var tree map[string]any + if json.Unmarshal(raw, &tree) == nil { + return tree + } + } + } + return map[string]any{ + "database": params["database"], + "root": map[string]any{ + "name": firstNonEmpty(params["root"], "/"), + "children": []map[string]any{}, + }, + "warnings": []string{"Live group traversal is not enabled in the local adapter yet."}, + } +} + +func (c *Client) localGraphLinks(ctx context.Context, params map[string]string) map[string]any { + uuid := firstNonEmpty(params["uuid"], params["record"]) + if uuid != "" { + args := map[string]any{"uuid": uuid} + if direction := strings.TrimSpace(params["direction"]); direction != "" { + args["direction"] = direction + } + if kind := strings.TrimSpace(params["kind"]); kind != "" { + args["kind"] = kind + } + if raw, ok, err := c.localMCPToolJSON(ctx, "get_record_links", args); err == nil && ok { + var links map[string]any + if json.Unmarshal(raw, &links) == nil { + return links + } + } + } + return map[string]any{ + "record": firstNonEmpty(params["uuid"], params["record"]), + "incoming": []map[string]any{}, + "outgoing": []map[string]any{}, + } +} + +func (c *Client) localGraphAudit(ctx context.Context, params map[string]string) map[string]any { + records := c.localRecordSearch(ctx, map[string]string{ + "query": firstNonEmpty(params["query"], "item:tagged"), + "limit": firstNonEmpty(params["limit"], "25"), + }) + return map[string]any{ + "query": params["query"], + "checked_at": time.Now().UTC().Format(time.RFC3339), + "issues": []map[string]any{}, + "sample": records, + "warnings": []string{"Graph audit is metadata-only in this build; no record content was exported."}, + } +} + +func (c *Client) localTagsAnalyze(ctx context.Context, params map[string]string) map[string]any { + args := map[string]any{} + if database := strings.TrimSpace(params["database"]); database != "" { + args["database_uuid"] = database + } + if raw, ok, err := c.localMCPToolJSON(ctx, "list_database_tags", args); err == nil && ok { + var tags any + if json.Unmarshal(raw, &tags) == nil { + return map[string]any{ + "database": params["database"], + "tags": tags, + } + } + } + return map[string]any{ + "database": params["database"], + "tags": []map[string]any{}, + "summary": map[string]any{ + "count": 0, + "duplicates": 0, + "orphans": 0, + }, + } +} + +func (c *Client) localSheet(ctx context.Context, params map[string]string) map[string]any { + if uuid := strings.TrimSpace(params["uuid"]); uuid != "" { + if raw, ok, err := c.localMCPToolJSON(ctx, "get_record_text", map[string]any{"uuid": uuid}); err == nil && ok { + var text any + if json.Unmarshal(raw, &text) == nil { + return map[string]any{"uuid": uuid, "text": text} + } + } + } + return map[string]any{ + "uuid": params["uuid"], + "columns": []string{}, + "rows": []map[string]any{}, + } +} + +func (c *Client) localAIRead(path string, params map[string]string) map[string]any { + return map[string]any{ + "status": "blocked", + "path": path, + "query": params["query"], + "message": "AI-backed DEVONthink reads require an explicit local MCP-backed implementation so content exposure can be audited first.", + } +} + +func (c *Client) localMCPTools(ctx context.Context) (json.RawMessage, int, error) { + if raw, ok, err := c.localMCPListTools(ctx); err == nil && ok { + var parsed any + if json.Unmarshal(raw, &parsed) == nil { + return localJSON(map[string]any{ + "source": "http://127.0.0.1", + "tools": parsed, + }) + } + } + path, ok := devonthinkMCPToolsPath() + if !ok { + return localJSON(map[string]any{"tools": []map[string]any{}, "error": "mcp-tools.json not found"}) + } + data, err := os.ReadFile(path) // #nosec G304 -- path comes from fixed DEVONthink application bundle candidates. + if err != nil { + return nil, 0, err + } + var parsed any + if err := json.Unmarshal(data, &parsed); err != nil { + return nil, 0, err + } + return localJSON(map[string]any{ + "source": path, + "tools": parsed, + }) +} + +func (c *Client) localMCPSchema(ctx context.Context) (json.RawMessage, int, error) { + return c.localMCPTools(ctx) +} + +func (c *Client) localMCPCall(ctx context.Context, params map[string]string, body any) (json.RawMessage, int, error) { + tool := firstNonEmpty(params["tool"], params["name"]) + if tool == "" { + return localJSON(map[string]any{ + "status": "blocked", + "message": "Pass a local MCP tool name.", + }) + } + args := map[string]any{} + if rawArgs := strings.TrimSpace(params["args"]); rawArgs != "" { + if err := json.Unmarshal([]byte(rawArgs), &args); err != nil { + return nil, 0, err + } + } + if bodyMap, ok := body.(map[string]any); ok && len(args) == 0 { + if raw, ok := bodyMap["args"].(string); ok && strings.TrimSpace(raw) != "" { + if err := json.Unmarshal([]byte(raw), &args); err != nil { + looseArgs, looseOK := parseLooseMCPArgs(raw) + if !looseOK { + return nil, 0, err + } + args = looseArgs + } + } else { + args = bodyMap + } + } + if raw, ok, err := c.localMCPToolJSON(ctx, tool, args); err != nil { + return nil, 0, err + } else if ok { + return raw, http.StatusOK, nil + } + if text, ok, err := c.localMCPToolText(ctx, tool, args); err != nil { + return nil, 0, err + } else if ok { + return localJSON(map[string]any{"tool": tool, "text": text}) + } + return localJSON(map[string]any{ + "status": "empty", + "tool": tool, + "message": "The local MCP tool returned no content.", + }) +} + +func (c *Client) localBatchPlan(params map[string]string, body any) map[string]any { + return map[string]any{ + "status": "planned", + "created_at": time.Now().UTC().Format(time.RFC3339), + "source": params["source"], + "actions": []map[string]any{}, + "input": body, + "review_note": "No write actions were generated by the local adapter.", + } +} + +func (c *Client) localBatchApply(params map[string]string, body any) map[string]any { + return map[string]any{ + "status": "blocked", + "plan": params["plan"], + "input": body, + "message": "Batch apply is disabled until the CLI has a confirmed local write bridge and review ledger.", + } +} + +func (c *Client) localLedgerList(params map[string]string) []map[string]any { + return []map[string]any{} +} + +func (c *Client) localLedgerShow(path string) map[string]any { + return map[string]any{ + "id": strings.TrimPrefix(path, "/ledger/"), + "status": "not-found", + "actions": []map[string]any{}, + } +} + +func (c *Client) localPrivacyAudit(ctx context.Context, params map[string]string) map[string]any { + query := firstNonEmpty(params["query"], "kind:document") + limit := firstNonEmpty(params["limit"], "25") + records := c.localRecordSearch(ctx, map[string]string{"query": query, "limit": limit}) + return map[string]any{ + "query": query, + "limit": limit, + "checked_at": time.Now().UTC().Format(time.RFC3339), + "local_only": true, + "summary": map[string]any{ + "records": len(records), + "estimated_characters": 0, + "external_calls": 0, + }, + "records": records, + "risks": []map[string]any{}, + "warnings": []string{ + "No record content was exported by this audit.", + "Use this command before AI workflows to make content exposure explicit.", + }, + } +} + +func (c *Client) localMirrorSync(params map[string]string) map[string]any { + return map[string]any{ + "status": "noop", + "mirror_path": params["path"], + "synced": 0, + "updated": 0, + "deleted": 0, + } +} + +func (c *Client) localMirrorSearch(ctx context.Context, params map[string]string) []map[string]any { + query := firstNonEmpty(params["query"], params["q"]) + limit := firstNonEmpty(params["limit"], "20") + records := c.localRecordSearch(ctx, map[string]string{"query": query, "limit": limit}) + if len(records) == 0 { + return []map[string]any{{ + "uuid": "mirror-fallback", + "name": "No local mirror rows matched " + query, + "path": "query:" + query, + "query": query, + "source": "live-metadata-fallback", + "note": "Run mirror sync to build the SQLite mirror, or use records search for live MCP metadata.", + }} + } + for i := range records { + records[i]["mirror_query"] = query + records[i]["source"] = "live-metadata-fallback" + } + return records +} + +func (c *Client) localContextPack(ctx context.Context, params map[string]string) map[string]any { + query := params["query"] + records := []map[string]any{} + if query != "" { + records = c.localRecordSearch(ctx, map[string]string{ + "query": query, + "limit": firstNonEmpty(params["limit"], "10"), + }) + } else { + records = c.localSelection(ctx) + } + markdown := "# DEVONthink Context Pack\n\n" + if query != "" { + markdown += "Query: " + query + "\n\n" + } + if len(records) == 0 { + markdown += "No matching records were available through the current local adapter.\n" + } else { + for _, record := range records { + name, _ := record["name"].(string) + uuid, _ := record["uuid"].(string) + location, _ := record["location"].(string) + markdown += "- " + firstNonEmpty(name, uuid, "record") + if location != "" { + markdown += " (" + location + ")" + } + if uuid != "" { + markdown += " [" + uuid + "]" + } + markdown += "\n" + } + } + return map[string]any{ + "created_at": time.Now().UTC().Format(time.RFC3339), + "query": query, + "token_budget": firstNonEmpty(params["token_budget"], params["token-budget"], "4000"), + "record_count": len(records), + "records": records, + "markdown": markdown, + "truncated": false, + "privacy_scope": "local", + } +} + +func (c *Client) localRecordPath(ctx context.Context, path string, params map[string]string) map[string]any { + uuid := recordIDFromPath(path) + switch { + case strings.HasSuffix(path, "/content"): + args := map[string]any{"uuid": uuid} + if query := strings.TrimSpace(params["query"]); query != "" { + args["query"] = query + } + if maxTokens, ok := intParam(params, "max_token_count"); ok { + args["max_token_count"] = maxTokens + } + if value, ok, err := c.localMCPToolValue(ctx, "extract_record_content", args); err == nil && ok { + return map[string]any{ + "uuid": uuid, + "content": value, + "format": firstNonEmpty(params["format"], "devonthink-extract"), + } + } + return map[string]any{ + "uuid": uuid, + "content": "", + "format": firstNonEmpty(params["format"], "text"), + "warnings": []string{"Live content extraction is not enabled in the local adapter yet."}, + } + case strings.HasSuffix(path, "/related"): + if value, ok, err := c.localMCPToolValue(ctx, "find_similar_records", map[string]any{"uuid": uuid, "limit": 25}); err == nil && ok { + return map[string]any{"uuid": uuid, "related": value} + } + return map[string]any{"uuid": uuid, "related": []map[string]any{}} + case strings.HasSuffix(path, "/highlights"): + if value, ok, err := c.localMCPToolValue(ctx, "extract_record_highlights", map[string]any{"uuid": uuid}); err == nil && ok { + return map[string]any{"uuid": uuid, "highlights": value} + } + return map[string]any{"uuid": uuid, "highlights": []map[string]any{}} + case strings.HasSuffix(path, "/versions"): + if value, ok, err := c.localMCPToolValue(ctx, "get_record_versions", map[string]any{"uuid": uuid}); err == nil && ok { + return map[string]any{"uuid": uuid, "versions": value} + } + return map[string]any{"uuid": uuid, "versions": []map[string]any{}} + default: + if value, ok, err := c.localMCPToolValue(ctx, "get_record_properties", map[string]any{"uuid": uuid}); err == nil && ok { + if record, ok := value.(map[string]any); ok { + return record + } + return map[string]any{"uuid": uuid, "record": value} + } + return map[string]any{ + "uuid": uuid, + "item_link": "x-devonthink-item://" + uuid, + "local": true, + "warnings": []string{"Live record metadata is not enabled in the local adapter yet."}, + } + } +} + +func (c *Client) localBlockedOperation(method, path string) map[string]any { + return map[string]any{ + "status": "blocked", + "method": method, + "path": path, + "message": "This local operation needs an explicit safe-write implementation before it can modify DEVONthink.", + } +} + +func (c *Client) localMCPToolJSON(ctx context.Context, tool string, args map[string]any) (json.RawMessage, bool, error) { + text, ok, err := c.localMCPToolText(ctx, tool, args) + if err != nil || !ok { + return nil, false, err + } + text = strings.TrimSpace(text) + if text == "" || !json.Valid([]byte(text)) { + return nil, false, nil + } + return json.RawMessage(text), true, nil +} + +func (c *Client) localMCPToolValue(ctx context.Context, tool string, args map[string]any) (any, bool, error) { + text, ok, err := c.localMCPToolText(ctx, tool, args) + if err != nil || !ok { + return nil, false, err + } + text = strings.TrimSpace(text) + if text == "" { + return "", true, nil + } + if json.Valid([]byte(text)) { + var value any + if err := json.Unmarshal([]byte(text), &value); err != nil { + return nil, false, err + } + return value, true, nil + } + return text, true, nil +} + +func (c *Client) localMCPToolText(ctx context.Context, tool string, args map[string]any) (string, bool, error) { + endpoint, token, err := devonthinkMCPEndpointAndToken() + if err != nil { + return "", false, err + } + payload := map[string]any{ + "jsonrpc": "2.0", + "id": 1, + "method": "tools/call", + "params": map[string]any{ + "name": tool, + "arguments": args, + }, + } + data, err := json.Marshal(payload) + if err != nil { + return "", false, err + } + timeoutCtx, cancel := context.WithTimeout(ctx, 30*time.Second) + defer cancel() + req, err := http.NewRequestWithContext(timeoutCtx, http.MethodPost, endpoint, bytes.NewReader(data)) + if err != nil { + return "", false, err + } + req.Header.Set("Content-Type", "application/json") + req.Header.Set("Accept", "application/json, text/event-stream") + if token != "" { + req.Header.Set("Authorization", "Bearer "+token) + } + httpClient := c.HTTPClient + if httpClient == nil { + httpClient = &http.Client{Timeout: 30 * time.Second} + } + resp, err := httpClient.Do(req) + if err != nil { + return "", false, err + } + defer resp.Body.Close() + body, err := io.ReadAll(resp.Body) + if err != nil { + return "", false, err + } + if resp.StatusCode < 200 || resp.StatusCode >= 300 { + return "", false, fmt.Errorf("local MCP %s returned HTTP %d", tool, resp.StatusCode) + } + var envelope struct { + Result struct { + Content []struct { + Type string `json:"type"` + Text string `json:"text"` + } `json:"content"` + IsError bool `json:"isError"` + } `json:"result"` + Error *struct { + Code int `json:"code"` + Message string `json:"message"` + } `json:"error"` + } + if err := json.Unmarshal(body, &envelope); err != nil { + return "", false, err + } + if envelope.Error != nil { + return "", false, fmt.Errorf("local MCP %s error %d: %s", tool, envelope.Error.Code, envelope.Error.Message) + } + if len(envelope.Result.Content) == 0 { + return "", false, nil + } + text := envelope.Result.Content[0].Text + if envelope.Result.IsError { + return text, false, fmt.Errorf("local MCP %s failed: %s", tool, text) + } + return text, true, nil +} + +func (c *Client) localMCPListTools(ctx context.Context) (json.RawMessage, bool, error) { + endpoint, token, err := devonthinkMCPEndpointAndToken() + if err != nil { + return nil, false, err + } + payload := map[string]any{ + "jsonrpc": "2.0", + "id": 1, + "method": "tools/list", + "params": map[string]any{}, + } + data, err := json.Marshal(payload) + if err != nil { + return nil, false, err + } + timeoutCtx, cancel := context.WithTimeout(ctx, 30*time.Second) + defer cancel() + req, err := http.NewRequestWithContext(timeoutCtx, http.MethodPost, endpoint, bytes.NewReader(data)) + if err != nil { + return nil, false, err + } + req.Header.Set("Content-Type", "application/json") + req.Header.Set("Accept", "application/json, text/event-stream") + if token != "" { + req.Header.Set("Authorization", "Bearer "+token) + } + httpClient := c.HTTPClient + if httpClient == nil { + httpClient = &http.Client{Timeout: 30 * time.Second} + } + resp, err := httpClient.Do(req) + if err != nil { + return nil, false, err + } + defer resp.Body.Close() + body, err := io.ReadAll(resp.Body) + if err != nil { + return nil, false, err + } + if resp.StatusCode < 200 || resp.StatusCode >= 300 { + return nil, false, fmt.Errorf("local MCP tools/list returned HTTP %d", resp.StatusCode) + } + var envelope struct { + Result json.RawMessage `json:"result"` + Error *struct { + Code int `json:"code"` + Message string `json:"message"` + } `json:"error"` + } + if err := json.Unmarshal(body, &envelope); err != nil { + return nil, false, err + } + if envelope.Error != nil { + return nil, false, fmt.Errorf("local MCP tools/list error %d: %s", envelope.Error.Code, envelope.Error.Message) + } + return envelope.Result, true, nil +} + +func devonthinkVersion(ctx context.Context) (string, error) { + out, err := runOSA(ctx, `tell application id "`+devonthinkAppID+`" to get version`) + return strings.TrimSpace(out), err +} + +func devonthinkDatabaseNames(ctx context.Context) ([]string, error) { + out, err := runOSA(ctx, `tell application id "`+devonthinkAppID+`" to get name of databases`) + if err != nil { + return nil, err + } + return splitOSAList(out), nil +} + +func devonthinkSelectedRecordNames(ctx context.Context) ([]map[string]any, error) { + out, err := runOSA(ctx, `tell application id "`+devonthinkAppID+`" to get name of selected records`) + if err != nil { + return []map[string]any{}, err + } + names := splitOSAList(out) + records := make([]map[string]any, 0, len(names)) + for _, name := range names { + records = append(records, map[string]any{"name": name}) + } + return records, nil +} + +func runOSA(ctx context.Context, script string) (string, error) { + timeoutCtx, cancel := context.WithTimeout(ctx, 3*time.Second) + defer cancel() + cmd := exec.CommandContext(timeoutCtx, "osascript", "-e", script) // #nosec G204 -- command is fixed to local osascript; script strings are internal adapter snippets. + out, err := cmd.CombinedOutput() + if timeoutCtx.Err() != nil { + return "", timeoutCtx.Err() + } + if err != nil { + return strings.TrimSpace(string(out)), err + } + return string(out), nil +} + +func splitOSAList(out string) []string { + out = strings.TrimSpace(out) + if out == "" || out == "missing value" { + return []string{} + } + parts := strings.Split(out, ", ") + values := make([]string, 0, len(parts)) + for _, part := range parts { + part = strings.TrimSpace(part) + if part != "" { + values = append(values, part) + } + } + return values +} + +func devonthinkMCPToolsPath() (string, bool) { + candidates := []string{ + "/Applications/DEVONthink.app/Contents/Resources/mcp-tools.json", + "/Applications/DEVONthink.app/Contents/Library/LoginItems/DEVONthink MCP.app/Contents/Resources/mcp-tools.json", + } + for _, candidate := range candidates { + if _, err := os.Stat(candidate); err == nil { + return candidate, true + } + } + return "", false +} + +func devonthinkMCPEndpointAndToken() (string, string, error) { + home, err := os.UserHomeDir() + if err != nil { + return "", "", err + } + configPath := filepath.Join(home, "Library", "Application Support", "DEVONthink", "MCP", "config.json") + data, err := os.ReadFile(configPath) // #nosec G304 -- path is the fixed DEVONthink MCP config under the user's home directory. + if err != nil { + return "", "", err + } + var cfg struct { + Auth struct { + BearerToken string `json:"bearerToken"` + } `json:"auth"` + Server struct { + Port int `json:"port"` + Name string `json:"name"` + } `json:"server"` + } + if err := json.Unmarshal(data, &cfg); err != nil { + return "", "", err + } + port := cfg.Server.Port + if port == 0 { + port = 8420 + } + return "http://127.0.0.1:" + strconv.Itoa(port) + "/mcp", cfg.Auth.BearerToken, nil +} + +func writeLocalOutput(path string, data []byte) error { + if path == "-" { + return nil + } + if err := os.MkdirAll(filepath.Dir(path), 0o700); err != nil { + return err + } + return os.WriteFile(path, data, 0o600) +} + +func recordIDFromPath(path string) string { + rest := strings.TrimPrefix(path, "/records/") + if idx := strings.Index(rest, "/"); idx >= 0 { + return rest[:idx] + } + return rest +} + +func firstNonEmpty(values ...string) string { + for _, value := range values { + if strings.TrimSpace(value) != "" { + return value + } + } + return "" +} + +func intParam(params map[string]string, key string) (int, bool) { + if params == nil { + return 0, false + } + value := strings.TrimSpace(params[key]) + if value == "" { + return 0, false + } + parsed, err := strconv.Atoi(value) + if err != nil { + return 0, false + } + return parsed, true +} + +func localMCPToolLooksReadOnly(tool string) bool { + tool = strings.ToLower(strings.TrimSpace(tool)) + if tool == "" { + return false + } + for _, prefix := range []string{"get_", "list_", "search_", "extract_", "find_", "is_"} { + if strings.HasPrefix(tool, prefix) { + return true + } + } + switch tool { + case "lookup_records": + return true + } + return false +} + +func parseLooseMCPArgs(raw string) (map[string]any, bool) { + raw = strings.TrimSpace(raw) + if raw == "" { + return map[string]any{}, true + } + out := map[string]any{} + for _, part := range strings.Split(raw, ",") { + part = strings.TrimSpace(part) + if part == "" { + continue + } + sep := "=" + if !strings.Contains(part, sep) { + sep = ":" + } + key, value, ok := strings.Cut(part, sep) + if !ok { + return nil, false + } + key = strings.Trim(strings.TrimSpace(key), `"'`) + value = strings.Trim(strings.TrimSpace(value), `"'`) + if key == "" { + return nil, false + } + out[key] = coerceLooseMCPValue(value) + } + return out, true +} + +func coerceLooseMCPValue(value string) any { + if value == "" { + return "" + } + if parsed, err := strconv.Atoi(value); err == nil { + return parsed + } + if strings.EqualFold(value, "true") { + return true + } + if strings.EqualFold(value, "false") { + return false + } + return value +} + +func errorString(err error) string { + if err == nil { + return "" + } + return fmt.Sprint(err) +} diff --git a/library/productivity/devonthink/internal/client/smart_group_scope.go b/library/productivity/devonthink/internal/client/smart_group_scope.go new file mode 100644 index 0000000000..6db3a0a90f --- /dev/null +++ b/library/productivity/devonthink/internal/client/smart_group_scope.go @@ -0,0 +1,155 @@ +// Copyright 2026 rowdy and contributors. Licensed under Apache-2.0. See LICENSE. +// Generated by CLI Printing Press (https://github.com/mvanhorn/cli-printing-press). DO NOT EDIT. + +package client + +import ( + "context" + "encoding/json" + "fmt" + "strings" +) + +type SmartGroupScope struct { + Type string `json:"type"` + Input string `json:"input"` + UUID string `json:"uuid"` + Name string `json:"name,omitempty"` + Location string `json:"location,omitempty"` + DatabaseName string `json:"databaseName,omitempty"` +} + +type smartGroupCandidate struct { + UUID string + Name string + Location string + Kind string + Type string + DatabaseName string +} + +func (c *Client) ResolveSmartGroupScope(ctx context.Context, input string) (SmartGroupScope, error) { + input = strings.TrimSpace(input) + if input == "" { + return SmartGroupScope{}, fmt.Errorf("--smart-group cannot be empty") + } + groups, err := c.smartGroupCandidates(ctx) + if err != nil { + return SmartGroupScope{}, err + } + matches := matchSmartGroupCandidates(groups, input) + if len(matches) == 0 { + return SmartGroupScope{}, fmt.Errorf("no Smart Group found for %q", input) + } + if len(matches) > 1 { + return SmartGroupScope{}, fmt.Errorf("ambiguous Smart Group %q: %s", input, describeSmartGroupCandidates(matches)) + } + g := matches[0] + return SmartGroupScope{ + Type: "smart_group", + Input: input, + UUID: g.UUID, + Name: g.Name, + Location: g.Location, + DatabaseName: g.DatabaseName, + }, nil +} + +func (c *Client) smartGroupCandidates(ctx context.Context) ([]smartGroupCandidate, error) { + raw, ok, err := c.localMCPToolJSON(ctx, "search_records", map[string]any{ + "query": "kind:smartgroup", + "limit": 1000, + "fields": []string{"uuid", "name", "location", "kind", "type", "databaseName"}, + }) + if err != nil { + return nil, fmt.Errorf("resolving Smart Groups through DEVONthink MCP: %w", err) + } + if !ok { + return nil, fmt.Errorf("resolving Smart Groups requires the local DEVONthink MCP server") + } + var envelope struct { + Results []map[string]any `json:"results"` + } + var records []map[string]any + if json.Unmarshal(raw, &envelope) == nil && envelope.Results != nil { + records = envelope.Results + } else if err := json.Unmarshal(raw, &records); err != nil { + return nil, fmt.Errorf("parsing Smart Group search results: %w", err) + } + groups := make([]smartGroupCandidate, 0, len(records)) + for _, record := range records { + g := smartGroupCandidate{ + UUID: stringField(record, "uuid"), + Name: stringField(record, "name"), + Location: stringField(record, "location"), + Kind: strings.ToLower(stringField(record, "kind")), + Type: strings.ToLower(stringField(record, "type")), + DatabaseName: stringField(record, "databaseName"), + } + if g.UUID == "" { + continue + } + if g.Kind != "" && g.Kind != "smartgroup" && g.Kind != "smart_group" { + continue + } + groups = append(groups, g) + } + return groups, nil +} + +func stringField(record map[string]any, key string) string { + v, ok := record[key] + if !ok || v == nil { + return "" + } + switch typed := v.(type) { + case string: + return strings.TrimSpace(typed) + default: + return strings.TrimSpace(fmt.Sprint(typed)) + } +} + +func matchSmartGroupCandidates(groups []smartGroupCandidate, input string) []smartGroupCandidate { + var matches []smartGroupCandidate + normalizedInput := strings.TrimSpace(input) + for _, g := range groups { + if g.UUID == normalizedInput || strings.EqualFold(g.Name, normalizedInput) || matchesSmartGroupPath(g, normalizedInput) { + matches = append(matches, g) + } + } + return matches +} + +func matchesSmartGroupPath(g smartGroupCandidate, input string) bool { + input = strings.Trim(input, "/") + location := strings.Trim(g.Location, "/") + switch { + case input == "" || location == "": + return false + case input == location: + return true + case g.Name != "" && input == strings.Trim(location+"/"+g.Name, "/"): + return true + default: + return false + } +} + +func describeSmartGroupCandidates(groups []smartGroupCandidate) string { + parts := make([]string, 0, len(groups)) + for _, g := range groups { + label := g.UUID + if g.Name != "" { + label = g.Name + " (" + g.UUID + ")" + } + if g.Location != "" { + label += " at " + g.Location + } + if g.DatabaseName != "" { + label += " in " + g.DatabaseName + } + parts = append(parts, label) + } + return strings.Join(parts, "; ") +} diff --git a/library/productivity/devonthink/internal/client/smart_group_scope_test.go b/library/productivity/devonthink/internal/client/smart_group_scope_test.go new file mode 100644 index 0000000000..f0a3f15021 --- /dev/null +++ b/library/productivity/devonthink/internal/client/smart_group_scope_test.go @@ -0,0 +1,59 @@ +// Copyright 2026 rowdy and contributors. Licensed under Apache-2.0. See LICENSE. +// Generated by CLI Printing Press (https://github.com/mvanhorn/cli-printing-press). DO NOT EDIT. + +package client + +import ( + "strings" + "testing" +) + +func TestMatchSmartGroupCandidates(t *testing.T) { + t.Parallel() + groups := []smartGroupCandidate{ + {UUID: "A", Name: "Offene Rückerstattungen", Location: "Finanzen/Smart Groups", Kind: "smartgroup", DatabaseName: "Archive"}, + {UUID: "B", Name: "Waiting", Location: "Tasks"}, + } + + cases := []struct { + name string + input string + want string + }{ + {"uuid", "A", "A"}, + {"name", "Offene Rückerstattungen", "A"}, + {"location", "Tasks", "B"}, + {"location plus name", "Finanzen/Smart Groups/Offene Rückerstattungen", "A"}, + } + for _, tc := range cases { + tc := tc + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + got := matchSmartGroupCandidates(groups, tc.input) + if len(got) != 1 { + t.Fatalf("matches = %d, want 1", len(got)) + } + if got[0].UUID != tc.want { + t.Fatalf("uuid = %q, want %q", got[0].UUID, tc.want) + } + }) + } +} + +func TestMatchSmartGroupCandidatesAmbiguousName(t *testing.T) { + t.Parallel() + groups := []smartGroupCandidate{ + {UUID: "A", Name: "Waiting", Location: "DB One"}, + {UUID: "B", Name: "Waiting", Location: "DB Two"}, + } + got := matchSmartGroupCandidates(groups, "Waiting") + if len(got) != 2 { + t.Fatalf("matches = %d, want 2", len(got)) + } + desc := describeSmartGroupCandidates(got) + for _, want := range []string{"Waiting (A) at DB One", "Waiting (B) at DB Two"} { + if !strings.Contains(desc, want) { + t.Fatalf("description %q missing %q", desc, want) + } + } +} diff --git a/library/productivity/devonthink/internal/cliutil/cliutil_test.go b/library/productivity/devonthink/internal/cliutil/cliutil_test.go new file mode 100644 index 0000000000..e0147856e6 --- /dev/null +++ b/library/productivity/devonthink/internal/cliutil/cliutil_test.go @@ -0,0 +1,839 @@ +// Copyright 2026 rowdy and contributors. Licensed under Apache-2.0. See LICENSE. +// Generated by CLI Printing Press (https://github.com/mvanhorn/cli-printing-press). DO NOT EDIT. + +package cliutil + +import ( + "bytes" + "context" + "errors" + "fmt" + "net/http" + "net/http/httptest" + "sync" + "sync/atomic" + "testing" + "time" +) + +// ---- CleanText ---- + +func TestCleanText(t *testing.T) { + cases := []struct { + name string + in string + want string + }{ + {"decodes numeric entity", "The Food Lab's Cookie", "The Food Lab's Cookie"}, + {"decodes named entity", "AT&T", "AT&T"}, + {"trims whitespace", " Chicken Tikka ", "Chicken Tikka"}, + {"empty input", "", ""}, + {"plain passthrough", "Already clean.", "Already clean."}, + // Single-pass unescape contract: nested &amp; decodes once to & + // but the inner & stays encoded. If a caller needs repeated + // unescaping they have a deeper upstream problem. + {"single pass on nested entity", "&amp;", "&"}, + } + for _, tc := range cases { + tc := tc + t.Run(tc.name, func(t *testing.T) { + if got := CleanText(tc.in); got != tc.want { + t.Errorf("CleanText(%q) = %q, want %q", tc.in, got, tc.want) + } + }) + } +} + +func TestParseStoredTime(t *testing.T) { + cases := []struct { + name string + in string + want time.Time + }{ + { + name: "rfc3339 nano", + in: "2026-04-21T09:02:49.123456789-07:00", + want: time.Date(2026, 4, 21, 9, 2, 49, 123456789, time.FixedZone("", -7*60*60)), + }, + { + name: "modernc go string", + in: "2026-04-21 09:02:49.123456789 -0700 PDT", + want: time.Date(2026, 4, 21, 9, 2, 49, 123456789, time.FixedZone("PDT", -7*60*60)), + }, + { + name: "blank", + in: "", + want: time.Time{}, + }, + { + name: "invalid", + in: "not a time", + want: time.Time{}, + }, + } + for _, tc := range cases { + tc := tc + t.Run(tc.name, func(t *testing.T) { + got := ParseStoredTime(tc.in) + if !got.Equal(tc.want) { + t.Fatalf("ParseStoredTime(%q) = %s, want %s", tc.in, got, tc.want) + } + }) + } +} + +// ---- FanoutRun ---- + +func TestFanoutRunAllSucceed(t *testing.T) { + sources := []string{"a", "b", "c"} + results, errs := FanoutRun(context.Background(), sources, + func(s string) string { return s }, + func(_ context.Context, s string) (string, error) { + return s + "!", nil + }, + ) + if len(errs) != 0 { + t.Fatalf("unexpected errors: %+v", errs) + } + if len(results) != 3 { + t.Fatalf("want 3 results, got %d", len(results)) + } + // Source-order contract: results must match input order. + for i, r := range results { + if r.Source != sources[i] { + t.Errorf("results[%d].Source = %q, want %q", i, r.Source, sources[i]) + } + if r.Value != sources[i]+"!" { + t.Errorf("results[%d].Value = %q, want %q", i, r.Value, sources[i]+"!") + } + } +} + +func TestFanoutRunMixed(t *testing.T) { + sources := []string{"good", "bad", "good2"} + results, errs := FanoutRun(context.Background(), sources, + func(s string) string { return s }, + func(_ context.Context, s string) (string, error) { + if s == "bad" { + return "", errors.New("intentional failure") + } + return "ok-" + s, nil + }, + ) + if len(results) != 2 || len(errs) != 1 { + t.Fatalf("want 2 results + 1 error, got %d results + %d errors", len(results), len(errs)) + } + if errs[0].Source != "bad" { + t.Errorf("error source = %q, want bad", errs[0].Source) + } + // Results must stay in source order even with failure in the middle. + if results[0].Source != "good" || results[1].Source != "good2" { + t.Errorf("results out of source order: %q %q", results[0].Source, results[1].Source) + } +} + +func TestFanoutRunCancelledBeforeStart(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + cancel() // cancelled up front + + sources := []string{"a", "b", "c"} + results, errs := FanoutRun(ctx, sources, + func(s string) string { return s }, + func(_ context.Context, s string) (string, error) { + return s, nil + }, + ) + if len(results) != 0 { + t.Fatalf("want no results on pre-cancel, got %d", len(results)) + } + // Every source must report ctx.Err() — no silent drops. + if len(errs) != len(sources) { + t.Fatalf("want %d cancel errors, got %d", len(sources), len(errs)) + } + for i, e := range errs { + if e.Source != sources[i] { + t.Errorf("errs[%d].Source = %q, want %q", i, e.Source, sources[i]) + } + if !errors.Is(e.Err, context.Canceled) { + t.Errorf("errs[%d].Err = %v, want context.Canceled", i, e.Err) + } + } +} + +func TestFanoutRunBoundedConcurrency(t *testing.T) { + // Atomic counter wraps fn to verify max concurrent executions never + // exceeds WithConcurrency(n). Directly tests the bounded-channel + // contract without relying on runtime.NumGoroutine (too noisy). + // + // A sync.WaitGroup inside fn blocks each worker until concurrency-many + // workers have entered fn, guaranteeing the overlap needed to observe + // the max-inflight bound. A busy-wait would be flaky on fast or loaded + // hardware; the WaitGroup-based barrier makes the overlap deterministic. + var inflight int64 + var maxInflight int64 + var barrier sync.WaitGroup + barrier.Add(4) // require all 4 workers at the barrier before proceeding + + sources := make([]int, 100) + for i := range sources { + sources[i] = i + } + // Only the first 4 calls participate in the barrier so the test doesn't + // deadlock — remaining 96 just run normally and contribute to max-inflight + // observations. + var gated int64 + + _, errs := FanoutRun(context.Background(), sources, + func(i int) string { return fmt.Sprintf("src-%d", i) }, + func(_ context.Context, _ int) (struct{}, error) { + cur := atomic.AddInt64(&inflight, 1) + for { + m := atomic.LoadInt64(&maxInflight) + if cur <= m || atomic.CompareAndSwapInt64(&maxInflight, m, cur) { + break + } + } + if atomic.AddInt64(&gated, 1) <= 4 { + barrier.Done() + barrier.Wait() + } + atomic.AddInt64(&inflight, -1) + return struct{}{}, nil + }, + WithConcurrency(4), + ) + if len(errs) != 0 { + t.Fatalf("unexpected errors: %+v", errs) + } + if maxInflight > 4 { + t.Errorf("max in-flight = %d, want <= 4", maxInflight) + } + if maxInflight < 4 { + // Barrier forces all 4 workers into fn simultaneously. If this + // assertion ever fails the bounded-channel contract is broken. + t.Errorf("max in-flight = %d, want = 4 (barrier guarantees all 4 workers reach fn together)", maxInflight) + } +} + +func TestFanoutRunEmptySources(t *testing.T) { + results, errs := FanoutRun(context.Background(), []string{}, + func(s string) string { return s }, + func(_ context.Context, s string) (string, error) { return s, nil }, + ) + if len(results) != 0 { + t.Errorf("empty sources should produce 0 results, got %d", len(results)) + } + if len(errs) != 0 { + t.Errorf("empty sources should produce 0 errors, got %d", len(errs)) + } +} + +func TestFanoutRunAllFail(t *testing.T) { + sources := []string{"a", "b", "c"} + results, errs := FanoutRun(context.Background(), sources, + func(s string) string { return s }, + func(_ context.Context, _ string) (string, error) { + return "", errors.New("boom") + }, + ) + if len(results) != 0 { + t.Errorf("want 0 results when all fail, got %d", len(results)) + } + if len(errs) != 3 { + t.Fatalf("want 3 errors, got %d", len(errs)) + } + // Source-order preserved even on all-fail. + for i, e := range errs { + if e.Source != sources[i] { + t.Errorf("errs[%d].Source = %q, want %q", i, e.Source, sources[i]) + } + } +} + +func TestFanoutRunWithConcurrencyClampsZero(t *testing.T) { + // WithConcurrency(0) and WithConcurrency(-1) must clamp to 1, not deadlock. + for _, n := range []int{0, -1, -100} { + sources := []string{"a", "b"} + results, errs := FanoutRun(context.Background(), sources, + func(s string) string { return s }, + func(_ context.Context, s string) (string, error) { return s, nil }, + WithConcurrency(n), + ) + if len(results) != 2 { + t.Errorf("WithConcurrency(%d): want 2 results, got %d", n, len(results)) + } + if len(errs) != 0 { + t.Errorf("WithConcurrency(%d): unexpected errors: %+v", n, errs) + } + } +} + +func TestFanoutRunRecoversPanic(t *testing.T) { + // An fn that panics must not crash the process. The panicking source + // gets a FanoutError; other sources complete normally. + sources := []string{"good1", "panic", "good2"} + results, errs := FanoutRun(context.Background(), sources, + func(s string) string { return s }, + func(_ context.Context, s string) (string, error) { + if s == "panic" { + panic("oops") + } + return "ok-" + s, nil + }, + ) + if len(results) != 2 { + t.Fatalf("want 2 results (good1, good2), got %d", len(results)) + } + if len(errs) != 1 { + t.Fatalf("want 1 error (panic), got %d", len(errs)) + } + if errs[0].Source != "panic" { + t.Errorf("panic error source = %q, want panic", errs[0].Source) + } + if errs[0].Err == nil || !containsString(errs[0].Err.Error(), "oops") { + t.Errorf("want panic error mentioning 'oops', got %v", errs[0].Err) + } +} + +func TestFanoutRunCancelMidFlight(t *testing.T) { + // Regression test: cancel while workers are mid-fn. Producer may still + // be feeding the bounded channel; some workers are executing fn; others + // may have already pulled the next job. The contract: every source ends + // up with either a result or an error, never neither and never both. + // Run with -race to catch any slot-write race that a naïve implementation + // would introduce. + ctx, cancel := context.WithCancel(context.Background()) + sources := make([]int, 30) + for i := range sources { + sources[i] = i + } + // Fire cancel after a few fn calls start. + var started int64 + results, errs := FanoutRun(ctx, sources, + func(i int) string { return fmt.Sprintf("src-%d", i) }, + func(c context.Context, _ int) (struct{}, error) { + if atomic.AddInt64(&started, 1) == 3 { + cancel() + } + // Brief wait so cancel can propagate and the bounded channel's + // drain/feed interaction actually exercises mid-flight state. + for j := 0; j < 10000; j++ { + _ = j + } + return struct{}{}, c.Err() + }, + ) + // Every source must be accounted for exactly once. + if total := len(results) + len(errs); total != len(sources) { + t.Errorf("results+errs = %d, want %d (every source accounted once)", total, len(sources)) + } + // No double-accounting: a source must not appear in both. + seen := map[string]int{} + for _, r := range results { + seen[r.Source]++ + } + for _, e := range errs { + seen[e.Source]++ + } + for src, n := range seen { + if n != 1 { + t.Errorf("source %q accounted %d times, want exactly 1", src, n) + } + } +} + +// containsString is a tiny strings.Contains alias so the test file only +// imports the stdlib packages it otherwise uses. +func containsString(haystack, needle string) bool { + for i := 0; i+len(needle) <= len(haystack); i++ { + if haystack[i:i+len(needle)] == needle { + return true + } + } + return false +} + +func TestFanoutRunSerialWithConcurrency1(t *testing.T) { + // Serial execution: completion order must equal source order because + // there's only one worker. Double-checks the source-order contract. + sources := []string{"a", "b", "c"} + var completionOrder []string + var mu sync.Mutex + + results, _ := FanoutRun(context.Background(), sources, + func(s string) string { return s }, + func(_ context.Context, s string) (string, error) { + mu.Lock() + completionOrder = append(completionOrder, s) + mu.Unlock() + return s, nil + }, + WithConcurrency(1), + ) + for i, r := range results { + if r.Source != sources[i] { + t.Errorf("results[%d].Source = %q, want %q", i, r.Source, sources[i]) + } + } + for i, s := range completionOrder { + if s != sources[i] { + t.Errorf("serial completion order[%d] = %q, want %q", i, s, sources[i]) + } + } +} + +// ---- FanoutReportErrors ---- + +func TestFanoutReportErrorsOrder(t *testing.T) { + errs := []FanoutError{ + {Source: "alpha", Err: errors.New("boom")}, + {Source: "beta", Err: errors.New("crash\nsecond line")}, + } + var buf bytes.Buffer + FanoutReportErrors(&buf, errs) + want := "warn: alpha: boom\nwarn: beta: crash\n" + if got := buf.String(); got != want { + t.Errorf("output mismatch.\n got: %q\nwant: %q", got, want) + } +} + +func TestFanoutReportErrorsEmpty(t *testing.T) { + var buf bytes.Buffer + FanoutReportErrors(&buf, nil) + if buf.Len() != 0 { + t.Errorf("expected no output for empty errs, got %q", buf.String()) + } +} + +func TestFanoutReportErrorsTruncates(t *testing.T) { + // A long single-line error gets truncated to 120 chars + ellipsis so + // stderr stays scan-able when 14 sources all fail with verbose messages. + longMsg := "" + for i := 0; i < 200; i++ { + longMsg += "x" + } + errs := []FanoutError{ + {Source: "verbose", Err: errors.New(longMsg)}, + } + var buf bytes.Buffer + FanoutReportErrors(&buf, errs) + out := buf.String() + // "warn: verbose: " is 15 chars; body must be 120 chars + "…" + "\n" + if !containsString(out, "…") { + t.Errorf("expected truncation ellipsis in output, got %q", out) + } + if len(out) > 160 { + t.Errorf("truncated line should be ~140 chars, got %d (%q)", len(out), out) + } +} + +// ---- ProbeReachable ---- + +// TestProbeReachable_200 asserts that a plain 2xx GET is classified +// reachable with the right code. +func TestProbeReachable_200(t *testing.T) { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusOK) + _, _ = w.Write([]byte("hello")) + })) + defer srv.Close() + + status, code, err := ProbeReachable(context.Background(), srv.Client(), srv.URL) + if err != nil { + t.Fatalf("unexpected err: %v", err) + } + if status != ReachabilityReachable { + t.Errorf("status: want %q, got %q", ReachabilityReachable, status) + } + if code != 200 { + t.Errorf("code: want 200, got %d", code) + } +} + +// TestProbeReachable_206_Reachable asserts hosts that honor Range +// (returning 206 Partial Content) are classified reachable. +func TestProbeReachable_206_Reachable(t *testing.T) { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusPartialContent) + })) + defer srv.Close() + + status, code, err := ProbeReachable(context.Background(), srv.Client(), srv.URL) + if err != nil { + t.Fatalf("unexpected err: %v", err) + } + if status != ReachabilityReachable { + t.Errorf("status: want reachable, got %q", status) + } + if code != 206 { + t.Errorf("code: want 206, got %d", code) + } +} + +// TestProbeReachable_416_Reachable asserts hosts that don't support +// Range (returning 416 Range Not Satisfiable) are still reachable — +// the headers came back, the host is up. This is the F4 motivating +// case: HEAD-then-GET probes incorrectly report unreachable here. +func TestProbeReachable_416_Reachable(t *testing.T) { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusRequestedRangeNotSatisfiable) + })) + defer srv.Close() + + status, code, err := ProbeReachable(context.Background(), srv.Client(), srv.URL) + if err != nil { + t.Fatalf("unexpected err: %v", err) + } + if status != ReachabilityReachable { + t.Errorf("status: want reachable (416 means headers came back), got %q", status) + } + if code != 416 { + t.Errorf("code: want 416, got %d", code) + } +} + +// TestProbeReachable_403_Blocked asserts CDN bot screens (4xx other +// than 416) are classified blocked, not unreachable. The host is up +// and refusing this request — a doctor command should render WARN +// rather than FAIL. +func TestProbeReachable_403_Blocked(t *testing.T) { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusForbidden) + })) + defer srv.Close() + + status, code, err := ProbeReachable(context.Background(), srv.Client(), srv.URL) + if err != nil { + t.Fatalf("unexpected err: %v", err) + } + if status != ReachabilityBlocked { + t.Errorf("status: want blocked, got %q", status) + } + if code != 403 { + t.Errorf("code: want 403, got %d", code) + } +} + +// TestProbeReachable_NetworkError_Unreachable asserts network-layer +// failures (DNS, connection refused, timeout) report unreachable with +// a non-nil err. +func TestProbeReachable_NetworkError_Unreachable(t *testing.T) { + // Use a port that nothing is listening on. + status, code, err := ProbeReachable(context.Background(), http.DefaultClient, "http://127.0.0.1:1") + if err == nil { + t.Fatal("expected non-nil err for unreachable host") + } + if status != ReachabilityUnreachable { + t.Errorf("status: want unreachable, got %q", status) + } + if code != 0 { + t.Errorf("code: want 0 (no response), got %d", code) + } +} + +// TestProbeReachable_NilClient_UsesDefault confirms the nil-client +// guard so doctor commands don't have to plumb an explicit *http.Client +// when default behavior is fine. +func TestProbeReachable_NilClient_UsesDefault(t *testing.T) { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusOK) + })) + defer srv.Close() + + status, _, err := ProbeReachable(context.Background(), nil, srv.URL) + if err != nil { + t.Fatalf("unexpected err: %v", err) + } + if status != ReachabilityReachable { + t.Errorf("status: want reachable, got %q", status) + } +} + +// TestProbeReachable_NilClient_HasTimeout asserts the nil-client +// fallback uses a bounded-timeout client rather than http.DefaultClient +// (which has no timeout). Without this, a probe against a slow host +// could hang indefinitely. The test starts a server that hangs forever +// and relies on the default 10s timeout to bail out — capped to 12s +// total so a regression that drops the timeout would surface as a +// test failure rather than a hung test. +func TestProbeReachable_NilClient_HasTimeout(t *testing.T) { + if testing.Short() { + t.Skip("skip slow timeout test in -short mode") + } + hang := make(chan struct{}) + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + <-hang // never returns until t.Cleanup closes it + })) + t.Cleanup(func() { + close(hang) + srv.Close() + }) + + done := make(chan struct{}) + var status string + var probeErr error + go func() { + status, _, probeErr = ProbeReachable(context.Background(), nil, srv.URL) + close(done) + }() + select { + case <-done: + // Probe returned within the bounded timeout — expected. + if probeErr == nil { + t.Fatalf("expected timeout err, got nil") + } + if status != ReachabilityUnreachable { + t.Errorf("status: want unreachable on timeout, got %q", status) + } + case <-time.After(12 * time.Second): + t.Fatalf("ProbeReachable hung past defaultProbeTimeout — nil-client fallback may be missing its bounded-timeout guard") + } +} + +// TestProbeReachable_SendsRangeHeader confirms the probe sends +// `Range: bytes=0-1023` so hosts that support Range bound the +// response body before we even read it. +func TestProbeReachable_SendsRangeHeader(t *testing.T) { + var receivedRange string + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + receivedRange = r.Header.Get("Range") + w.WriteHeader(http.StatusOK) + })) + defer srv.Close() + + _, _, err := ProbeReachable(context.Background(), srv.Client(), srv.URL) + if err != nil { + t.Fatalf("unexpected err: %v", err) + } + if receivedRange != "bytes=0-1023" { + t.Errorf("Range header: want %q, got %q", "bytes=0-1023", receivedRange) + } +} + +// ---- AdaptiveLimiter / RateLimitError / RetryAfter / Backoff ---- + +func TestAdaptiveLimiter_NewNilOnNonPositive(t *testing.T) { + if NewAdaptiveLimiter(0) != nil { + t.Fatal("NewAdaptiveLimiter(0) should return nil") + } + if NewAdaptiveLimiter(-1) != nil { + t.Fatal("NewAdaptiveLimiter(-1) should return nil") + } +} + +func TestAdaptiveLimiter_NilSafeMethods(t *testing.T) { + var l *AdaptiveLimiter + l.Wait() + l.OnSuccess() + l.OnRateLimit() + if got := l.Rate(); got != 0 { + t.Errorf("nil limiter Rate() = %v, want 0", got) + } +} + +func TestAdaptiveLimiter_RampsUpAfterSuccesses(t *testing.T) { + l := NewAdaptiveLimiter(2.0) + startRate := l.Rate() + for i := 0; i < l.rampAfter; i++ { + l.OnSuccess() + } + if got := l.Rate(); got <= startRate { + t.Errorf("Rate() after rampAfter successes = %v, want > %v", got, startRate) + } +} + +func TestAdaptiveLimiter_HalvesOnRateLimit(t *testing.T) { + l := NewAdaptiveLimiter(8.0) + startRate := l.Rate() + l.OnRateLimit() + got := l.Rate() + if got != startRate/2 { + t.Errorf("Rate() after OnRateLimit = %v, want %v", got, startRate/2) + } +} + +func TestAdaptiveLimiter_FloorsAtHalfRPS(t *testing.T) { + l := NewAdaptiveLimiter(2.0) + for i := 0; i < 10; i++ { + l.OnRateLimit() + } + if got := l.Rate(); got < 0.5 { + t.Errorf("Rate() after many OnRateLimit = %v, want >= 0.5", got) + } +} + +func TestAdaptiveLimiter_DoesNotRaiseSubFloorRateOnRateLimit(t *testing.T) { + l := NewAdaptiveLimiter(0.3) + startRate := l.Rate() + l.OnRateLimit() + if got := l.Rate(); got > startRate { + t.Errorf("Rate() after OnRateLimit = %v, want <= %v", got, startRate) + } +} + +func TestAdaptiveLimiter_DoesNotRampBelowFloorAfterRateLimit(t *testing.T) { + l := NewAdaptiveLimiter(0.3) + floor := l.Rate() + l.OnRateLimit() + for i := 0; i < l.rampAfter; i++ { + l.OnSuccess() + } + if got := l.Rate(); got < floor { + t.Errorf("Rate() after ramping from rate-limit ceiling = %v, want >= %v", got, floor) + } +} + +func TestAdaptiveLimiter_WaitEnforcesPacing(t *testing.T) { + l := NewAdaptiveLimiter(10.0) + l.Wait() + start := time.Now() + l.Wait() + elapsed := time.Since(start) + if elapsed < 80*time.Millisecond { + t.Errorf("second Wait() took %v, want >= 80ms", elapsed) + } +} + +func TestRateLimitError_ErrorMessage(t *testing.T) { + cases := []struct { + name string + err *RateLimitError + want string + }{ + { + name: "with retry-after and body", + err: &RateLimitError{URL: "https://api.example.com/x", RetryAfter: 5 * time.Second, Body: "slow down"}, + want: "rate limited: HTTP 429 for https://api.example.com/x; retry after 5s: slow down", + }, + { + name: "with retry-after no body", + err: &RateLimitError{URL: "https://api.example.com/x", RetryAfter: 2 * time.Second}, + want: "rate limited: HTTP 429 for https://api.example.com/x; retry after 2s", + }, + { + name: "no retry-after with body", + err: &RateLimitError{URL: "https://api.example.com/x", Body: "later"}, + want: "rate limited: HTTP 429 for https://api.example.com/x: later", + }, + { + name: "no retry-after no body", + err: &RateLimitError{URL: "https://api.example.com/x"}, + want: "rate limited: HTTP 429 for https://api.example.com/x", + }, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + if got := tc.err.Error(); got != tc.want { + t.Errorf("Error() = %q, want %q", got, tc.want) + } + }) + } +} + +func TestRateLimitError_ErrorsAs(t *testing.T) { + var err error = &RateLimitError{URL: "https://x", RetryAfter: time.Second} + var target *RateLimitError + if !errors.As(err, &target) { + t.Fatal("errors.As should match *RateLimitError") + } + if target.URL != "https://x" { + t.Errorf("target.URL = %q, want %q", target.URL, "https://x") + } +} + +func TestRetryAfter_Seconds(t *testing.T) { + resp := &http.Response{Header: http.Header{}} + resp.Header.Set("Retry-After", "10") + if got := RetryAfter(resp); got != 10*time.Second { + t.Errorf("RetryAfter(10) = %v, want 10s", got) + } +} + +func TestRetryAfter_HTTPDate(t *testing.T) { + future := time.Now().Add(7 * time.Second).UTC().Format(http.TimeFormat) + resp := &http.Response{Header: http.Header{}} + resp.Header.Set("Retry-After", future) + got := RetryAfter(resp) + if got < 5*time.Second || got > 8*time.Second { + t.Errorf("RetryAfter(http-date 7s ahead) = %v, want ~7s", got) + } +} + +func TestRetryAfter_EpochSeconds(t *testing.T) { + future := time.Now().Add(7 * time.Second) + resp := &http.Response{Header: http.Header{}} + resp.Header.Set("Retry-After", fmt.Sprint(future.Unix())) + got := RetryAfter(resp) + if got < 5*time.Second || got > 8*time.Second { + t.Errorf("RetryAfter(epoch seconds 7s ahead) = %v, want ~7s", got) + } +} + +func TestRetryAfter_EpochMilliseconds(t *testing.T) { + future := time.Now().Add(7 * time.Second) + resp := &http.Response{Header: http.Header{}} + resp.Header.Set("Retry-After", fmt.Sprint(future.UnixMilli())) + got := RetryAfter(resp) + if got < 5*time.Second || got > 8*time.Second { + t.Errorf("RetryAfter(epoch milliseconds 7s ahead) = %v, want ~7s", got) + } +} + +func TestRetryAfter_Cap(t *testing.T) { + resp := &http.Response{Header: http.Header{}} + resp.Header.Set("Retry-After", "600") + if got := RetryAfter(resp); got != MaxRetryWait { + t.Errorf("RetryAfter(600) = %v, want capped at %v", got, MaxRetryWait) + } +} + +func TestRetryAfter_Missing(t *testing.T) { + resp := &http.Response{Header: http.Header{}} + if got := RetryAfter(resp); got != 5*time.Second { + t.Errorf("RetryAfter(missing) = %v, want 5s default", got) + } +} + +func TestRetryAfter_MalformedFallsBackToDefault(t *testing.T) { + resp := &http.Response{Header: http.Header{}} + resp.Header.Set("Retry-After", "not-a-number") + if got := RetryAfter(resp); got != 5*time.Second { + t.Errorf("RetryAfter(garbage) = %v, want 5s default", got) + } +} + +func TestRetryAfter_NilResp(t *testing.T) { + if got := RetryAfter(nil); got != 5*time.Second { + t.Errorf("RetryAfter(nil) = %v, want 5s default", got) + } +} + +func TestBackoff_DoublesPerAttempt(t *testing.T) { + cases := []struct { + attempt int + want time.Duration + }{ + {0, 1 * time.Second}, + {1, 2 * time.Second}, + {2, 4 * time.Second}, + {3, 8 * time.Second}, + {4, 16 * time.Second}, + } + for _, tc := range cases { + if got := Backoff(tc.attempt); got != tc.want { + t.Errorf("Backoff(%d) = %v, want %v", tc.attempt, got, tc.want) + } + } +} + +func TestBackoff_CapsAtMax(t *testing.T) { + if got := Backoff(20); got != MaxBackoff { + t.Errorf("Backoff(20) = %v, want capped at %v", got, MaxBackoff) + } +} + +func TestBackoff_NegativeAttemptClampsToZero(t *testing.T) { + if got := Backoff(-3); got != 1*time.Second { + t.Errorf("Backoff(-3) = %v, want 1s (clamped to 0)", got) + } +} diff --git a/library/productivity/devonthink/internal/cliutil/duration.go b/library/productivity/devonthink/internal/cliutil/duration.go new file mode 100644 index 0000000000..f7fb970a92 --- /dev/null +++ b/library/productivity/devonthink/internal/cliutil/duration.go @@ -0,0 +1,41 @@ +// Copyright 2026 rowdy and contributors. Licensed under Apache-2.0. See LICENSE. +// Generated by CLI Printing Press (https://github.com/mvanhorn/cli-printing-press). DO NOT EDIT. + +package cliutil + +import ( + "strconv" + "strings" + "time" +) + +// ParseDurationLoose parses a duration string, extending Go's +// time.ParseDuration with day ("d") and week ("w") suffixes so hand-coded +// time-window flags accept the same 7d/30d/1w/4w shorthand the framework's +// `sync --since` advertises. Everything else (h, m, s, ms, us, ns, and +// compound forms like "1h30m") falls through to time.ParseDuration unchanged. +// +// The day/week suffix takes an integer count (e.g. "7d", "-2w"); fractional +// windows should use a stdlib unit ("36h" rather than "1.5d"). Hand-coded +// duration flags should declare a StringVar (not DurationVar) and post-parse +// with this helper. +func ParseDurationLoose(s string) (time.Duration, error) { + trimmed := strings.TrimSpace(s) + if n := len(trimmed); n >= 2 { + switch unit := trimmed[n-1]; unit { + case 'd', 'w': + // No stdlib duration unit ends in 'd' or 'w', so a trailing + // d/w is unambiguously our extension. If the count is not a + // plain integer, fall through to time.ParseDuration to surface + // its standard error. + if count, err := strconv.ParseInt(trimmed[:n-1], 10, 64); err == nil { + hoursPerUnit := int64(24) + if unit == 'w' { + hoursPerUnit = 24 * 7 + } + return time.Duration(count) * time.Hour * time.Duration(hoursPerUnit), nil + } + } + } + return time.ParseDuration(trimmed) +} diff --git a/library/productivity/devonthink/internal/cliutil/duration_test.go b/library/productivity/devonthink/internal/cliutil/duration_test.go new file mode 100644 index 0000000000..8e39470182 --- /dev/null +++ b/library/productivity/devonthink/internal/cliutil/duration_test.go @@ -0,0 +1,56 @@ +// Copyright 2026 rowdy and contributors. Licensed under Apache-2.0. See LICENSE. +// Generated by CLI Printing Press (https://github.com/mvanhorn/cli-printing-press). DO NOT EDIT. + +package cliutil + +import ( + "testing" + "time" +) + +func TestParseDurationLoose(t *testing.T) { + t.Parallel() + + cases := []struct { + name string + input string + want time.Duration + wantErr bool + }{ + {"days", "7d", 7 * 24 * time.Hour, false}, + {"thirty days", "30d", 30 * 24 * time.Hour, false}, + {"two weeks window", "14d", 14 * 24 * time.Hour, false}, + {"weeks", "4w", 4 * 7 * 24 * time.Hour, false}, + {"one week", "1w", 7 * 24 * time.Hour, false}, + {"zero days", "0d", 0, false}, + {"negative days", "-2d", -2 * 24 * time.Hour, false}, + {"surrounding spaces", " 7d ", 7 * 24 * time.Hour, false}, + {"stdlib hours", "24h", 24 * time.Hour, false}, + {"stdlib minutes", "30m", 30 * time.Minute, false}, + {"stdlib compound", "1h30m", 90 * time.Minute, false}, + {"stdlib millis", "500ms", 500 * time.Millisecond, false}, + {"fractional day falls through to stdlib error", "1.5d", 0, true}, + {"bare unit", "d", 0, true}, + {"empty", "", 0, true}, + {"garbage", "abc", 0, true}, + } + + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + got, err := ParseDurationLoose(tc.input) + if tc.wantErr { + if err == nil { + t.Fatalf("ParseDurationLoose(%q) = %v, want error", tc.input, got) + } + return + } + if err != nil { + t.Fatalf("ParseDurationLoose(%q) unexpected error: %v", tc.input, err) + } + if got != tc.want { + t.Fatalf("ParseDurationLoose(%q) = %v, want %v", tc.input, got, tc.want) + } + }) + } +} diff --git a/library/productivity/devonthink/internal/cliutil/extractnumber.go b/library/productivity/devonthink/internal/cliutil/extractnumber.go new file mode 100644 index 0000000000..1d15198f86 --- /dev/null +++ b/library/productivity/devonthink/internal/cliutil/extractnumber.go @@ -0,0 +1,67 @@ +// Copyright 2026 rowdy and contributors. Licensed under Apache-2.0. See LICENSE. +// Generated by CLI Printing Press (https://github.com/mvanhorn/cli-printing-press). DO NOT EDIT. + +package cliutil + +import ( + "encoding/json" + "strconv" +) + +// ExtractNumber reads a numeric value from a json.RawMessage map, accepting +// either a JSON number (1.91) or a JSON-encoded string ("1.91"). Returns +// (value, ok). ok=false when the key is missing, JSON null, an empty raw +// payload, an empty string, or unparseable as a float64. +// +// Why this exists: many streaming and market-data APIs (Binance, Coinbase, +// Kraken, Stripe *_decimal fields, vendor-specific WebSocket feeds) encode +// numeric values as JSON-encoded strings. json.Unmarshal of "1.91" into a +// float64 struct field silently zeroes the field — no error is returned. +// Combined with a NULL-on-zero pattern (common for SQL aggregation +// correctness) every such field becomes a SQL NULL and downstream +// aggregations return empty. The failure is silent at every structural +// test layer; only frame-level inspection reveals it. +// +// Prefer this helper over hand-rolled float64 struct fields when authoring +// streaming frame normalizers or any decoder whose wire shape is not +// fully fixed by the spec. +func ExtractNumber(probe map[string]json.RawMessage, key string) (float64, bool) { + raw, present := probe[key] + if !present || len(raw) == 0 || string(raw) == "null" { + return 0, false + } + var f float64 + if err := json.Unmarshal(raw, &f); err == nil { + return f, true + } + var s string + if err := json.Unmarshal(raw, &s); err == nil && s != "" { + if v, perr := strconv.ParseFloat(s, 64); perr == nil { + return v, true + } + } + return 0, false +} + +// ExtractInt is the integer companion of ExtractNumber. It accepts a JSON +// integer (123) or a JSON-encoded integer string ("123"). Returns +// (value, ok). ok=false when the key is missing, null, empty, or +// unparseable as an int64. A non-integer JSON number (1.5) is rejected +// rather than truncated. +func ExtractInt(probe map[string]json.RawMessage, key string) (int64, bool) { + raw, present := probe[key] + if !present || len(raw) == 0 || string(raw) == "null" { + return 0, false + } + var n int64 + if err := json.Unmarshal(raw, &n); err == nil { + return n, true + } + var s string + if err := json.Unmarshal(raw, &s); err == nil && s != "" { + if v, perr := strconv.ParseInt(s, 10, 64); perr == nil { + return v, true + } + } + return 0, false +} diff --git a/library/productivity/devonthink/internal/cliutil/extractnumber_test.go b/library/productivity/devonthink/internal/cliutil/extractnumber_test.go new file mode 100644 index 0000000000..6626b37ac6 --- /dev/null +++ b/library/productivity/devonthink/internal/cliutil/extractnumber_test.go @@ -0,0 +1,117 @@ +// Copyright 2026 rowdy and contributors. Licensed under Apache-2.0. See LICENSE. +// Generated by CLI Printing Press (https://github.com/mvanhorn/cli-printing-press). DO NOT EDIT. + +package cliutil + +import ( + "encoding/json" + "math" + "testing" +) + +func TestExtractNumber(t *testing.T) { + t.Parallel() + + cases := []struct { + name string + input string + key string + wantV float64 + wantOk bool + }{ + {"json number", `{"price":1.91}`, "price", 1.91, true}, + {"json integer", `{"price":42}`, "price", 42, true}, + {"json zero", `{"price":0}`, "price", 0, true}, + {"json negative number", `{"price":-3.5}`, "price", -3.5, true}, + {"string-encoded number", `{"price":"1.91"}`, "price", 1.91, true}, + {"string-encoded integer", `{"qty":"100"}`, "qty", 100, true}, + {"string-encoded negative", `{"x":"-3.5"}`, "x", -3.5, true}, + {"string-encoded exponent", `{"x":"1e2"}`, "x", 100, true}, + {"null", `{"price":null}`, "price", 0, false}, + {"missing key", `{}`, "price", 0, false}, + {"empty string", `{"price":""}`, "price", 0, false}, + {"unparseable string", `{"price":"abc"}`, "price", 0, false}, + {"bool true", `{"price":true}`, "price", 0, false}, + {"object", `{"price":{"a":1}}`, "price", 0, false}, + {"array", `{"price":[1,2]}`, "price", 0, false}, + } + + for _, tc := range cases { + tc := tc + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + var probe map[string]json.RawMessage + if err := json.Unmarshal([]byte(tc.input), &probe); err != nil { + t.Fatalf("unmarshal probe: %v", err) + } + got, ok := ExtractNumber(probe, tc.key) + if ok != tc.wantOk { + t.Fatalf("ExtractNumber ok = %v, want %v", ok, tc.wantOk) + } + if ok && math.Abs(got-tc.wantV) > 1e-9 { + t.Fatalf("ExtractNumber value = %v, want %v", got, tc.wantV) + } + }) + } +} + +func TestExtractNumber_NilProbe(t *testing.T) { + t.Parallel() + + got, ok := ExtractNumber(nil, "anything") + if ok || got != 0 { + t.Fatalf("ExtractNumber(nil, ...) = (%v, %v), want (0, false)", got, ok) + } +} + +func TestExtractInt(t *testing.T) { + t.Parallel() + + cases := []struct { + name string + input string + key string + wantV int64 + wantOk bool + }{ + {"json integer", `{"id":123}`, "id", 123, true}, + {"json zero", `{"id":0}`, "id", 0, true}, + {"json negative", `{"id":-5}`, "id", -5, true}, + {"string-encoded int", `{"id":"123"}`, "id", 123, true}, + {"string-encoded negative", `{"id":"-5"}`, "id", -5, true}, + {"null", `{"id":null}`, "id", 0, false}, + {"missing key", `{}`, "id", 0, false}, + {"empty string", `{"id":""}`, "id", 0, false}, + {"unparseable string", `{"id":"abc"}`, "id", 0, false}, + {"float rejected", `{"id":1.5}`, "id", 0, false}, + {"float string rejected", `{"id":"1.5"}`, "id", 0, false}, + {"bool", `{"id":true}`, "id", 0, false}, + } + + for _, tc := range cases { + tc := tc + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + var probe map[string]json.RawMessage + if err := json.Unmarshal([]byte(tc.input), &probe); err != nil { + t.Fatalf("unmarshal probe: %v", err) + } + got, ok := ExtractInt(probe, tc.key) + if ok != tc.wantOk { + t.Fatalf("ExtractInt ok = %v, want %v", ok, tc.wantOk) + } + if ok && got != tc.wantV { + t.Fatalf("ExtractInt value = %v, want %v", got, tc.wantV) + } + }) + } +} + +func TestExtractInt_NilProbe(t *testing.T) { + t.Parallel() + + got, ok := ExtractInt(nil, "anything") + if ok || got != 0 { + t.Fatalf("ExtractInt(nil, ...) = (%v, %v), want (0, false)", got, ok) + } +} diff --git a/library/productivity/devonthink/internal/cliutil/fanout.go b/library/productivity/devonthink/internal/cliutil/fanout.go new file mode 100644 index 0000000000..b4c85c5b8d --- /dev/null +++ b/library/productivity/devonthink/internal/cliutil/fanout.go @@ -0,0 +1,202 @@ +// Copyright 2026 rowdy and contributors. Licensed under Apache-2.0. See LICENSE. +// Generated by CLI Printing Press (https://github.com/mvanhorn/cli-printing-press). DO NOT EDIT. + +// Package cliutil contains shared helpers emitted into every generated CLI +// by the Printing Press. Helpers live in their own package (not in package +// cli) to avoid symbol collisions with agent-authored commands in package +// cli. Callers import as `cliutil` and invoke `cliutil.FanoutRun(...)`, +// `cliutil.CleanText(...)`, etc. +package cliutil + +import ( + "context" + "fmt" + "io" + "strings" + "sync" +) + +// FanoutError represents one source's failure from a FanoutRun call. +// Source identifies which input produced the error; Err is the error returned +// by the caller's fn. +type FanoutError struct { + Source string + Err error +} + +// FanoutResult pairs a successful fn return value with its source name so +// callers can iterate results without a separate source lookup. +type FanoutResult[T any] struct { + Source string + Value T +} + +// FanoutOption configures a FanoutRun call. Use the With* constructors. +type FanoutOption func(*fanoutOptions) + +type fanoutOptions struct { + concurrency int +} + +// defaultFanoutConcurrency is the worker count when the caller passes no +// WithConcurrency option. 4 matches the existing sync.go worker-pool idiom +// and is safe for scraping CLIs where per-host 429 pressure is real. +const defaultFanoutConcurrency = 4 + +// WithConcurrency overrides the default worker count for a single FanoutRun +// call. Use higher values for fan-outs without external rate limits; keep +// the default (4) for scraping CLIs. Values below 1 are clamped to 1. +func WithConcurrency(n int) FanoutOption { + return func(o *fanoutOptions) { + if n < 1 { + n = 1 + } + o.concurrency = n + } +} + +// FanoutRun invokes fn concurrently for each source and collects successful +// results plus per-source errors. It never returns a top-level error and +// recovers panics from fn as per-source FanoutErrors — partial failures +// surface via the returned errors slice, which should be piped to +// FanoutReportErrors so no source is silently dropped. +// +// Contract: +// - Workers respect ctx: on ctx.Done() they stop pulling new jobs, and +// in-flight fn calls receive the cancelled ctx. +// - Unpulled sources produce a FanoutError{Err: ctx.Err()} so reporting +// stays complete — cancel never silently drops a source. +// - Errors are collected by source index and returned in source order, +// not completion order, so FanoutReportErrors output is deterministic +// across runs. +// - The jobs channel is bounded at 2*concurrency so large source lists +// don't buffer one goroutine per source. +// +// Per-source rate limiting is the caller's responsibility. Wrap fn with a +// limiter (e.g., golang.org/x/time/rate) if you're fanning out to sites +// that enforce per-host throttles; naïve scrape fan-out triggers 429s. +func FanoutRun[S, T any]( + ctx context.Context, + sources []S, + name func(S) string, + fn func(context.Context, S) (T, error), + opts ...FanoutOption, +) ([]FanoutResult[T], []FanoutError) { + cfg := fanoutOptions{concurrency: defaultFanoutConcurrency} + for _, o := range opts { + o(&cfg) + } + if cfg.concurrency < 1 { + cfg.concurrency = 1 + } + + // Parallel slices indexed by source position so output stays in source + // order regardless of completion order. Using pointers lets us detect + // "no result and no error" (shouldn't happen but is a defensive signal). + type slot struct { + result *FanoutResult[T] + err *FanoutError + } + slots := make([]slot, len(sources)) + + type job struct{ idx int } + jobs := make(chan job, cfg.concurrency*2) + + var wg sync.WaitGroup + for w := 0; w < cfg.concurrency; w++ { + wg.Add(1) + go func() { + defer wg.Done() + for j := range jobs { + idx := j.idx + func() { + // Recover panics from fn so one bad source doesn't kill + // the whole process. The panic becomes a per-source + // FanoutError alongside regular errors. + defer func() { + if r := recover(); r != nil { + slots[idx].err = &FanoutError{ + Source: name(sources[idx]), + Err: fmt.Errorf("panic in fanout fn: %v", r), + } + } + }() + // Respect cancellation: if ctx is already done, record + // the cancel error rather than running fn with a + // useless context. + if err := ctx.Err(); err != nil { + slots[idx].err = &FanoutError{Source: name(sources[idx]), Err: err} + return + } + val, err := fn(ctx, sources[idx]) + if err != nil { + slots[idx].err = &FanoutError{Source: name(sources[idx]), Err: err} + } else { + v := val + slots[idx].result = &FanoutResult[T]{Source: name(sources[idx]), Value: v} + } + }() + } + }() + } + + // Feed jobs, but stop feeding if ctx cancels so unpulled sources get a + // ctx.Err() FanoutError rather than being silently dropped. + func() { + defer close(jobs) + for i := range sources { + select { + case <-ctx.Done(): + // Mark this and all remaining sources as cancelled, then stop. + for j := i; j < len(sources); j++ { + slots[j].err = &FanoutError{Source: name(sources[j]), Err: ctx.Err()} + } + return + case jobs <- job{idx: i}: + } + } + }() + + wg.Wait() + + results := make([]FanoutResult[T], 0, len(sources)) + errs := make([]FanoutError, 0, len(slots)) + for _, s := range slots { + if s.result != nil { + results = append(results, *s.result) + } + if s.err != nil { + errs = append(errs, *s.err) + } + } + return results, errs +} + +// FanoutReportErrors writes one warning line per FanoutError to w in source +// order. Format: "warn: <source>: <short-error>\n" where <short-error> is +// the error's first line truncated to 120 chars. No-op when errs is empty. +// +// Call this after FanoutRun so partial failures never get silently dropped +// — the warning surface is the whole reason for the helper. +func FanoutReportErrors(w io.Writer, errs []FanoutError) { + for _, e := range errs { + fmt.Fprintf(w, "warn: %s: %s\n", e.Source, shortFanoutErr(e.Err)) + } +} + +// shortFanoutErr condenses an error to a single-line reason string for +// stderr display alongside many sources. +func shortFanoutErr(err error) string { + if err == nil { + return "" + } + s := err.Error() + if i := strings.Index(s, "\n"); i >= 0 { + s = s[:i] + } + const max = 120 + if len(s) > max { + s = s[:max] + "…" + } + return s +} diff --git a/library/productivity/devonthink/internal/cliutil/jwtshape.go b/library/productivity/devonthink/internal/cliutil/jwtshape.go new file mode 100644 index 0000000000..daf06d1ae3 --- /dev/null +++ b/library/productivity/devonthink/internal/cliutil/jwtshape.go @@ -0,0 +1,96 @@ +// Copyright 2026 rowdy and contributors. Licensed under Apache-2.0. See LICENSE. +// Generated by CLI Printing Press (https://github.com/mvanhorn/cli-printing-press). DO NOT EDIT. + +package cliutil + +import "strings" + +// JWT shape floors — empirically chosen. +// +// Real-world Auth0 / Cognito / Firebase / Supabase RS256 access tokens are +// 600–1200 chars. The smallest plausible legitimate HS256 JWT (header ~30 +// chars, minimal claims ~80 chars, signature ~40 chars) is around 150 chars +// total with a header segment ≥ 20 chars. +// +// Anti-targets: Cloudflare Bot Management cookies (`__cf_bm`, ~30 chars), some +// Mixpanel distinct IDs, segment-style A/B test identifiers — all of which +// happen to have three dot-separated base64url chunks but contain no JWT +// payload. Without a length floor these slip past a segment-count + charset +// check and get saved as access tokens, with the failure mode of a confusing +// HTTP 401 chain on every subsequent API call. +const ( + minJWTTotalLen = 150 + minJWTHeaderLen = 20 +) + +// LooksLikeJWT reports whether s is shaped like a JWT — three base64url +// segments separated by dots, with a length floor that filters out short +// tracking cookies and CSRF tokens that share the segment shape. +// +// The function is permissive about a leading `Bearer ` prefix (it strips it +// before measuring) so callers can pass either the raw token or the wire +// value of an Authorization header. +// +// This is a shape check, not a signature verification. Callers that need to +// know whether a token is currently valid for a specific audience should +// decode it via the standard JWT library; this helper exists for the +// upstream gate — "should we even attempt to save this string as a +// credential" — that runs before the token ever reaches the API. +func LooksLikeJWT(s string) bool { + s = strings.TrimSpace(s) + s = strings.TrimPrefix(s, "Bearer ") + if len(s) < minJWTTotalLen { + return false + } + parts := strings.Split(s, ".") + if len(parts) != 3 { + return false + } + if len(parts[0]) < minJWTHeaderLen { + return false + } + for _, p := range parts { + if p == "" { + return false + } + for _, r := range p { + isAlnum := (r >= 'A' && r <= 'Z') || (r >= 'a' && r <= 'z') || (r >= '0' && r <= '9') + if !isAlnum && r != '-' && r != '_' && r != '=' { + return false + } + } + } + return true +} + +// FindJWTInCookieJar scans a "name=value; name=value; ..." cookie jar string +// for the first value that satisfies LooksLikeJWT, returning "" when nothing +// matches. Useful when a CLI's auth flow extracts a browser cookie jar and +// the underlying API actually expects a Bearer JWT (which some sites surface +// as a cookie alongside their session cookies). +// +// The length floor in LooksLikeJWT does the heavy lifting here — without it, +// jar scans on Cloudflare-fronted sites trip on `__cf_bm` and similar. +// +// The returned value is the bare token (any leading `Bearer ` is stripped), +// matching LooksLikeJWT's input normalization. A caller that builds an +// Authorization header from the result therefore prepends `Bearer ` exactly +// once; passing the value through verbatim does not produce a double prefix +// even when the cookie's value carries the Authorization wire form. +func FindJWTInCookieJar(jar string) string { + for _, raw := range strings.Split(jar, ";") { + raw = strings.TrimSpace(raw) + if raw == "" { + continue + } + eq := strings.IndexByte(raw, '=') + if eq < 0 { + continue + } + value := strings.TrimPrefix(strings.TrimSpace(raw[eq+1:]), "Bearer ") + if LooksLikeJWT(value) { + return value + } + } + return "" +} diff --git a/library/productivity/devonthink/internal/cliutil/jwtshape_test.go b/library/productivity/devonthink/internal/cliutil/jwtshape_test.go new file mode 100644 index 0000000000..6eff93618d --- /dev/null +++ b/library/productivity/devonthink/internal/cliutil/jwtshape_test.go @@ -0,0 +1,128 @@ +// Copyright 2026 rowdy and contributors. Licensed under Apache-2.0. See LICENSE. +// Generated by CLI Printing Press (https://github.com/mvanhorn/cli-printing-press). DO NOT EDIT. + +package cliutil + +import ( + "strings" + "testing" +) + +func TestLooksLikeJWT(t *testing.T) { + t.Parallel() + + // realAuth0RS256 is a realistically-sized RS256 token (header ~64 chars, + // payload ~430 chars, signature ~344 chars). Synthetic — not a credential. + realAuth0RS256 := "eyJhbGciOiJSUzI1NiIsInR5cCI6IkpXVCIsImtpZCI6ImFvNi1PYzAyQlZHTF9GbnBDeE5JMiJ9." + + strings.Repeat("a", 430) + "." + strings.Repeat("b", 344) + + // Exact boundary fixtures. The floor is 150 chars total; these straddle it. + // atFloor149: h×36 + "." + p×72 + "." + s×39 = 36+1+72+1+39 = 149 — must reject. + // atFloor150: h×36 + "." + p×72 + "." + s×40 = 36+1+72+1+40 = 150 — must accept. + // A constant change (150 -> 155) or an off-by-one (< vs <=) would flip one + // of these and trip the test. + atFloor149 := strings.Repeat("h", 36) + "." + strings.Repeat("p", 72) + "." + strings.Repeat("s", 39) + atFloor150 := strings.Repeat("h", 36) + "." + strings.Repeat("p", 72) + "." + strings.Repeat("s", 40) + + // minimalHS256 is a realistic minimum-sized JWT (~158 chars). The exact + // boundary cases above test the floor; this case exists so a regression + // that breaks normal-sized small JWTs surfaces independently. + minimalHS256 := strings.Repeat("h", 36) + "." + strings.Repeat("p", 80) + "." + strings.Repeat("s", 40) + + // Factor75 false-positive — Cloudflare-shaped cookie value with 3 + // base64url segments and 31 chars total. The original looksLikeJWT + // heuristic accepted this; the length-floored shape check rejects it. + cfCookieShaped := "01KRPVRYA2SNQT9BAGD6984WAG_.tt.1" + + cases := []struct { + name string + in string + want bool + }{ + {"real Auth0 RS256 token", realAuth0RS256, true}, + {"exactly at 150-char floor", atFloor150, true}, + {"realistic minimum HS256", minimalHS256, true}, + {"with Bearer prefix", "Bearer " + realAuth0RS256, true}, + + {"exactly one char under floor (149)", atFloor149, false}, + {"factor75 Cloudflare cookie", cfCookieShaped, false}, + {"empty string", "", false}, + {"whitespace only", " \t\n", false}, + {"single segment", strings.Repeat("a", 200), false}, + {"two segments", strings.Repeat("a", 100) + "." + strings.Repeat("b", 100), false}, + {"four segments", "aaa.bbb.ccc.ddd", false}, + {"empty middle segment", strings.Repeat("a", 80) + ".." + strings.Repeat("b", 80), false}, + {"header segment too short", "aaa." + strings.Repeat("p", 80) + "." + strings.Repeat("s", 80), false}, + {"invalid charset", strings.Repeat("a", 60) + ".pay!load." + strings.Repeat("s", 80), false}, + } + + for _, tc := range cases { + tc := tc + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + got := LooksLikeJWT(tc.in) + if got != tc.want { + t.Fatalf("LooksLikeJWT(%q) = %v, want %v", tc.in, got, tc.want) + } + }) + } +} + +func TestFindJWTInCookieJar(t *testing.T) { + t.Parallel() + + realJWT := "eyJhbGciOiJSUzI1NiIsInR5cCI6IkpXVCIsImtpZCI6ImFvNi1PYzAyQlZHTF9GbnBDeE5JMiJ9." + + strings.Repeat("a", 430) + "." + strings.Repeat("b", 344) + + cases := []struct { + name string + jar string + want string + }{ + { + name: "JWT alongside CF tracking cookies", + jar: "__cf_bm=01KRPVRYA2SNQT9BAGD6984WAG_.tt.1; auth_token=" + realJWT + "; csrf=abc123", + want: realJWT, + }, + { + // A cookie whose value carries the Authorization wire form + // ("Bearer eyJ...") must not double-prefix downstream. The + // returned value should be the bare token so callers building + // an Authorization header always prepend "Bearer " exactly once. + name: "cookie value carries Bearer prefix", + jar: "auth=Bearer " + realJWT + "; csrf=abc", + want: realJWT, + }, + { + name: "no JWT, only short shaped cookies", + jar: "__cf_bm=01KRPVRYA2SNQT9BAGD6984WAG_.tt.1; _ga=GA1.1.123.456", + want: "", + }, + { + name: "empty jar", + jar: "", + want: "", + }, + { + name: "single JWT cookie", + jar: "token=" + realJWT, + want: realJWT, + }, + { + name: "malformed cookie without equals", + jar: "notacookie; " + realJWT, + want: "", + }, + } + + for _, tc := range cases { + tc := tc + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + got := FindJWTInCookieJar(tc.jar) + if got != tc.want { + t.Fatalf("FindJWTInCookieJar(...) returned %d-char string; want %d-char", len(got), len(tc.want)) + } + }) + } +} diff --git a/library/productivity/devonthink/internal/cliutil/odata_date.go b/library/productivity/devonthink/internal/cliutil/odata_date.go new file mode 100644 index 0000000000..557cb91487 --- /dev/null +++ b/library/productivity/devonthink/internal/cliutil/odata_date.go @@ -0,0 +1,37 @@ +// Copyright 2026 rowdy and contributors. Licensed under Apache-2.0. See LICENSE. +// Generated by CLI Printing Press (https://github.com/mvanhorn/cli-printing-press). DO NOT EDIT. + +package cliutil + +import ( + "regexp" + "strconv" + "time" +) + +// odataDateRE matches the canonical OData v3 "/Date(milliseconds)/" literal, +// with an optional signed timezone offset that OData includes only as a display +// hint (the millisecond count is already a UTC epoch value). Both wrapping +// slashes are required, matching the form real OData v3 producers emit. +var odataDateRE = regexp.MustCompile(`^/Date\((-?\d+)(?:[+-]\d{4})?\)/$`) + +// ParseODataDate decodes an OData v3 "/Date(ms)/" literal to a UTC time.Time. +// It falls back to RFC3339 (normalising the result to UTC) so callers can pass +// any datetime field without dispatching on format. Returns ok=false (and the +// zero time) when the input matches neither form. +// +// Why this exists: OData v3 APIs (Exact Online, Microsoft Dynamics 365 +// Business Central, Dynamics NAV) encode dates as "/Date(1715731200000)/" +// string literals that no standard parser accepts. Without this helper every +// OData CLI re-implements the same regex, usually inline and inconsistently. +func ParseODataDate(s string) (time.Time, bool) { + if m := odataDateRE.FindStringSubmatch(s); m != nil { + if ms, err := strconv.ParseInt(m[1], 10, 64); err == nil { + return time.UnixMilli(ms).UTC(), true + } + } + if t, err := time.Parse(time.RFC3339, s); err == nil { + return t.UTC(), true + } + return time.Time{}, false +} diff --git a/library/productivity/devonthink/internal/cliutil/odata_date_test.go b/library/productivity/devonthink/internal/cliutil/odata_date_test.go new file mode 100644 index 0000000000..c4786a82a7 --- /dev/null +++ b/library/productivity/devonthink/internal/cliutil/odata_date_test.go @@ -0,0 +1,62 @@ +// Copyright 2026 rowdy and contributors. Licensed under Apache-2.0. See LICENSE. +// Generated by CLI Printing Press (https://github.com/mvanhorn/cli-printing-press). DO NOT EDIT. + +package cliutil + +import ( + "testing" + "time" +) + +func TestParseODataDate(t *testing.T) { + t.Parallel() + + wantMS := time.UnixMilli(1715731200000).UTC() + + t.Run("date literal to UTC", func(t *testing.T) { + t.Parallel() + got, ok := ParseODataDate("/Date(1715731200000)/") + if !ok { + t.Fatal("expected ok=true for /Date(ms)/ literal") + } + if !got.Equal(wantMS) || got.Location() != time.UTC { + t.Fatalf("got %v, want %v (UTC)", got, wantMS) + } + }) + + t.Run("date literal with timezone offset", func(t *testing.T) { + t.Parallel() + got, ok := ParseODataDate("/Date(1715731200000-0500)/") + if !ok || !got.Equal(wantMS) { + t.Fatalf("offset literal: got %v ok=%v, want %v", got, ok, wantMS) + } + }) + + t.Run("rfc3339 fallback", func(t *testing.T) { + t.Parallel() + want, _ := time.Parse(time.RFC3339, "2026-05-17T12:34:56Z") + got, ok := ParseODataDate("2026-05-17T12:34:56Z") + if !ok || !got.Equal(want) { + t.Fatalf("rfc3339: got %v ok=%v, want %v", got, ok, want) + } + }) + + t.Run("rfc3339 non-UTC offset normalised to UTC", func(t *testing.T) { + t.Parallel() + got, ok := ParseODataDate("2026-05-17T12:34:56+05:30") + if !ok { + t.Fatal("expected ok=true for RFC3339 with non-UTC offset") + } + if got.Location() != time.UTC { + t.Fatalf("expected UTC location, got %v", got.Location()) + } + }) + + t.Run("garbage returns zero and false", func(t *testing.T) { + t.Parallel() + got, ok := ParseODataDate("not-a-date") + if ok || !got.IsZero() { + t.Fatalf("garbage: got %v ok=%v, want zero+false", got, ok) + } + }) +} diff --git a/library/productivity/devonthink/internal/cliutil/probe.go b/library/productivity/devonthink/internal/cliutil/probe.go new file mode 100644 index 0000000000..aaaeb62aa5 --- /dev/null +++ b/library/productivity/devonthink/internal/cliutil/probe.go @@ -0,0 +1,104 @@ +// Copyright 2026 rowdy and contributors. Licensed under Apache-2.0. See LICENSE. +// Generated by CLI Printing Press (https://github.com/mvanhorn/cli-printing-press). DO NOT EDIT. + +package cliutil + +import ( + "context" + "fmt" + "io" + "net/http" + "time" +) + +// defaultProbeTimeout caps the request when the caller passes a nil +// client and didn't set a context deadline. Without this cap, a probe +// against a non-responsive host could hang indefinitely (the global +// http.DefaultClient has no Timeout). Callers who pass their own +// *http.Client are expected to set Timeout there; this value only +// applies to the nil-client fallback. +const defaultProbeTimeout = 10 * time.Second + +// ReachabilityStatus is one of the strings returned by ProbeReachable. +// Callers (typically a doctor command listing per-source health) match +// on these constants when deciding whether to render OK/WARN/FAIL. +const ( + // ReachabilityReachable means the host responded with a 2xx, a 206 + // (partial — Range honored), or a 416 (Range not honored, but the + // host did respond with headers). All three are evidence that the + // host is alive and responding to GET. + ReachabilityReachable = "reachable" + // ReachabilityBlocked means the host responded with a 4xx (other + // than 416) or 5xx. The host is up but is refusing this request — + // usually a CDN bot screen, a paywall, or a server error. + ReachabilityBlocked = "blocked" + // ReachabilityUnreachable means the request errored at the network + // layer — DNS failure, connection refused, TLS shutdown, timeout. + ReachabilityUnreachable = "unreachable" +) + +// ProbeReachable does a lightweight reachability probe against url +// using client and returns a (status, code, err) triple. The probe +// uses GET with a `Range: bytes=0-1023` header so it never pulls more +// than ~1 KB of body, regardless of how the host responds; the body +// is read and discarded so the connection can be released. +// +// Why not HEAD: many recipe-site CDNs (BBC, RecipeTin Eats, AllRecipes, +// Serious Eats, The Kitchn) terminate HEAD requests with a TLS +// shutdown / EOF even though they serve GET cleanly. A HEAD-based +// probe lies — reporting "unreachable EOF" for hosts that work fine +// for the real fetch path. recipe-goat hit this in retro #301 +// finding F4: doctor reported six sites unreachable that the goat +// ranker was successfully scraping. +// +// Use this from any doctor or health-check command that does +// per-source reachability fan-out, with the same client that the real +// fetch path uses (typically a Surf-Chrome client built via +// surf.NewClient().Builder().Impersonate().Chrome().Build()). Probe +// drift between the doctor probe and the fetch path is the bug class +// this helper exists to prevent. +// +// Returned values: +// - status is one of ReachabilityReachable, ReachabilityBlocked, or +// ReachabilityUnreachable. +// - code is the HTTP status code, or 0 when the request errored at +// the network layer. +// - err is non-nil only for network-layer failures. A 4xx or 5xx +// response is reported via status/code with err == nil. +func ProbeReachable(ctx context.Context, client *http.Client, url string) (status string, code int, err error) { + if client == nil { + // Build a copy of DefaultClient with a bounded timeout — the + // global DefaultClient has none, so a nil-client probe against + // a slow host would hang. Callers passing their own client are + // expected to set Timeout themselves. + client = &http.Client{Timeout: defaultProbeTimeout} + } + req, reqErr := http.NewRequestWithContext(ctx, "GET", url, nil) + if reqErr != nil { + return ReachabilityUnreachable, 0, fmt.Errorf("building request: %w", reqErr) + } + // Range: bytes=0-1023 keeps body bounded for hosts that honor it. + // Hosts that don't support Range respond with 200 + full body or + // 416 — both are caught below as "reachable", and the limited + // io.Copy below ensures we never pull more than 1 KiB anyway. + req.Header.Set("Range", "bytes=0-1023") + resp, doErr := client.Do(req) + if doErr != nil { + return ReachabilityUnreachable, 0, doErr + } + defer resp.Body.Close() + // Drain up to 2 KiB so the connection can be reused. We read past + // the 1024-byte Range hint to cover hosts that ignored it. + _, _ = io.Copy(io.Discard, io.LimitReader(resp.Body, 2048)) + switch { + case resp.StatusCode >= 200 && resp.StatusCode < 300: + return ReachabilityReachable, resp.StatusCode, nil + case resp.StatusCode == http.StatusRequestedRangeNotSatisfiable: + // 416 means the host doesn't support Range. We still got + // headers back, so the host is up and responding to GET — + // classify as reachable. + return ReachabilityReachable, resp.StatusCode, nil + default: + return ReachabilityBlocked, resp.StatusCode, nil + } +} diff --git a/library/productivity/devonthink/internal/cliutil/ratelimit.go b/library/productivity/devonthink/internal/cliutil/ratelimit.go new file mode 100644 index 0000000000..be6337ec38 --- /dev/null +++ b/library/productivity/devonthink/internal/cliutil/ratelimit.go @@ -0,0 +1,207 @@ +// Copyright 2026 rowdy and contributors. Licensed under Apache-2.0. See LICENSE. +// Generated by CLI Printing Press (https://github.com/mvanhorn/cli-printing-press). DO NOT EDIT. + +package cliutil + +import ( + "fmt" + "math" + "net/http" + "strconv" + "strings" + "sync" + "time" +) + +// AdaptiveLimiter paces outbound requests with adaptive ceiling discovery. +// Starts at a floor rate, ramps up after consecutive successes, halves on 429 +// and records a ceiling. Per-session only — not persisted. Methods are safe +// to call on a nil receiver. +type AdaptiveLimiter struct { + mu sync.Mutex + rate float64 + floor float64 + ceiling float64 + successes int + rampAfter int + lastRequest time.Time // zero-value: first Wait() returns immediately +} + +// NewAdaptiveLimiter returns a limiter starting at ratePerSec, or nil when +// rate-limiting should be disabled. Methods on the nil limiter no-op. +func NewAdaptiveLimiter(ratePerSec float64) *AdaptiveLimiter { + if ratePerSec <= 0 { + return nil + } + floor := 0.5 + if ratePerSec < floor { + floor = ratePerSec + } + return &AdaptiveLimiter{ + rate: ratePerSec, + floor: floor, + rampAfter: 10, + } +} + +func (l *AdaptiveLimiter) Wait() { + if l == nil { + return + } + l.mu.Lock() + delay := time.Duration(float64(time.Second) / l.rate) + elapsed := time.Since(l.lastRequest) + var sleep time.Duration + if elapsed < delay { + sleep = delay - elapsed + } + l.lastRequest = time.Now().Add(sleep) + l.mu.Unlock() + if sleep > 0 { + time.Sleep(sleep) + } +} + +func (l *AdaptiveLimiter) OnSuccess() { + if l == nil { + return + } + l.mu.Lock() + defer l.mu.Unlock() + l.successes++ + if l.successes >= l.rampAfter { + newRate := l.rate * 1.25 + if l.ceiling > 0 && newRate > l.ceiling*0.9 { + newRate = l.ceiling * 0.9 + } + if newRate < l.floor { + newRate = l.floor + } + l.rate = newRate + l.successes = 0 + } +} + +func (l *AdaptiveLimiter) OnRateLimit() { + if l == nil { + return + } + l.mu.Lock() + defer l.mu.Unlock() + l.ceiling = l.rate + l.rate = l.rate / 2 + if l.rate < l.floor { + l.rate = l.floor + } + l.successes = 0 +} + +func (l *AdaptiveLimiter) Rate() float64 { + if l == nil { + return 0 + } + l.mu.Lock() + defer l.mu.Unlock() + return l.rate +} + +// RateLimitError signals an upstream returned 429 after retries were +// exhausted. Callers must surface this as a hard error rather than empty +// results — empty-on-throttle is indistinguishable from "no data exists" +// and silently corrupts downstream queries. +type RateLimitError struct { + URL string + RetryAfter time.Duration + Body string +} + +func (e *RateLimitError) Error() string { + msg := fmt.Sprintf("rate limited: HTTP 429 for %s", e.URL) + if e.RetryAfter > 0 { + msg += fmt.Sprintf("; retry after %s", e.RetryAfter) + } + if body := strings.TrimSpace(e.Body); body != "" { + msg += ": " + body + } + return msg +} + +// MaxRetryWait caps the wait derived from a Retry-After header so a buggy +// or hostile upstream cannot pin a CLI for hours. +const MaxRetryWait = 60 * time.Second + +const ( + defaultRetryWait = 5 * time.Second + unixEpochSecondsThreshold = 1_000_000_000 + unixEpochMillisecondsThreshold = 1_000_000_000_000 +) + +// RetryAfter parses an HTTP Retry-After header (RFC 7231: delta-seconds or +// HTTP-date), plus common Unix epoch seconds/milliseconds variants emitted by +// some APIs. Waits are capped at MaxRetryWait. Returns 5s when missing or +// unparseable. +func RetryAfter(resp *http.Response) time.Duration { + if resp == nil { + return defaultRetryWait + } + header := strings.TrimSpace(resp.Header.Get("Retry-After")) + if header == "" { + return defaultRetryWait + } + if value, err := strconv.ParseInt(header, 10, 64); err == nil { + return retryAfterFromNumber(value) + } + if t, err := http.ParseTime(header); err == nil { + wait := time.Until(t) + if wait > MaxRetryWait { + return MaxRetryWait + } + if wait > 0 { + return wait + } + } + return defaultRetryWait +} + +func retryAfterFromNumber(value int64) time.Duration { + if value <= 0 { + return defaultRetryWait + } + if value > int64(MaxRetryWait/time.Second) { + if wait := retryAfterEpochWait(value); wait > 0 { + if wait > MaxRetryWait { + return MaxRetryWait + } + return wait + } + return MaxRetryWait + } + return time.Duration(value) * time.Second +} + +func retryAfterEpochWait(value int64) time.Duration { + switch { + case value >= unixEpochMillisecondsThreshold: + return time.Until(time.UnixMilli(value)) + case value >= unixEpochSecondsThreshold: + return time.Until(time.Unix(value, 0)) + default: + return 0 + } +} + +// MaxBackoff caps Backoff so tests stay bounded. Callers needing jitter +// add their own; the bare exponential keeps the contract deterministic. +const MaxBackoff = 30 * time.Second + +// Backoff returns 2^attempt seconds capped at MaxBackoff. +func Backoff(attempt int) time.Duration { + if attempt < 0 { + attempt = 0 + } + wait := time.Duration(math.Pow(2, float64(attempt))) * time.Second + if wait > MaxBackoff { + return MaxBackoff + } + return wait +} diff --git a/library/productivity/devonthink/internal/cliutil/text.go b/library/productivity/devonthink/internal/cliutil/text.go new file mode 100644 index 0000000000..ae3284dab7 --- /dev/null +++ b/library/productivity/devonthink/internal/cliutil/text.go @@ -0,0 +1,50 @@ +// Copyright 2026 rowdy and contributors. Licensed under Apache-2.0. See LICENSE. +// Generated by CLI Printing Press (https://github.com/mvanhorn/cli-printing-press). DO NOT EDIT. + +package cliutil + +import ( + "html" + "strings" + "time" +) + +// CleanText normalizes scraped text by trimming whitespace and decoding +// HTML entities. Always use this when extracting strings from HTML or +// schema.org JSON-LD. Skipping this step is how recipe-goat's +// "The Food Lab's" bug shipped: schema.org strings passed through +// unescaped because the JSON-LD parser didn't normalize. +// +// Single unescape pass: "&amp;" -> "&" (matches html.UnescapeString +// stdlib behavior). If you need multiple passes you almost always have a +// deeper escaping problem upstream — fix there, not here. +func CleanText(s string) string { + return html.UnescapeString(strings.TrimSpace(s)) +} + +// ParseStoredTime parses timestamps read back from SQLite-backed generated +// stores. modernc.org/sqlite can serialize time.Time using Go's native +// time.String format, while hand-written sync code often stores RFC3339. +// Use this helper instead of a single time.Parse(time.RFC3339, value) call +// when scanning timestamp columns from the store. +func ParseStoredTime(s string) time.Time { + s = strings.TrimSpace(s) + if s == "" { + return time.Time{} + } + for _, layout := range []string{ + time.RFC3339Nano, + time.RFC3339, + "2006-01-02 15:04:05.999999999 -0700 MST", + "2006-01-02 15:04:05.999999 -0700 MST", + "2006-01-02 15:04:05.999 -0700 MST", + "2006-01-02 15:04:05 -0700 MST", + "2006-01-02 15:04:05.999999999 -0700", + "2006-01-02 15:04:05 -0700", + } { + if t, err := time.Parse(layout, s); err == nil { + return t + } + } + return time.Time{} +} diff --git a/library/productivity/devonthink/internal/cliutil/verifyenv.go b/library/productivity/devonthink/internal/cliutil/verifyenv.go new file mode 100644 index 0000000000..b8283d508a --- /dev/null +++ b/library/productivity/devonthink/internal/cliutil/verifyenv.go @@ -0,0 +1,93 @@ +// Copyright 2026 rowdy and contributors. Licensed under Apache-2.0. See LICENSE. +// Generated by CLI Printing Press (https://github.com/mvanhorn/cli-printing-press). DO NOT EDIT. + +package cliutil + +import "os" + +// VerifyEnvVar is the env var the printing-press verifier sets in every +// mock-mode subprocess. Generated commands that perform visible side +// effects (open browser tabs, send notifications, dial out to OS +// handlers) MUST short-circuit when this env var is "1" to avoid +// spamming the user's environment during verify runs. +// +// The transport layer in internal/client also gates mutating HTTP verbs +// (DELETE/POST/PUT/PATCH) on this var: under verify mode such requests +// short-circuit with a synthetic envelope and never dial. The verifier +// itself opts back in to the real wire path via VerifyLiveHTTPEnvVar +// so its httptest mock-server flow keeps exercising the real client. +const VerifyEnvVar = "PRINTING_PRESS_VERIFY" + +// VerifyLiveHTTPEnvVar opts a verify-mode subprocess back in to the +// real HTTP wire path for mutating verbs. It is intentionally +// asymmetric with VerifyEnvVar: setting LIVE_HTTP=1 alone (with VERIFY +// unset) has no behavioral effect, because the gate only consults this +// var when IsVerifyEnv() is also true. The verify pipeline and +// narrative full-example runner both set BOTH vars in their mock-mode +// subprocesses (so the httptest server keeps exercising the real wire +// path); agents and ad-hoc operators leave LIVE_HTTP unset so mutating +// requests no-op. Live verifiers (live_dogfood, workflow_verify) strip +// both vars from subprocess env entirely so they cannot inherit a +// verify-mode short-circuit from the operator's shell. +const VerifyLiveHTTPEnvVar = "PRINTING_PRESS_VERIFY_LIVE_HTTP" + +// DogfoodEnvVar is the env var the printing-press live-dogfood runner +// sets in every subprocess. Distinct from VerifyEnvVar because dogfood +// is a real-network matrix: commands may still perform actual API +// calls, just curtailed (paginate-once, bounded crawl, etc.) so the +// runner's flat 30s per-command timeout doesn't trip. +const DogfoodEnvVar = "PRINTING_PRESS_DOGFOOD" + +// IsVerifyEnv reports whether the current process is running under the +// printing-press verifier in mock mode. Generated commands with side +// effects pair this check with print-by-default + explicit opt-in +// (--launch, --send, --play) so a verify pass on a fresh CLI does not +// pop browser tabs or fire off real notifications. +// +// Defense-in-depth: even if the verifier's heuristic side-effect +// classifier misses a command, this env-var short-circuit catches it. +// +// if cliutil.IsVerifyEnv() { +// fmt.Fprintln(cmd.OutOrStdout(), "would launch:", url) +// return nil +// } +func IsVerifyEnv() bool { + return os.Getenv(VerifyEnvVar) == "1" +} + +// IsVerifyLiveHTTPEnv reports whether the current process has opted +// back in to the real HTTP wire path while running under the verifier. +// Only meaningful when IsVerifyEnv() is also true; on its own this +// returns true does NOT enable any sandbox behavior — see +// VerifyLiveHTTPEnvVar's docstring for the asymmetric semantics. +// +// The generated client uses this gate as: +// +// if !readOnlyIntent && isMutatingVerb(method) && cliutil.IsVerifyEnv() && !cliutil.IsVerifyLiveHTTPEnv() { +// // synthetic envelope, no network call +// } +// +// readOnlyIntent is set by Client.doRead() callers (the PostQuery* +// family used by codegen-marked `mcp:read-only` operations on mutating +// verbs — GraphQL queries, JSON-RPC reads, POST-based search). +func IsVerifyLiveHTTPEnv() bool { + return os.Getenv(VerifyLiveHTTPEnvVar) == "1" +} + +// IsDogfoodEnv reports whether the current process is running under +// the printing-press live-dogfood matrix. Long-running commands (full +// sync loops, content crawlers, bulk archive walks) should use this +// to curtail work so the flat 30s per-command timeout doesn't kill an +// otherwise healthy happy_path test. Typical pattern: paginate once, +// fetch a bounded sample, or honor a smaller --limit default. +// +// if cliutil.IsDogfoodEnv() { +// return crawl(ctx, opts.WithMaxPages(1)) +// } +// +// Unlike IsVerifyEnv this does NOT mean "don't hit the network" — +// dogfood is a real-API matrix. Use this only to bound work, never to +// substitute mock data for real calls. +func IsDogfoodEnv() bool { + return os.Getenv(DogfoodEnvVar) == "1" +} diff --git a/library/productivity/devonthink/internal/config/config.go b/library/productivity/devonthink/internal/config/config.go new file mode 100644 index 0000000000..ef845dfcd8 --- /dev/null +++ b/library/productivity/devonthink/internal/config/config.go @@ -0,0 +1,165 @@ +// Copyright 2026 rowdy and contributors. Licensed under Apache-2.0. See LICENSE. +// Generated by CLI Printing Press (https://github.com/mvanhorn/cli-printing-press). DO NOT EDIT. + +package config + +import ( + "fmt" + "os" + "path/filepath" + "strings" + + "github.com/pelletier/go-toml/v2" +) + +type Config struct { + BaseURL string `toml:"base_url"` + AuthHeaderVal string `toml:"auth_header"` + Headers map[string]string `toml:"headers,omitempty"` + AuthSource string `toml:"-"` + Path string `toml:"-"` + envOverrides map[string]bool `toml:"-"` + fileConfig *Config `toml:"-"` +} + +func Load(configPath string) (*Config, error) { + cfg := &Config{} + + // Resolve config path + path := configPath + if path == "" { + path = os.Getenv("DEVONTHINK_CONFIG") + } + if path == "" { + home, _ := os.UserHomeDir() + path = filepath.Join(home, ".config", "devonthink-pp-cli", "config.toml") + } + cfg.Path = path + + // Try to load config file + data, err := os.ReadFile(path) // #nosec G304,G703 -- config path is an explicit local CLI input/env override. + if err == nil { + if err := toml.Unmarshal(data, cfg); err != nil { + return nil, fmt.Errorf("parsing config %s: %w", path, err) + } + } + + cfg.snapshotFileConfig() + + // Env var overrides + + // Label config-file-derived credentials so doctor can distinguish + // "credentials persisted on disk" from "no credentials at all" — without + // this, users who saved via set-token without an env var see a blank + // auth_source and can't tell whether their config is being picked up. + // The label is the literal "config" rather than "config:<path>"; the + // config file path is exposed separately as report["config_path"], and + // embedding it in auth_source leaks the user's home directory through + // doctor's JSON envelope. + if cfg.AuthSource == "" && (cfg.AuthHeaderVal != "") { + cfg.AuthSource = "config" + } + + // Base URL override (used by printing-press verify to point at mock/test servers) + if v := os.Getenv("DEVONTHINK_BASE_URL"); v != "" { + cfg.BaseURL = v + } + return cfg, nil +} + +func (c *Config) AuthHeader() string { + if c.AuthHeaderVal != "" { + return c.AuthHeaderVal + } + return "" +} + +func applyAuthFormat(format string, replacements map[string]string) string { + if format == "" { + return "" + } + for key, value := range replacements { + format = strings.ReplaceAll(format, "{"+key+"}", value) + } + if strings.Contains(format, "{") { + return "" + } + return format +} + +func (c *Config) markEnvOverride(field string) { + if c.envOverrides == nil { + c.envOverrides = map[string]bool{} + } + c.envOverrides[field] = true +} + +// cloneStringMap returns an independent copy of m (nil stays nil). The fileConfig +// snapshot must not share reference-type map fields (such as Headers) with the +// live config, or a later mutation to one would silently track in the other. +func cloneStringMap(m map[string]string) map[string]string { + if m == nil { + return nil + } + out := make(map[string]string, len(m)) + for k, v := range m { + out[k] = v + } + return out +} + +func (c *Config) snapshotFileConfig() { + snapshot := *c + snapshot.envOverrides = nil + snapshot.fileConfig = nil + // *c is a shallow copy: map fields are reference types, so the snapshot would + // share them with c and silently track later mutations, defeating the + // isolation this snapshot exists to provide. Clone them. + snapshot.Headers = cloneStringMap(c.Headers) + c.fileConfig = &snapshot +} + +func (c *Config) configForSave() Config { + out := *c + if c.fileConfig != nil { + } + out.envOverrides = nil + out.fileConfig = nil + return out +} + +func (c *Config) updateFileConfigField(field string) { + if c.fileConfig == nil || c.envOverrides[field] { + return + } + switch field { + case "AuthHeaderVal": + c.fileConfig.AuthHeaderVal = c.AuthHeaderVal + } +} + +func (c *Config) save() error { + dir := filepath.Dir(c.Path) + if err := os.MkdirAll(dir, 0o700); err != nil { + return fmt.Errorf("creating config dir: %w", err) + } + persisted := c.configForSave() + data, err := toml.Marshal(persisted) + if err != nil { + return fmt.Errorf("marshaling config: %w", err) + } + if err := os.WriteFile(c.Path, data, 0o600); err != nil { + return err + } + c.fileConfig = &persisted + c.fileConfig.envOverrides = nil + c.fileConfig.fileConfig = nil + // persisted shares its map fields with c (configForSave shallow-copies *c), + // so isolate the stored fileConfig the same way snapshotFileConfig does; + // otherwise later mutations to c's maps leak into the on-disk snapshot. + c.fileConfig.Headers = cloneStringMap(c.fileConfig.Headers) + return nil +} + +// Ensure strings import is used +var _ = strings.ReplaceAll diff --git a/library/productivity/devonthink/internal/mcp/cobratree/classify.go b/library/productivity/devonthink/internal/mcp/cobratree/classify.go new file mode 100644 index 0000000000..0e89941054 --- /dev/null +++ b/library/productivity/devonthink/internal/mcp/cobratree/classify.go @@ -0,0 +1,115 @@ +// Copyright 2026 rowdy and contributors. Licensed under Apache-2.0. See LICENSE. +// Generated by CLI Printing Press (https://github.com/mvanhorn/cli-printing-press). DO NOT EDIT. + +package cobratree + +import ( + "strings" + + "github.com/spf13/cobra" +) + +const ( + EndpointAnnotation = "pp:endpoint" + HiddenAnnotation = "mcp:hidden" + // ReadOnlyAnnotation, when set on a Cobra command to "true"/"1"/"yes", + // causes the runtime walker to register the resulting MCP tool with + // readOnlyHint=true. Use for novel CLI commands that don't mutate + // external state — read-only API queries, local cache reads, etc. + // Without it, hosts like Claude Desktop default to "could write or + // delete" and demand permission per call. + ReadOnlyAnnotation = "mcp:read-only" +) + +type commandKind int + +const ( + commandNovel commandKind = iota + commandEndpoint + commandFramework + commandHidden +) + +// frameworkCommands are top-level CLI commands the walker should skip when +// mirroring the Cobra tree. Two cases qualify: +// +// 1. A typed MCP tool already covers the same capability (the typed tool's +// schema is strictly better than a shell-out). Examples: `sql`, `search`, +// `context`/`about`/`agent-context`, `api` (endpoint mirror tools cover it). +// 2. The command is non-functional via MCP (interactive setup, shell-only +// ergonomics, trivial introspection, local-only feedback). Examples: +// `auth`, `completion`, `doctor`, `version`, `feedback`, `profile`, +// `which`, `help`. +// +// Commands that DO have agent value — `sync` (populates the store that `sql` +// and `search` query), `stale`/`orphans`/`reconcile`/`load` (store +// diagnostics), `export`/`import` (data movement), `workflow` +// (compound operations), `analytics` (aggregations) — must NOT be in this +// list. Excluding `sync` while exposing `sql` is a broken contract because +// the typed `sql` tool returns empty results until something populates the +// store. See AGENTS.md "Agent-Native Surface" for the principle. +// +// Adding a new generator-emitted command means deciding which of the two +// cases above applies. When in doubt, leave it out — the walker registers +// any user-facing command as a shell-out tool, and the cost of a slightly +// underused tool is much smaller than the cost of a broken contract like +// `sql` without `sync`. +var frameworkCommands = map[string]bool{ + "about": true, + "agent-context": true, + "api": true, + "auth": true, + "completion": true, + "doctor": true, + "feedback": true, + "help": true, + "profile": true, + "search": true, + "sql": true, + "version": true, + "which": true, +} + +func classify(cmd *cobra.Command) commandKind { + if cmd == nil || cmd.Hidden || isMCPHidden(cmd) { + return commandHidden + } + if endpointID(cmd) != "" { + return commandEndpoint + } + if isTopLevelFrameworkCommand(cmd) { + return commandFramework + } + return commandNovel +} + +func isTopLevelFrameworkCommand(cmd *cobra.Command) bool { + if cmd == nil || !frameworkCommands[cmd.Name()] { + return false + } + parent := cmd.Parent() + return parent != nil && parent.Parent() == nil +} + +func endpointID(cmd *cobra.Command) string { + if cmd == nil || cmd.Annotations == nil { + return "" + } + return strings.TrimSpace(cmd.Annotations[EndpointAnnotation]) +} + +func isMCPHidden(cmd *cobra.Command) bool { + return annotationIsTrue(cmd, HiddenAnnotation) +} + +func isMCPReadOnly(cmd *cobra.Command) bool { + return annotationIsTrue(cmd, ReadOnlyAnnotation) +} + +func annotationIsTrue(cmd *cobra.Command, key string) bool { + if cmd == nil || cmd.Annotations == nil { + return false + } + v := strings.ToLower(strings.TrimSpace(cmd.Annotations[key])) + return v == "true" || v == "1" || v == "yes" +} diff --git a/library/productivity/devonthink/internal/mcp/cobratree/cli_path.go b/library/productivity/devonthink/internal/mcp/cobratree/cli_path.go new file mode 100644 index 0000000000..be759f0ab8 --- /dev/null +++ b/library/productivity/devonthink/internal/mcp/cobratree/cli_path.go @@ -0,0 +1,35 @@ +// Copyright 2026 rowdy and contributors. Licensed under Apache-2.0. See LICENSE. +// Generated by CLI Printing Press (https://github.com/mvanhorn/cli-printing-press). DO NOT EDIT. + +package cobratree + +import ( + "os" + "os/exec" + "path/filepath" + "runtime" +) + +// SiblingCLIPath resolves the companion CLI via sibling-of-executable, +// DEVONTHINK_CLI_PATH env var, then PATH. +func SiblingCLIPath() (string, error) { + cliName := cliExecutableName(runtime.GOOS) + if exe, err := os.Executable(); err == nil { + candidate := filepath.Join(filepath.Dir(exe), cliName) + if _, err := os.Stat(candidate); err == nil { + return candidate, nil + } + } + if v := os.Getenv("DEVONTHINK_CLI_PATH"); v != "" { + return v, nil + } + return exec.LookPath(cliName) +} + +func cliExecutableName(goos string) string { + name := "devonthink-pp-cli" + if goos == "windows" { + return name + ".exe" + } + return name +} diff --git a/library/productivity/devonthink/internal/mcp/cobratree/names.go b/library/productivity/devonthink/internal/mcp/cobratree/names.go new file mode 100644 index 0000000000..d444919f11 --- /dev/null +++ b/library/productivity/devonthink/internal/mcp/cobratree/names.go @@ -0,0 +1,25 @@ +// Copyright 2026 rowdy and contributors. Licensed under Apache-2.0. See LICENSE. +// Generated by CLI Printing Press (https://github.com/mvanhorn/cli-printing-press). DO NOT EDIT. + +package cobratree + +import ( + "strings" + "unicode" +) + +func toolNameForPath(parts []string) string { + var out []rune + for _, part := range parts { + for _, r := range part { + switch { + case unicode.IsLetter(r) || unicode.IsDigit(r): + out = append(out, unicode.ToLower(r)) + default: + out = append(out, '_') + } + } + out = append(out, '_') + } + return strings.Trim(strings.Join(strings.FieldsFunc(string(out), func(r rune) bool { return r == '_' }), "_"), "_") +} diff --git a/library/productivity/devonthink/internal/mcp/cobratree/shellout.go b/library/productivity/devonthink/internal/mcp/cobratree/shellout.go new file mode 100644 index 0000000000..4ca9b37991 --- /dev/null +++ b/library/productivity/devonthink/internal/mcp/cobratree/shellout.go @@ -0,0 +1,161 @@ +// Copyright 2026 rowdy and contributors. Licensed under Apache-2.0. See LICENSE. +// Generated by CLI Printing Press (https://github.com/mvanhorn/cli-printing-press). DO NOT EDIT. + +package cobratree + +import ( + "bytes" + "context" + "fmt" + "os/exec" + "sort" + "strconv" + "strings" + + mcplib "github.com/mark3labs/mcp-go/mcp" + "github.com/mark3labs/mcp-go/server" +) + +func shellOutToCLI(cliPath func() (string, error), commandPath []string) server.ToolHandlerFunc { + lookupPath, lookupErr := cliPath() + prefixArgs := append([]string{}, commandPath...) + return func(ctx context.Context, req mcplib.CallToolRequest) (*mcplib.CallToolResult, error) { + if lookupErr != nil { + return mcplib.NewToolResultError(fmt.Sprintf("companion CLI binary not found: %v\nTried sibling lookup, DEVONTHINK_CLI_PATH env var, and PATH.", lookupErr)), nil + } + args := req.GetArguments() + finalArgs := append([]string{}, prefixArgs...) + finalArgs = append(finalArgs, cliArgsFromMCP(args)...) + if raw, _ := args["args"].(string); strings.TrimSpace(raw) != "" { + tokens := SplitShellArgs(raw) + for _, t := range tokens { + if strings.HasPrefix(t, "-") { + return mcplib.NewToolResultError(fmt.Sprintf("flag-like argument %q not allowed in positional args field; use structured tool parameters instead", t)), nil + } + } + finalArgs = append(finalArgs, tokens...) + } + out, err := RunCLICommand(ctx, lookupPath, finalArgs) + if err != nil { + return mcplib.NewToolResultError(err.Error()), nil + } + return mcplib.NewToolResultText(out), nil + } +} + +// blockedRootFlags are root-level CLI flags that an MCP client must not be +// able to override via structured tool parameters. Allowing them lets a +// caller swap auth credentials, redirect the API base URL, select a different +// per-client filesystem, load a malicious config file, or change the delivery +// target, all of which sit outside the per-command surface the agent is +// supposed to be calling. +var blockedRootFlags = map[string]bool{ + "args": true, + "base-url": true, + "client": true, + "config": true, + "deliver": true, + "profile": true, + "token": true, +} + +func cliArgsFromMCP(args map[string]any) []string { + keys := make([]string, 0, len(args)) + for k := range args { + if !blockedRootFlags[k] { + keys = append(keys, k) + } + } + sort.Strings(keys) + + var out []string + for _, k := range keys { + v := args[k] + switch tv := v.(type) { + case bool: + if tv { + out = append(out, "--"+k) + } + case float64: + out = append(out, "--"+k, strconv.FormatFloat(tv, 'f', -1, 64)) + case string: + if tv != "" { + out = append(out, "--"+k, tv) + } + case []any: + if len(tv) > 0 { + parts := make([]string, 0, len(tv)) + for _, item := range tv { + parts = append(parts, fmt.Sprintf("%v", item)) + } + out = append(out, "--"+k, strings.Join(parts, ",")) + } + default: + if v != nil { + out = append(out, "--"+k, fmt.Sprintf("%v", v)) + } + } + } + return out +} + +// SplitShellArgs whitespace-splits with shell-style quote preservation. +func SplitShellArgs(s string) []string { + var tokens []string + var cur []rune + inSingleQuote := false + inDoubleQuote := false + escaped := false + for _, r := range s { + switch { + case escaped: + cur = append(cur, r) + escaped = false + case r == '\\' && !inSingleQuote: + escaped = true + case r == '\'' && !inDoubleQuote: + inSingleQuote = !inSingleQuote + case r == '"' && !inSingleQuote: + inDoubleQuote = !inDoubleQuote + case (r == ' ' || r == '\t') && !inSingleQuote && !inDoubleQuote: + if len(cur) > 0 { + tokens = append(tokens, string(cur)) + cur = cur[:0] + } + default: + cur = append(cur, r) + } + } + if escaped { + cur = append(cur, '\\') + } + if len(cur) > 0 { + tokens = append(tokens, string(cur)) + } + return tokens +} + +// RunCLICommand executes the companion CLI while preserving stdout as the +// machine-readable channel. Stderr is included only in error text so post-run +// telemetry or quota output cannot corrupt JSON results. +func RunCLICommand(ctx context.Context, binPath string, args []string) (string, error) { + cmd := exec.CommandContext(ctx, binPath, args...) // #nosec G204 -- binPath is the current companion CLI binary selected by the MCP server. + var stdout, stderr bytes.Buffer + cmd.Stdout = &stdout + cmd.Stderr = &stderr + if err := cmd.Run(); err != nil { + msg := strings.TrimSpace(stderr.String()) + if msg == "" { + msg = strings.TrimSpace(stdout.String()) + } + if msg != "" { + label := "stderr" + if strings.TrimSpace(stderr.String()) == "" { + label = "output" + } + return stdout.String(), fmt.Errorf("cli %s: %w (%s: %s)", binPath, err, label, msg) + } + return stdout.String(), fmt.Errorf("cli %s: %w", binPath, err) + } + return stdout.String(), nil +} diff --git a/library/productivity/devonthink/internal/mcp/cobratree/shellout_test.go b/library/productivity/devonthink/internal/mcp/cobratree/shellout_test.go new file mode 100644 index 0000000000..9023bd33bd --- /dev/null +++ b/library/productivity/devonthink/internal/mcp/cobratree/shellout_test.go @@ -0,0 +1,228 @@ +// Copyright 2026 rowdy and contributors. Licensed under Apache-2.0. See LICENSE. +// Generated by CLI Printing Press (https://github.com/mvanhorn/cli-printing-press). DO NOT EDIT. + +package cobratree + +import ( + "context" + "os" + "path/filepath" + "reflect" + "runtime" + "strings" + "testing" +) + +// TestSplitShellArgs pins the whitespace + quote splitting used by +// the args-field passthrough. Behavior we rely on: bare whitespace splits, +// tabs split, quoted spans stay together, empty input yields nil. +func TestSplitShellArgs(t *testing.T) { + cases := []struct { + name string + in string + want []string + }{ + {"empty", "", nil}, + {"single token", "contacts", []string{"contacts"}}, + {"two tokens", "inbox health", []string{"inbox", "health"}}, + {"extra whitespace", " foo bar ", []string{"foo", "bar"}}, + {"tabs", "foo\tbar", []string{"foo", "bar"}}, + {"quoted token", `"hello world"`, []string{"hello world"}}, + {"mixed quoted and bare", `contacts "john doe" active`, []string{"contacts", "john doe", "active"}}, + {"double quoted apostrophe", `"My Club's Night"`, []string{"My Club's Night"}}, + {"single quoted span", `'a b' c`, []string{"a b", "c"}}, + {"escaped whitespace", `a\ b c`, []string{"a b", "c"}}, + } + for _, tc := range cases { + tc := tc + t.Run(tc.name, func(t *testing.T) { + got := SplitShellArgs(tc.in) + if !reflect.DeepEqual(got, tc.want) { + t.Errorf("splitShellArgs(%q) = %v, want %v", tc.in, got, tc.want) + } + }) + } +} + +// TestCliArgsFromMCP_BlocksRootFlags pins the structured-parameter half of +// the control-plane-injection guard: even when an MCP client wraps the +// flag in the structured args map (instead of the free-form "args" +// string), the root flags listed in blockedRootFlags must be dropped +// before they reach exec.CommandContext. A regression here would let a +// caller redirect --base-url, swap --token, switch --client filesystems, or +// load a malicious --config. +func TestCliArgsFromMCP_BlocksRootFlags(t *testing.T) { + in := map[string]any{ + "args": "contacts", + "base-url": "https://evil.example.com", + "client": "attacker-client", + "config": "/tmp/evil.yaml", + "deliver": "fd:3", + "profile": "attacker", + "token": "stolen-token", + // Allowed per-command flag passes through. + "limit": float64(10), + } + got := cliArgsFromMCP(in) + want := []string{"--limit", "10"} + if !reflect.DeepEqual(got, want) { + t.Fatalf("cliArgsFromMCP dropped/kept wrong keys: got %v, want %v", got, want) + } + for _, blocked := range []string{"--base-url", "--client", "--config", "--deliver", "--profile", "--token", "--args"} { + for _, tok := range got { + if tok == blocked { + t.Errorf("blocked flag %q leaked through cliArgsFromMCP", blocked) + } + } + } +} + +// TestCliArgsFromMCP_AllowsPerCommandFlags is the don't-overcorrect half: +// any flag NOT in blockedRootFlags must still pass through, including +// strings, bools, numbers, and []any. Without this, a tightening change +// to the blocklist that accidentally drops legitimate per-command flags +// would silently break every MCP-driven command. cliArgsFromMCP also +// guarantees sorted-key output (so generated commands see deterministic +// argv ordering); compare directly to catch a regression in either the +// blocklist or the sort. +func TestCliArgsFromMCP_AllowsPerCommandFlags(t *testing.T) { + in := map[string]any{ + "query": "alpha", + "verbose": true, + "limit": float64(25), + "tags": []any{"a", "b"}, + } + got := cliArgsFromMCP(in) + want := []string{"--limit", "25", "--query", "alpha", "--tags", "a,b", "--verbose"} + if !reflect.DeepEqual(got, want) { + t.Fatalf("cliArgsFromMCP per-command passthrough: got %v, want %v", got, want) + } +} + +// TestArgsFieldRejectsFlagLikeTokens covers the free-form "args" string +// half of the control-plane-injection guard. shellOutToCLI is a closure +// that requires a real binary on PATH; we exercise the same guard logic +// here so a regression that drops the strings.HasPrefix("-") check is +// caught at unit-test scope rather than only via end-to-end MCP runs. +func TestArgsFieldRejectsFlagLikeTokens(t *testing.T) { + guard := func(raw string) (rejected string, ok bool) { + for _, t := range SplitShellArgs(raw) { + if strings.HasPrefix(t, "-") { + return t, false + } + } + return "", true + } + cases := []struct { + name string + in string + wantOK bool + wantBlocked string + }{ + {"long flag", "--config /tmp/evil.yaml", false, "--config"}, + {"short flag", "-c", false, "-c"}, + {"bare double dash", "--", false, "--"}, + {"flag with equals", "--config=/tmp/evil.yaml", false, "--config=/tmp/evil.yaml"}, + {"flag mid-args", "contacts --verbose", false, "--verbose"}, + {"clean positional", "contacts", true, ""}, + {"two positionals", "inbox health", true, ""}, + {"empty", "", true, ""}, + // Shell metachars in unquoted positional content stay verbatim; + // the CLI is invoked via exec.CommandContext (no /bin/sh), so + // $VAR / && / | are inert. Pin that they don't trip the guard. + {"shell metachars in positional", `name with $VAR && pipe|stuff`, true, ""}, + } + for _, tc := range cases { + tc := tc + t.Run(tc.name, func(t *testing.T) { + tok, ok := guard(tc.in) + if ok != tc.wantOK { + t.Errorf("guard(%q) ok = %v, want %v", tc.in, ok, tc.wantOK) + } + if !tc.wantOK && tok != tc.wantBlocked { + t.Errorf("guard(%q) blocked = %q, want %q", tc.in, tok, tc.wantBlocked) + } + }) + } +} + +func TestRunCLICommandKeepsStdoutSeparateFromStderr(t *testing.T) { + bin := writeShelloutHelper(t, "success") + got, err := RunCLICommand(context.Background(), bin, []string{"alpha", "beta"}) + if err != nil { + t.Fatalf("RunCLICommand success returned error: %v", err) + } + if !strings.Contains(got, `{"args":"alpha beta"}`) { + t.Fatalf("stdout result missing JSON payload: %q", got) + } + if strings.Contains(got, "telemetry") { + t.Fatalf("stderr telemetry leaked into stdout: %q", got) + } +} + +func TestRunCLICommandReportsStderrOnFailure(t *testing.T) { + bin := writeShelloutHelper(t, "fail-stderr") + _, err := RunCLICommand(context.Background(), bin, nil) + if err == nil { + t.Fatal("RunCLICommand fail-stderr succeeded unexpectedly") + } + if !strings.Contains(err.Error(), "boom from stderr") { + t.Fatalf("error did not include stderr: %v", err) + } +} + +func TestRunCLICommandFallsBackToStdoutOnFailureWithoutStderr(t *testing.T) { + bin := writeShelloutHelper(t, "fail-stdout") + _, err := RunCLICommand(context.Background(), bin, nil) + if err == nil { + t.Fatal("RunCLICommand fail-stdout succeeded unexpectedly") + } + if !strings.Contains(err.Error(), "stdout failure") { + t.Fatalf("error did not include stdout fallback: %v", err) + } + if !strings.Contains(err.Error(), "output: stdout failure") { + t.Fatalf("error did not label stdout fallback as output: %v", err) + } + if strings.Contains(err.Error(), "stderr: stdout failure") { + t.Fatalf("stdout fallback mislabeled as stderr: %v", err) + } +} + +func writeShelloutHelper(t *testing.T, mode string) string { + t.Helper() + dir := t.TempDir() + if runtime.GOOS == "windows" { + path := filepath.Join(dir, "helper.bat") + var body string + switch mode { + case "success": + body = "@echo off\r\necho telemetry 1>&2\r\necho {\"args\":\"%*\"}\r\n" + case "fail-stderr": + body = "@echo off\r\necho boom from stderr 1>&2\r\nexit /b 7\r\n" + case "fail-stdout": + body = "@echo off\r\necho stdout failure\r\nexit /b 7\r\n" + default: + t.Fatalf("unknown mode %q", mode) + } + if err := os.WriteFile(path, []byte(body), 0o755); err != nil { + t.Fatalf("write helper: %v", err) + } + return path + } + path := filepath.Join(dir, "helper.sh") + var body string + switch mode { + case "success": + body = "#!/bin/sh\necho telemetry >&2\nprintf '{\"args\":\"%s\"}\\n' \"$*\"\n" + case "fail-stderr": + body = "#!/bin/sh\necho boom from stderr >&2\nexit 7\n" + case "fail-stdout": + body = "#!/bin/sh\necho stdout failure\nexit 7\n" + default: + t.Fatalf("unknown mode %q", mode) + } + if err := os.WriteFile(path, []byte(body), 0o755); err != nil { + t.Fatalf("write helper: %v", err) + } + return path +} diff --git a/library/productivity/devonthink/internal/mcp/cobratree/typemap.go b/library/productivity/devonthink/internal/mcp/cobratree/typemap.go new file mode 100644 index 0000000000..7f89d45438 --- /dev/null +++ b/library/productivity/devonthink/internal/mcp/cobratree/typemap.go @@ -0,0 +1,76 @@ +// Copyright 2026 rowdy and contributors. Licensed under Apache-2.0. See LICENSE. +// Generated by CLI Printing Press (https://github.com/mvanhorn/cli-printing-press). DO NOT EDIT. + +package cobratree + +import ( + "regexp" + "strings" + + mcplib "github.com/mark3labs/mcp-go/mcp" + "github.com/spf13/cobra" + "github.com/spf13/pflag" +) + +var positionalPattern = regexp.MustCompile(`(?:^|\s)(?:<[^>]+>|\[[^\]]+\])`) + +func toolOptionsForFlags(cmd *cobra.Command) []mcplib.ToolOption { + var opts []mcplib.ToolOption + seen := map[string]bool{} + addFlag := func(flag *pflag.Flag) { + if flag == nil || flag.Hidden || flag.Deprecated != "" { + return + } + if seen[flag.Name] { + return + } + seen[flag.Name] = true + opts = append(opts, toolOptionForFlag(flag)) + } + cmd.InheritedFlags().VisitAll(addFlag) + cmd.NonInheritedFlags().VisitAll(addFlag) + return opts +} + +func toolOptionForFlag(flag *pflag.Flag) mcplib.ToolOption { + propOpts := []mcplib.PropertyOption{mcplib.Description(flagDescription(flag))} + if isRequired(flag) { + propOpts = append(propOpts, mcplib.Required()) + } + switch flag.Value.Type() { + case "bool": + return mcplib.WithBoolean(flag.Name, propOpts...) + case "int", "int8", "int16", "int32", "int64", + "uint", "uint8", "uint16", "uint32", "uint64", + "float32", "float64", "count": + return mcplib.WithNumber(flag.Name, propOpts...) + case "string", "stringSlice", "stringArray", "duration": + return mcplib.WithString(flag.Name, propOpts...) + default: + propOpts[0] = mcplib.Description(flagDescription(flag) + " (unknown Cobra flag type " + flag.Value.Type() + "; pass as a string)") + return mcplib.WithString(flag.Name, propOpts...) + } +} + +func flagDescription(flag *pflag.Flag) string { + usage := strings.TrimSpace(flag.Usage) + if usage == "" { + usage = "Value for --" + flag.Name + } + if flag.DefValue != "" && flag.DefValue != "[]" { + usage += " (default: " + flag.DefValue + ")" + } + return usage +} + +func isRequired(flag *pflag.Flag) bool { + if flag == nil || flag.Annotations == nil { + return false + } + _, ok := flag.Annotations[cobra.BashCompOneRequiredFlag] + return ok +} + +func commandTakesArgs(cmd *cobra.Command) bool { + return positionalPattern.MatchString(cmd.Use) +} diff --git a/library/productivity/devonthink/internal/mcp/cobratree/walker.go b/library/productivity/devonthink/internal/mcp/cobratree/walker.go new file mode 100644 index 0000000000..7aa8fc9a47 --- /dev/null +++ b/library/productivity/devonthink/internal/mcp/cobratree/walker.go @@ -0,0 +1,66 @@ +// Copyright 2026 rowdy and contributors. Licensed under Apache-2.0. See LICENSE. +// Generated by CLI Printing Press (https://github.com/mvanhorn/cli-printing-press). DO NOT EDIT. + +package cobratree + +import ( + mcplib "github.com/mark3labs/mcp-go/mcp" + "github.com/mark3labs/mcp-go/server" + "github.com/spf13/cobra" +) + +// RegisterAll walks root's user-facing Cobra commands and registers shell-out +// MCP tools for commands that are not already covered by typed endpoint tools. +func RegisterAll(s *server.MCPServer, root *cobra.Command, cliPath func() (string, error)) { + if root == nil { + return + } + walk(root, nil, func(cmd *cobra.Command, path []string) { + switch classify(cmd) { + case commandHidden: + return + case commandEndpoint, commandFramework: + return + } + if !cmd.Runnable() { + return + } + + toolName := toolNameForPath(path) + if toolName == "" { + return + } + options := []mcplib.ToolOption{mcplib.WithDescription(descriptionFor(cmd))} + options = append(options, toolOptionsForFlags(cmd)...) + if commandTakesArgs(cmd) { + options = append(options, mcplib.WithString("args", mcplib.Description("Additional positional arguments or raw CLI flags to append to the command."))) + } + if isMCPReadOnly(cmd) { + options = append(options, mcplib.WithReadOnlyHintAnnotation(true), mcplib.WithDestructiveHintAnnotation(false)) + } + s.AddTool(mcplib.NewTool(toolName, options...), shellOutToCLI(cliPath, path)) + }) +} + +func walk(cmd *cobra.Command, path []string, visit func(*cobra.Command, []string)) { + for _, child := range cmd.Commands() { + if child.Hidden || isMCPHidden(child) { + continue + } + childPath := append(append([]string{}, path...), child.Name()) + visit(child, childPath) + if kind := classify(child); kind != commandHidden && kind != commandFramework { + walk(child, childPath, visit) + } + } +} + +func descriptionFor(cmd *cobra.Command) string { + if cmd.Long != "" { + return cmd.Long + } + if cmd.Short != "" { + return cmd.Short + } + return "Run `" + cmd.CommandPath() + "` through the companion CLI binary." +} diff --git a/library/productivity/devonthink/internal/mcp/tools.go b/library/productivity/devonthink/internal/mcp/tools.go new file mode 100644 index 0000000000..3383bf215b --- /dev/null +++ b/library/productivity/devonthink/internal/mcp/tools.go @@ -0,0 +1,1230 @@ +// Copyright 2026 rowdy and contributors. Licensed under Apache-2.0. See LICENSE. +// Generated by CLI Printing Press (https://github.com/mvanhorn/cli-printing-press). DO NOT EDIT. + +package mcp + +import ( + "context" + "encoding/base64" + "encoding/json" + "fmt" + "math" + "os" + "path/filepath" + "strconv" + "strings" + "time" + + mcplib "github.com/mark3labs/mcp-go/mcp" + "github.com/mark3labs/mcp-go/server" + "github.com/mvanhorn/printing-press-library/library/productivity/devonthink/internal/cli" + "github.com/mvanhorn/printing-press-library/library/productivity/devonthink/internal/client" + "github.com/mvanhorn/printing-press-library/library/productivity/devonthink/internal/config" + "github.com/mvanhorn/printing-press-library/library/productivity/devonthink/internal/mcp/cobratree" + "github.com/mvanhorn/printing-press-library/library/productivity/devonthink/internal/store" +) + +const ( + mcpToolResultMaxBytes = 60000 + mcpToolResultMaxItems = 50 + // MCP hosts can fan out tool calls faster than a human CLI session. + // Keep them on the same polite-client limiter path instead of disabling + // pacing with rate=0; users can still tune human CLI calls with --rate-limit. + defaultMCPRateLimit = 2 +) + +// RegisterTools registers all API operations as MCP tools. +func RegisterTools(s *server.MCPServer) { + s.AddTool( + mcplib.NewTool("ai_ask", + mcplib.WithDescription("Ask DEVONthink AI about selected local records with explicit cloud-use warnings. Required: prompt. Optional: uuid, selection, dry-run. Returns the new TextResult."), + mcplib.WithString("prompt", mcplib.Required(), mcplib.Description("Prompt to send")), + mcplib.WithString("uuid", mcplib.Description("Record UUID to attach")), + mcplib.WithBoolean("selection", mcplib.Description("Attach the current selection")), + mcplib.WithBoolean("dry-run", mcplib.Description("Preview attachments without sending")), + mcplib.WithDestructiveHintAnnotation(false), + mcplib.WithOpenWorldHintAnnotation(true), + ), + makeAPIHandler("POST", "/ai/ask", false, false, nil, []mcpParamBinding{{PublicName: "prompt", WireName: "prompt", Location: "query"}, {PublicName: "uuid", WireName: "uuid", Location: "body"}, {PublicName: "selection", WireName: "selection", Location: "body"}, {PublicName: "dry-run", WireName: "dry_run", Location: "body"}}, []string{"prompt"}), + ) + s.AddTool( + mcplib.NewTool("ai_summarize", + mcplib.WithDescription("Summarize records or highlights. Required: uuid. Optional: highlights, dry-run. Returns the new TextResult."), + mcplib.WithString("uuid", mcplib.Required(), mcplib.Description("Record UUID or item link")), + mcplib.WithBoolean("highlights", mcplib.Description("Summarize highlights only")), + mcplib.WithBoolean("dry-run", mcplib.Description("Preview attachments without sending")), + mcplib.WithDestructiveHintAnnotation(false), + mcplib.WithOpenWorldHintAnnotation(true), + ), + makeAPIHandler("POST", "/ai/summarize", false, false, nil, []mcpParamBinding{{PublicName: "uuid", WireName: "uuid", Location: "query"}, {PublicName: "highlights", WireName: "highlights", Location: "body"}, {PublicName: "dry-run", WireName: "dry_run", Location: "body"}}, []string{"uuid"}), + ) + s.AddTool( + mcplib.NewTool("batch_apply", + mcplib.WithDescription("Apply a previously reviewed local JSON plan. Required: plan. Optional: yes. Returns the new OperationResult."), + mcplib.WithString("plan", mcplib.Required(), mcplib.Description("Plan JSON path")), + mcplib.WithBoolean("yes", mcplib.Description("Confirm apply without an interactive prompt")), + mcplib.WithDestructiveHintAnnotation(false), + mcplib.WithOpenWorldHintAnnotation(true), + ), + makeAPIHandler("POST", "/batch/apply", false, false, nil, []mcpParamBinding{{PublicName: "plan", WireName: "plan", Location: "query"}, {PublicName: "yes", WireName: "yes", Location: "body"}}, []string{"plan"}), + ) + s.AddTool( + mcplib.NewTool("batch_plan", + mcplib.WithDescription("Stage multi-record changes as a local JSON plan. Optional: query, from, add-tag (plus 3 more). Returns the new BatchPlan."), + mcplib.WithString("query", mcplib.Description("Search query selecting records")), + mcplib.WithString("from", mcplib.Description("Source selector such as selection or file")), + mcplib.WithString("add-tag", mcplib.Description("Tag to add")), + mcplib.WithString("move-to", mcplib.Description("Destination group path or UUID")), + mcplib.WithString("output", mcplib.Description("Plan output path")), + mcplib.WithBoolean("dry-run", mcplib.Description("Preview only")), + mcplib.WithDestructiveHintAnnotation(false), + mcplib.WithOpenWorldHintAnnotation(true), + ), + makeAPIHandler("POST", "/batch/plan", false, false, nil, []mcpParamBinding{{PublicName: "query", WireName: "query", Location: "body"}, {PublicName: "from", WireName: "from", Location: "body"}, {PublicName: "add-tag", WireName: "add_tag", Location: "body"}, {PublicName: "move-to", WireName: "move_to", Location: "body"}, {PublicName: "output", WireName: "output", Location: "body"}, {PublicName: "dry-run", WireName: "dry_run", Location: "body"}}, []string{}), + ) + s.AddTool( + mcplib.NewTool("context_pack", + mcplib.WithDescription("Build a compact local context pack from records, selection, or search. Optional: query, uuid, selection (plus 1 more). Returns the ContextPack."), + mcplib.WithString("query", mcplib.Description("Search query to seed the pack")), + mcplib.WithString("uuid", mcplib.Description("Record UUID to seed the pack")), + mcplib.WithBoolean("selection", mcplib.Description("Use current DEVONthink selection")), + mcplib.WithNumber("token-budget", mcplib.Description("Approximate token budget")), + mcplib.WithReadOnlyHintAnnotation(true), + mcplib.WithDestructiveHintAnnotation(false), + mcplib.WithOpenWorldHintAnnotation(true), + ), + makeAPIHandler("GET", "/context/pack", true, false, nil, []mcpParamBinding{{PublicName: "query", WireName: "query", Location: "query"}, {PublicName: "uuid", WireName: "uuid", Location: "query"}, {PublicName: "selection", WireName: "selection", Location: "query"}, {PublicName: "token-budget", WireName: "token_budget", Location: "query", Default: "6000"}}, []string{}), + ) + s.AddTool( + mcplib.NewTool("databases_list", + mcplib.WithDescription("List open databases. Returns array of Database."), + mcplib.WithReadOnlyHintAnnotation(true), + mcplib.WithDestructiveHintAnnotation(false), + mcplib.WithOpenWorldHintAnnotation(true), + ), + makeAPIHandler("GET", "/databases", true, false, nil, []mcpParamBinding{}, []string{}), + ) + s.AddTool( + mcplib.NewTool("graph_audit", + mcplib.WithDescription("Detect orphans, unresolved wiki links, weak hubs, and tag-only clusters. Optional: database, limit (default: 50). Returns the GraphAudit."), + mcplib.WithString("database", mcplib.Description("Database name or UUID")), + mcplib.WithNumber("limit", mcplib.Description("Maximum sample records per issue")), + mcplib.WithReadOnlyHintAnnotation(true), + mcplib.WithDestructiveHintAnnotation(false), + mcplib.WithOpenWorldHintAnnotation(true), + ), + makeAPIHandler("GET", "/graph/audit", true, false, nil, []mcpParamBinding{{PublicName: "database", WireName: "database", Location: "query"}, {PublicName: "limit", WireName: "limit", Location: "query", Default: "50"}}, []string{}), + ) + s.AddTool( + mcplib.NewTool("graph_links", + mcplib.WithDescription("List item links, wiki links, mentions, and unresolved wiki names. Optional: uuid, database, direction (default: both). Returns the GraphLinks."), + mcplib.WithString("uuid", mcplib.Description("Record UUID or item link")), + mcplib.WithString("database", mcplib.Description("Database name or UUID")), + mcplib.WithString("direction", mcplib.Description("incoming, outgoing, or both")), + mcplib.WithReadOnlyHintAnnotation(true), + mcplib.WithDestructiveHintAnnotation(false), + mcplib.WithOpenWorldHintAnnotation(true), + ), + makeAPIHandler("GET", "/graph/links", true, false, nil, []mcpParamBinding{{PublicName: "uuid", WireName: "uuid", Location: "query"}, {PublicName: "database", WireName: "database", Location: "query"}, {PublicName: "direction", WireName: "direction", Location: "query", Default: "both"}}, []string{}), + ) + s.AddTool( + mcplib.NewTool("groups_tree", + mcplib.WithDescription("Render a bounded group tree. Optional: database, group, depth (default: 2). Returns the GroupTree."), + mcplib.WithString("database", mcplib.Description("Database name or UUID")), + mcplib.WithString("group", mcplib.Description("Root group UUID or path")), + mcplib.WithNumber("depth", mcplib.Description("Maximum tree depth")), + mcplib.WithReadOnlyHintAnnotation(true), + mcplib.WithDestructiveHintAnnotation(false), + mcplib.WithOpenWorldHintAnnotation(true), + ), + makeAPIHandler("GET", "/groups/tree", true, false, nil, []mcpParamBinding{{PublicName: "database", WireName: "database", Location: "query"}, {PublicName: "group", WireName: "group", Location: "query"}, {PublicName: "depth", WireName: "depth", Location: "query", Default: "2"}}, []string{}), + ) + s.AddTool( + mcplib.NewTool("ingest_file", + mcplib.WithDescription("Import or index a file or folder. Required: path. Optional: destination, mode (default: import). Returns the new OperationResult."), + mcplib.WithString("path", mcplib.Required(), mcplib.Description("File or folder path")), + mcplib.WithString("destination", mcplib.Description("Destination group path or UUID")), + mcplib.WithString("mode", mcplib.Description("import or index")), + mcplib.WithDestructiveHintAnnotation(false), + mcplib.WithOpenWorldHintAnnotation(true), + ), + makeAPIHandler("POST", "/ingest/file", false, false, nil, []mcpParamBinding{{PublicName: "path", WireName: "path", Location: "query"}, {PublicName: "destination", WireName: "destination", Location: "body"}, {PublicName: "mode", WireName: "mode", Location: "body"}}, []string{"path"}), + ) + s.AddTool( + mcplib.NewTool("ingest_url", + mcplib.WithDescription("Capture a URL as Markdown, HTML, PDF, bookmark, or webarchive. Required: url. Optional: destination, type (default: markdown). Returns the new OperationResult."), + mcplib.WithString("url", mcplib.Required(), mcplib.Description("URL to capture")), + mcplib.WithString("destination", mcplib.Description("Destination group path or UUID")), + mcplib.WithString("type", mcplib.Description("Capture format")), + mcplib.WithDestructiveHintAnnotation(false), + mcplib.WithOpenWorldHintAnnotation(true), + ), + makeAPIHandler("POST", "/ingest/url", false, false, nil, []mcpParamBinding{{PublicName: "url", WireName: "url", Location: "query"}, {PublicName: "destination", WireName: "destination", Location: "body"}, {PublicName: "type", WireName: "type", Location: "body"}}, []string{"url"}), + ) + s.AddTool( + mcplib.NewTool("inventory_export", + mcplib.WithDescription("Export databases, groups, tags, and selected document metadata for downstream tools. Optional: database, format (default: maintenance), include-text (plus 1 more). Returns the InventoryExport."), + mcplib.WithString("database", mcplib.Description("Database name or UUID")), + mcplib.WithString("format", mcplib.Description("Export format: maintenance, llm-wiki, raw")), + mcplib.WithBoolean("include-text", mcplib.Description("Include extracted text where safe")), + mcplib.WithString("output", mcplib.Description("Output JSON path")), + mcplib.WithReadOnlyHintAnnotation(true), + mcplib.WithDestructiveHintAnnotation(false), + mcplib.WithOpenWorldHintAnnotation(true), + ), + makeAPIHandler("GET", "/inventory/export", true, false, nil, []mcpParamBinding{{PublicName: "database", WireName: "database", Location: "query"}, {PublicName: "format", WireName: "format", Location: "query", Default: "maintenance"}, {PublicName: "include-text", WireName: "include_text", Location: "query"}, {PublicName: "output", WireName: "output", Location: "query"}}, []string{}), + ) + s.AddTool( + mcplib.NewTool("ledger_list", + mcplib.WithDescription("List recent CLI operation ledger entries. Optional: since, limit (default: 50). Returns array of LedgerEntry."), + mcplib.WithString("since", mcplib.Description("Time window such as 7d")), + mcplib.WithNumber("limit", mcplib.Description("Maximum entries")), + mcplib.WithReadOnlyHintAnnotation(true), + mcplib.WithDestructiveHintAnnotation(false), + mcplib.WithOpenWorldHintAnnotation(true), + ), + makeAPIHandler("GET", "/ledger", true, false, nil, []mcpParamBinding{{PublicName: "since", WireName: "since", Location: "query"}, {PublicName: "limit", WireName: "limit", Location: "query", Default: "50"}}, []string{}), + ) + s.AddTool( + mcplib.NewTool("ledger_show", + mcplib.WithDescription("Show one ledger entry with target proofs and rollback hints. Required: id. Returns the LedgerEntry."), + mcplib.WithString("id", mcplib.Required(), mcplib.Description("Ledger entry ID")), + mcplib.WithReadOnlyHintAnnotation(true), + mcplib.WithDestructiveHintAnnotation(false), + mcplib.WithOpenWorldHintAnnotation(true), + ), + makeAPIHandler("GET", "/ledger/{id}", true, false, nil, []mcpParamBinding{{PublicName: "id", WireName: "id", Location: "path"}}, []string{"id"}), + ) + s.AddTool( + mcplib.NewTool("mcp_call", + mcplib.WithDescription("Call a local official DEVONthink MCP tool by name. Required: tool. Optional: args. Returns the new TextResult."), + mcplib.WithString("tool", mcplib.Required(), mcplib.Description("Official MCP tool name")), + mcplib.WithString("args", mcplib.Description("JSON argument object")), + mcplib.WithDestructiveHintAnnotation(false), + mcplib.WithOpenWorldHintAnnotation(true), + ), + makeAPIHandler("POST", "/mcp/call", false, false, nil, []mcpParamBinding{{PublicName: "tool", WireName: "tool", Location: "query"}, {PublicName: "args", WireName: "args", Location: "body"}}, []string{"tool"}), + ) + s.AddTool( + mcplib.NewTool("mcp_schema", + mcplib.WithDescription("Emit cached MCP tool schemas. Returns the TextResult."), + mcplib.WithReadOnlyHintAnnotation(true), + mcplib.WithDestructiveHintAnnotation(false), + mcplib.WithOpenWorldHintAnnotation(true), + ), + makeAPIHandler("GET", "/mcp/schema", true, false, nil, []mcpParamBinding{}, []string{}), + ) + s.AddTool( + mcplib.NewTool("mcp_tools", + mcplib.WithDescription("List official DEVONthink MCP tools when local MCP HTTP is enabled. Returns array of MCPTool."), + mcplib.WithReadOnlyHintAnnotation(true), + mcplib.WithDestructiveHintAnnotation(false), + mcplib.WithOpenWorldHintAnnotation(true), + ), + makeAPIHandler("GET", "/mcp/tools", true, false, nil, []mcpParamBinding{}, []string{}), + ) + s.AddTool( + mcplib.NewTool("media_ocr", + mcplib.WithDescription("OCR an image or scanned PDF. Required: uuid. Optional: format (default: pdf). Returns the new OperationResult."), + mcplib.WithString("uuid", mcplib.Required(), mcplib.Description("Record UUID or item link")), + mcplib.WithString("format", mcplib.Description("Output format")), + mcplib.WithDestructiveHintAnnotation(false), + mcplib.WithOpenWorldHintAnnotation(true), + ), + makeAPIHandler("POST", "/media/{uuid}/ocr", false, false, nil, []mcpParamBinding{{PublicName: "uuid", WireName: "uuid", Location: "path"}, {PublicName: "format", WireName: "format", Location: "body"}}, []string{"uuid"}), + ) + s.AddTool( + mcplib.NewTool("media_transcribe", + mcplib.WithDescription("Transcribe audio, video, image, or PDF content. Required: uuid. Optional: language, timestamps. Returns the new TextResult."), + mcplib.WithString("uuid", mcplib.Required(), mcplib.Description("Record UUID or item link")), + mcplib.WithString("language", mcplib.Description("ISO language code")), + mcplib.WithBoolean("timestamps", mcplib.Description("Include timestamps when available")), + mcplib.WithDestructiveHintAnnotation(false), + mcplib.WithOpenWorldHintAnnotation(true), + ), + makeAPIHandler("POST", "/media/{uuid}/transcribe", false, false, nil, []mcpParamBinding{{PublicName: "uuid", WireName: "uuid", Location: "path"}, {PublicName: "language", WireName: "language", Location: "body"}, {PublicName: "timestamps", WireName: "timestamps", Location: "body"}}, []string{"uuid"}), + ) + s.AddTool( + mcplib.NewTool("mirror_search", + mcplib.WithDescription("Search the local mirror with FTS. Required: query. Optional: limit (default: 20). Returns array of Record."), + mcplib.WithString("query", mcplib.Required(), mcplib.Description("FTS query")), + mcplib.WithNumber("limit", mcplib.Description("Maximum rows")), + mcplib.WithReadOnlyHintAnnotation(true), + mcplib.WithDestructiveHintAnnotation(false), + mcplib.WithOpenWorldHintAnnotation(true), + ), + makeAPIHandler("GET", "/mirror/search", true, false, nil, []mcpParamBinding{{PublicName: "query", WireName: "query", Location: "query"}, {PublicName: "limit", WireName: "limit", Location: "query", Default: "20"}}, []string{"query"}), + ) + s.AddTool( + mcplib.NewTool("mirror_sync", + mcplib.WithDescription("Refresh the local SQLite mirror from open DEVONthink databases. Optional: database, full. Returns the SyncResult."), + mcplib.WithString("database", mcplib.Description("Database name or UUID")), + mcplib.WithBoolean("full", mcplib.Description("Force full sync")), + mcplib.WithReadOnlyHintAnnotation(true), + mcplib.WithDestructiveHintAnnotation(false), + mcplib.WithOpenWorldHintAnnotation(true), + ), + makeAPIHandler("GET", "/mirror/sync", true, false, nil, []mcpParamBinding{{PublicName: "database", WireName: "database", Location: "query"}, {PublicName: "full", WireName: "full", Location: "query"}}, []string{}), + ) + s.AddTool( + mcplib.NewTool("privacy_audit", + mcplib.WithDescription("Preview database scope, content-size budget, and cloud/MCP exposure before handoff. Optional: query, uuid, limit (default: 20). Returns the PrivacyAudit."), + mcplib.WithString("query", mcplib.Description("Search query to audit")), + mcplib.WithString("uuid", mcplib.Description("Record UUID or item link")), + mcplib.WithNumber("limit", mcplib.Description("Maximum records to inspect")), + mcplib.WithReadOnlyHintAnnotation(true), + mcplib.WithDestructiveHintAnnotation(false), + mcplib.WithOpenWorldHintAnnotation(true), + ), + makeAPIHandler("GET", "/privacy/audit", true, false, nil, []mcpParamBinding{{PublicName: "query", WireName: "query", Location: "query"}, {PublicName: "uuid", WireName: "uuid", Location: "query"}, {PublicName: "limit", WireName: "limit", Location: "query", Default: "20"}}, []string{}), + ) + s.AddTool( + mcplib.NewTool("records_content", + mcplib.WithDescription("Extract text content with length and redaction controls. Required: uuid. Optional: max-chars (default: 8000). Returns the RecordContent."), + mcplib.WithString("uuid", mcplib.Required(), mcplib.Description("Record UUID or item link")), + mcplib.WithNumber("max-chars", mcplib.Description("Maximum characters to emit")), + mcplib.WithReadOnlyHintAnnotation(true), + mcplib.WithDestructiveHintAnnotation(false), + mcplib.WithOpenWorldHintAnnotation(true), + ), + makeAPIHandler("GET", "/records/{uuid}/content", true, false, nil, []mcpParamBinding{{PublicName: "uuid", WireName: "uuid", Location: "path"}, {PublicName: "max-chars", WireName: "max_chars", Location: "query", Default: "8000"}}, []string{"uuid"}), + ) + s.AddTool( + mcplib.NewTool("records_create", + mcplib.WithDescription("Create a record or group after validating destination. Required: title. Optional: type (default: markdown), destination, content (plus 1 more). Returns the new OperationResult."), + mcplib.WithString("title", mcplib.Required(), mcplib.Description("Record title")), + mcplib.WithString("type", mcplib.Description("Record type")), + mcplib.WithString("destination", mcplib.Description("Destination group path or UUID")), + mcplib.WithString("content", mcplib.Description("Inline text content")), + mcplib.WithString("tags", mcplib.Description("Comma-separated tags")), + mcplib.WithDestructiveHintAnnotation(false), + mcplib.WithOpenWorldHintAnnotation(true), + ), + makeAPIHandler("POST", "/records/create", false, false, nil, []mcpParamBinding{{PublicName: "title", WireName: "title", Location: "body"}, {PublicName: "type", WireName: "type", Location: "body"}, {PublicName: "destination", WireName: "destination", Location: "body"}, {PublicName: "content", WireName: "content", Location: "body"}, {PublicName: "tags", WireName: "tags", Location: "body"}}, []string{}), + ) + s.AddTool( + mcplib.NewTool("records_get", + mcplib.WithDescription("Get record metadata. Required: uuid. Returns the Record."), + mcplib.WithString("uuid", mcplib.Required(), mcplib.Description("Record UUID or item link")), + mcplib.WithReadOnlyHintAnnotation(true), + mcplib.WithDestructiveHintAnnotation(false), + mcplib.WithOpenWorldHintAnnotation(true), + ), + makeAPIHandler("GET", "/records/{uuid}", true, false, nil, []mcpParamBinding{{PublicName: "uuid", WireName: "uuid", Location: "path"}}, []string{"uuid"}), + ) + s.AddTool( + mcplib.NewTool("records_highlights", + mcplib.WithDescription("Extract highlights and annotations. Required: uuid. Returns the TextResult."), + mcplib.WithString("uuid", mcplib.Required(), mcplib.Description("Record UUID or item link")), + mcplib.WithReadOnlyHintAnnotation(true), + mcplib.WithDestructiveHintAnnotation(false), + mcplib.WithOpenWorldHintAnnotation(true), + ), + makeAPIHandler("GET", "/records/{uuid}/highlights", true, false, nil, []mcpParamBinding{{PublicName: "uuid", WireName: "uuid", Location: "path"}}, []string{"uuid"}), + ) + s.AddTool( + mcplib.NewTool("records_lookup", + mcplib.WithDescription("Look up records by exact name, URL, path, filename, location, or comment. Optional: name, url, path (plus 3 more). Returns array of Record."), + mcplib.WithString("name", mcplib.Description("Exact record name")), + mcplib.WithString("url", mcplib.Description("Exact record URL")), + mcplib.WithString("path", mcplib.Description("Exact filesystem path")), + mcplib.WithString("location", mcplib.Description("Exact DEVONthink location")), + mcplib.WithString("filename", mcplib.Description("Exact filename")), + mcplib.WithString("comment", mcplib.Description("Exact comment")), + mcplib.WithReadOnlyHintAnnotation(true), + mcplib.WithDestructiveHintAnnotation(false), + mcplib.WithOpenWorldHintAnnotation(true), + ), + makeAPIHandler("GET", "/records/lookup", true, false, nil, []mcpParamBinding{{PublicName: "name", WireName: "name", Location: "query"}, {PublicName: "url", WireName: "url", Location: "query"}, {PublicName: "path", WireName: "path", Location: "query"}, {PublicName: "location", WireName: "location", Location: "query"}, {PublicName: "filename", WireName: "filename", Location: "query"}, {PublicName: "comment", WireName: "comment", Location: "query"}}, []string{}), + ) + s.AddTool( + mcplib.NewTool("records_move", + mcplib.WithDescription("Move, duplicate, replicate, or trash a record with dry-run proof. Required: uuid. Optional: destination, mode (default: move), dry-run. Returns the new OperationResult."), + mcplib.WithString("uuid", mcplib.Required(), mcplib.Description("Record UUID or item link")), + mcplib.WithString("destination", mcplib.Description("Destination group path or UUID")), + mcplib.WithString("mode", mcplib.Description("Operation mode: move, duplicate, replicate, trash")), + mcplib.WithBoolean("dry-run", mcplib.Description("Show the planned move without writing")), + mcplib.WithDestructiveHintAnnotation(false), + mcplib.WithOpenWorldHintAnnotation(true), + ), + makeAPIHandler("POST", "/records/{uuid}/move", false, false, nil, []mcpParamBinding{{PublicName: "uuid", WireName: "uuid", Location: "path"}, {PublicName: "destination", WireName: "destination", Location: "body"}, {PublicName: "mode", WireName: "mode", Location: "body"}, {PublicName: "dry-run", WireName: "dry_run", Location: "body"}}, []string{"uuid"}), + ) + s.AddTool( + mcplib.NewTool("records_related", + mcplib.WithDescription("Find related records using DEVONthink similarity. Required: uuid. Optional: limit (default: 20). Returns array of Record."), + mcplib.WithString("uuid", mcplib.Required(), mcplib.Description("Record UUID or item link")), + mcplib.WithNumber("limit", mcplib.Description("Maximum related records")), + mcplib.WithReadOnlyHintAnnotation(true), + mcplib.WithDestructiveHintAnnotation(false), + mcplib.WithOpenWorldHintAnnotation(true), + ), + makeAPIHandler("GET", "/records/{uuid}/related", true, false, nil, []mcpParamBinding{{PublicName: "uuid", WireName: "uuid", Location: "path"}, {PublicName: "limit", WireName: "limit", Location: "query", Default: "20"}}, []string{"uuid"}), + ) + s.AddTool( + mcplib.NewTool("records_search", + mcplib.WithDescription("Search records using DEVONthink query syntax or local mirror fallback. Required: query. Optional: database, group, limit (default: 20). Returns array of Record."), + mcplib.WithString("query", mcplib.Required(), mcplib.Description("DEVONthink search query")), + mcplib.WithString("database", mcplib.Description("Database name or UUID")), + mcplib.WithString("group", mcplib.Description("Group UUID or path")), + mcplib.WithNumber("limit", mcplib.Description("Maximum records to return")), + mcplib.WithReadOnlyHintAnnotation(true), + mcplib.WithDestructiveHintAnnotation(false), + mcplib.WithOpenWorldHintAnnotation(true), + ), + makeAPIHandler("GET", "/records/search", true, false, nil, []mcpParamBinding{{PublicName: "query", WireName: "query", Location: "query"}, {PublicName: "database", WireName: "database", Location: "query"}, {PublicName: "group", WireName: "group", Location: "query"}, {PublicName: "limit", WireName: "limit", Location: "query", Default: "20"}}, []string{"query"}), + ) + s.AddTool( + mcplib.NewTool("records_update", + mcplib.WithDescription("Update record text, properties, tags, comment, URL, aliases, or rating. Required: uuid. Optional: title, tags, add-tag (plus 3 more). Returns the updated OperationResult."), + mcplib.WithString("uuid", mcplib.Required(), mcplib.Description("Record UUID or item link")), + mcplib.WithString("title", mcplib.Description("New title")), + mcplib.WithString("tags", mcplib.Description("Replacement comma-separated tags")), + mcplib.WithString("add-tag", mcplib.Description("Tag to add")), + mcplib.WithString("comment", mcplib.Description("New comment")), + mcplib.WithString("append", mcplib.Description("Text to append")), + mcplib.WithBoolean("dry-run", mcplib.Description("Show the planned update without writing")), + mcplib.WithOpenWorldHintAnnotation(true), + ), + makeAPIHandler("PATCH", "/records/{uuid}", false, false, nil, []mcpParamBinding{{PublicName: "uuid", WireName: "uuid", Location: "path"}, {PublicName: "title", WireName: "title", Location: "body"}, {PublicName: "tags", WireName: "tags", Location: "body"}, {PublicName: "add-tag", WireName: "add_tag", Location: "body"}, {PublicName: "comment", WireName: "comment", Location: "body"}, {PublicName: "append", WireName: "append", Location: "body"}, {PublicName: "dry-run", WireName: "dry_run", Location: "body"}}, []string{"uuid"}), + ) + s.AddTool( + mcplib.NewTool("records_versions", + mcplib.WithDescription("List saved record versions. Required: uuid. Returns array of Record."), + mcplib.WithString("uuid", mcplib.Required(), mcplib.Description("Record UUID or item link")), + mcplib.WithReadOnlyHintAnnotation(true), + mcplib.WithDestructiveHintAnnotation(false), + mcplib.WithOpenWorldHintAnnotation(true), + ), + makeAPIHandler("GET", "/records/{uuid}/versions", true, false, nil, []mcpParamBinding{{PublicName: "uuid", WireName: "uuid", Location: "path"}}, []string{"uuid"}), + ) + s.AddTool( + mcplib.NewTool("runtime_doctor", + mcplib.WithDescription("Check DEVONthink app, AppleScript, optional MCP, and local mirror readiness. Returns the RuntimeStatus."), + mcplib.WithReadOnlyHintAnnotation(true), + mcplib.WithDestructiveHintAnnotation(false), + mcplib.WithOpenWorldHintAnnotation(true), + ), + makeAPIHandler("GET", "/runtime/doctor", true, false, nil, []mcpParamBinding{}, []string{}), + ) + s.AddTool( + mcplib.NewTool("selection_get", + mcplib.WithDescription("Return currently selected records. Returns array of Record."), + mcplib.WithReadOnlyHintAnnotation(true), + mcplib.WithDestructiveHintAnnotation(false), + mcplib.WithOpenWorldHintAnnotation(true), + ), + makeAPIHandler("GET", "/selection", true, false, nil, []mcpParamBinding{}, []string{}), + ) + s.AddTool( + mcplib.NewTool("selection_snapshot", + mcplib.WithDescription("Capture the current selection as a reusable workflow seed. Optional: note, output. Returns the SelectionSnapshot."), + mcplib.WithString("note", mcplib.Description("Operator note to include in the snapshot")), + mcplib.WithString("output", mcplib.Description("Optional output JSON file")), + mcplib.WithReadOnlyHintAnnotation(true), + mcplib.WithDestructiveHintAnnotation(false), + mcplib.WithOpenWorldHintAnnotation(true), + ), + makeAPIHandler("GET", "/selection/snapshot", true, false, nil, []mcpParamBinding{{PublicName: "note", WireName: "note", Location: "query"}, {PublicName: "output", WireName: "output", Location: "query"}}, []string{}), + ) + s.AddTool( + mcplib.NewTool("sheets_get", + mcplib.WithDescription("Read a sheet as structured rows. Required: uuid. Returns the SheetResult."), + mcplib.WithString("uuid", mcplib.Required(), mcplib.Description("Sheet UUID or item link")), + mcplib.WithReadOnlyHintAnnotation(true), + mcplib.WithDestructiveHintAnnotation(false), + mcplib.WithOpenWorldHintAnnotation(true), + ), + makeAPIHandler("GET", "/sheets/{uuid}", true, false, nil, []mcpParamBinding{{PublicName: "uuid", WireName: "uuid", Location: "path"}}, []string{"uuid"}), + ) + s.AddTool( + mcplib.NewTool("tags_analyze", + mcplib.WithDescription("Analyze tags for duplicates, case drift, action tags, and maintenance tags. Optional: database. Returns the TagAnalysis."), + mcplib.WithString("database", mcplib.Description("Database name or UUID")), + mcplib.WithReadOnlyHintAnnotation(true), + mcplib.WithDestructiveHintAnnotation(false), + mcplib.WithOpenWorldHintAnnotation(true), + ), + makeAPIHandler("GET", "/tags/analyze", true, false, nil, []mcpParamBinding{{PublicName: "database", WireName: "database", Location: "query"}}, []string{}), + ) + // Search tool — faster than iterating list endpoints for finding specific items + s.AddTool( + mcplib.NewTool("search", + mcplib.WithDescription("Full-text search across all synced data. Faster than paginating list endpoints. Requires sync first."), + mcplib.WithString("query", mcplib.Required(), mcplib.Description("Search query (supports FTS5 syntax: AND, OR, NOT, quotes for phrases)")), + mcplib.WithNumber("limit", mcplib.Description("Max results (default 25)")), + mcplib.WithReadOnlyHintAnnotation(true), + mcplib.WithDestructiveHintAnnotation(false), + ), + handleSearch, + ) + // SQL tool — ad-hoc analysis on synced data without API calls + s.AddTool( + mcplib.NewTool("sql", + mcplib.WithDescription("Run read-only SQL against local database. Use for ad-hoc analysis, aggregations, and joins across synced resources. Requires sync first."), + mcplib.WithString("query", mcplib.Required(), mcplib.Description("SQL query (SELECT or WITH...SELECT). Synced records live in resources(resource_type, id, data); filter by resource_type and use json_extract on data, e.g. SELECT json_extract(data,'$.name') FROM resources WHERE resource_type='items'.")), + mcplib.WithReadOnlyHintAnnotation(true), + mcplib.WithDestructiveHintAnnotation(false), + ), + handleSQL, + ) + + // Context tool — front-loaded domain knowledge for agents. + // Call this first to understand the API taxonomy, query patterns, and capabilities. + s.AddTool( + mcplib.NewTool("context", + mcplib.WithDescription("Get API domain context: resource taxonomy, auth requirements, query tips, and unique capabilities. Call this first."), + mcplib.WithReadOnlyHintAnnotation(true), + mcplib.WithDestructiveHintAnnotation(false), + ), + handleContext, + ) + + // Runtime Cobra-tree mirror — exposes every user-facing command that is + // not already covered by a typed endpoint or framework MCP tool. + cobratree.RegisterAll(s, cli.RootCmd(), cobratree.SiblingCLIPath) +} + +type mcpParamBinding struct { + PublicName string + WireName string + Location string + Default string +} + +func formatMCPParamValue(v any) string { + switch tv := v.(type) { + case string: + return tv + case bool: + return strconv.FormatBool(tv) + case float64: + if math.IsNaN(tv) || math.IsInf(tv, 0) { + return strconv.FormatFloat(tv, 'f', -1, 64) + } + if math.Trunc(tv) == tv && math.Abs(tv) < 1e15 { + return strconv.FormatInt(int64(tv), 10) + } + return strconv.FormatFloat(tv, 'f', -1, 64) + case float32: + f := float64(tv) + if math.IsNaN(f) || math.IsInf(f, 0) { + return strconv.FormatFloat(f, 'f', -1, 32) + } + if math.Trunc(f) == f && math.Abs(f) < 1e15 { + return strconv.FormatInt(int64(f), 10) + } + return strconv.FormatFloat(f, 'f', -1, 32) + default: + return fmt.Sprintf("%v", v) + } +} + +// makeAPIHandler creates a generic MCP tool handler for an API endpoint. +func makeAPIHandler(method, pathTemplate string, readOnly bool, binaryResponse bool, headerOverrides map[string]string, bindings []mcpParamBinding, positionalParams []string) server.ToolHandlerFunc { + return func(ctx context.Context, req mcplib.CallToolRequest) (*mcplib.CallToolResult, error) { + c, err := newMCPClient() + if err != nil { + return mcplib.NewToolResultError(err.Error()), nil + } + + // mcp-go v0.47+ made CallToolParams.Arguments an `any` to support + // non-map payloads; GetArguments() returns the map[string]any shape + // we rely on here (or an empty map when the payload is something else). + args := req.GetArguments() + + // positionalParams mixes real URL path params with CLI positional + // args that map to query params (e.g. `search <query>` -> ?query=); + // the placeholder check below disambiguates them at runtime. + path := pathTemplate + knownArgs := make(map[string]bool, len(bindings)) + pathParams := make(map[string]bool, len(positionalParams)) + params := make(map[string]string) + bodyArgs := make(map[string]any) + var headers map[string]string + if len(headerOverrides) > 0 { + headers = make(map[string]string, len(headerOverrides)+1) + for k, v := range headerOverrides { + headers[k] = v + } + } + if binaryResponse { + if headers == nil { + headers = map[string]string{} + } + headers[client.BinaryResponseHeader] = "true" + } + for _, binding := range bindings { + knownArgs[binding.PublicName] = true + v, ok := args[binding.PublicName] + if !ok { + if binding.Default != "" { + v = binding.Default + } else { + continue + } + } + switch binding.Location { + case "path": + placeholder := "{" + binding.WireName + "}" + pathParams[binding.PublicName] = true + path = strings.Replace(path, placeholder, formatMCPParamValue(v), 1) + case "body": + bodyArgs[binding.WireName] = v + default: + params[binding.WireName] = formatMCPParamValue(v) + } + } + for _, p := range positionalParams { + placeholder := "{" + p + "}" + if !strings.Contains(pathTemplate, placeholder) { + continue + } + pathParams[p] = true + if v, ok := args[p]; ok { + path = strings.Replace(path, placeholder, formatMCPParamValue(v), 1) + } + } + + for k, v := range args { + if pathParams[k] || knownArgs[k] { + continue + } + switch method { + case "POST", "PUT", "PATCH": + bodyArgs[k] = v + default: + params[k] = formatMCPParamValue(v) + } + } + + var data json.RawMessage + switch method { + case "GET": + if len(headers) > 0 { + data, err = c.GetWithHeaders(ctx, path, params, headers) + break + } + data, err = c.Get(ctx, path, params) + case "POST": + if len(headers) > 0 { + if readOnly { + data, _, err = c.PostQueryWithParamsAndHeaders(ctx, path, params, bodyArgs, headers) + } else { + data, _, err = c.PostWithParamsAndHeaders(ctx, path, params, bodyArgs, headers) + } + break + } + if readOnly { + data, _, err = c.PostQueryWithParams(ctx, path, params, bodyArgs) + } else { + data, _, err = c.PostWithParams(ctx, path, params, bodyArgs) + } + case "PUT": + if len(headers) > 0 { + data, _, err = c.PutWithParamsAndHeaders(ctx, path, params, bodyArgs, headers) + break + } + data, _, err = c.PutWithParams(ctx, path, params, bodyArgs) + case "PATCH": + if len(headers) > 0 { + data, _, err = c.PatchWithParamsAndHeaders(ctx, path, params, bodyArgs, headers) + break + } + data, _, err = c.PatchWithParams(ctx, path, params, bodyArgs) + case "DELETE": + if len(headers) > 0 { + data, _, err = c.DeleteWithParamsAndHeaders(ctx, path, params, headers) + break + } + data, _, err = c.DeleteWithParams(ctx, path, params) + default: + return mcplib.NewToolResultError("unsupported method: " + method), nil + } + + if err != nil { + msg := err.Error() + switch { + case strings.Contains(msg, "HTTP 409"): + return mcplib.NewToolResultText("already exists (no-op)"), nil + case strings.Contains(msg, "HTTP 401"): + return mcplib.NewToolResultError("authentication failed: " + msg + + "\nhint: check your API credentials." + + "\n Run 'devonthink-pp-cli doctor' to check auth status."), nil + case strings.Contains(msg, "HTTP 403"): + return mcplib.NewToolResultError("permission denied: " + msg + + "\nhint: this API is configured without credentials; the service may be blocking the request by rate limit, geography, bot protection, or endpoint policy." + + "\n Run 'devonthink-pp-cli doctor' to check auth status."), nil + case strings.Contains(msg, "HTTP 404"): + if method == "DELETE" { + return mcplib.NewToolResultText("already deleted (no-op)"), nil + } + return mcplib.NewToolResultError("not found: " + msg), nil + case strings.Contains(msg, "HTTP 429"): + return mcplib.NewToolResultError("rate limited: " + msg), nil + default: + return mcplib.NewToolResultError(msg), nil + } + } + + if binaryResponse { + out, _ := json.Marshal(map[string]any{ + "content_encoding": "base64", + "data_base64": base64.StdEncoding.EncodeToString(data), + "byte_count": len(data), + }) + return mcplib.NewToolResultText(string(out)), nil + } + return mcpToolResultText(method, data), nil + } +} + +func mcpToolResultText(method string, data json.RawMessage) *mcplib.CallToolResult { + trimmed := strings.TrimSpace(string(data)) + if strings.EqualFold(method, "GET") && len(trimmed) > 0 && trimmed[0] == '[' { + var items []json.RawMessage + if json.Unmarshal(data, &items) == nil { + return mcplib.NewToolResultText(string(mcpBoundedListEnvelope("items", items, len(data)))) + } + } + if len(data) <= mcpToolResultMaxBytes { + return mcplib.NewToolResultText(string(data)) + } + if strings.EqualFold(method, "GET") { + if out, ok := mcpBoundedSingleArrayObject(data); ok { + return mcplib.NewToolResultText(string(out)) + } + } + return mcplib.NewToolResultText(string(mcpOversizedPreviewEnvelope(data))) +} + +func mcpBoundedSingleArrayObject(data json.RawMessage) ([]byte, bool) { + var obj map[string]json.RawMessage + if json.Unmarshal(data, &obj) != nil { + return nil, false + } + arrayField := "" + var items []json.RawMessage + for key, raw := range obj { + trimmed := strings.TrimSpace(string(raw)) + if len(trimmed) == 0 || trimmed[0] != '[' { + continue + } + var candidate []json.RawMessage + if json.Unmarshal(raw, &candidate) != nil { + continue + } + if arrayField != "" { + return nil, false + } + arrayField = key + items = candidate + } + if arrayField == "" { + return nil, false + } + build := func(subset []json.RawMessage) any { + out := make(map[string]any, len(obj)+6) + for key, raw := range obj { + if key == arrayField { + out[key] = subset + continue + } + out[key] = raw + } + if len(subset) < len(items) { + out["_pp_truncated"] = true + out["_pp_total_count"] = len(items) + out["_pp_returned_count"] = len(subset) + out["_pp_original_bytes"] = len(data) + out["_pp_max_bytes"] = mcpToolResultMaxBytes + out["_pp_note"] = "Typed MCP endpoint response exceeded the tool result budget. Narrow the request with limit, offset, filters, search/sql, or a command-mirror tool with --agent/--compact/--select." + } + return out + } + out := mcpFitJSONItems(items, build) + if len(out) > mcpToolResultMaxBytes { + return nil, false + } + return out, true +} + +func mcpBoundedListEnvelope(field string, items []json.RawMessage, originalBytes int) []byte { + build := func(subset []json.RawMessage) any { + out := map[string]any{ + "count": len(items), + field: subset, + } + if len(subset) < len(items) { + out["truncated"] = true + out["returned_count"] = len(subset) + out["original_bytes"] = originalBytes + out["max_bytes"] = mcpToolResultMaxBytes + out["note"] = "Typed MCP endpoint response exceeded the tool result budget. Narrow the request with limit, offset, filters, search/sql, or a command-mirror tool with --agent/--compact/--select." + } + return out + } + return mcpFitJSONItems(items, build) +} + +func mcpFitJSONItems(items []json.RawMessage, build func([]json.RawMessage) any) []byte { + limit := len(items) + if limit > mcpToolResultMaxItems { + limit = mcpToolResultMaxItems + } + for n := limit; n >= 0; n-- { + out, err := json.Marshal(build(items[:n])) + if err != nil { + continue + } + if len(out) <= mcpToolResultMaxBytes || n == 0 { + return out + } + } + out, _ := json.Marshal(build(items[:0])) + return out +} + +func mcpOversizedPreviewEnvelope(data json.RawMessage) []byte { + previewBytes := data + if len(previewBytes) > 4000 { + previewBytes = previewBytes[:4000] + } + out, _ := json.Marshal(map[string]any{ + "truncated": true, + "original_bytes": len(data), + "max_bytes": mcpToolResultMaxBytes, + "preview": string(previewBytes), + "note": "Typed MCP endpoint response exceeded the tool result budget and was not a recognized list envelope. Narrow the request with filters, search/sql, or a command-mirror tool with --agent/--compact/--select.", + }) + return out +} + +func newMCPClient() (*client.Client, error) { + home, _ := os.UserHomeDir() + cfgPath := filepath.Join(home, ".config", "devonthink-pp-cli", "config.toml") + cfg, err := config.Load(cfgPath) + if err != nil { + return nil, fmt.Errorf("loading config: %w", err) + } + c := client.New(cfg, 60*time.Second, defaultMCPRateLimit) + // Agents calling through MCP need fresh data every call. The on-disk + // response cache survives across MCP server invocations, so a + // DELETE/PATCH followed by a GET would otherwise return the + // pre-mutation snapshot for up to the cache TTL. The interactive CLI + // constructs its own client and is unaffected. + c.NoCache = true + return c, nil +} + +func dbPath() string { + home, _ := os.UserHomeDir() + return filepath.Join(home, ".local", "share", "devonthink-pp-cli", "data.db") +} + +// Note: MCP tools use their own dbPath() because they are in a separate package (main, not cli). +// The CLI's defaultDBPath() in the cli package uses the same canonical path. + +func handleSearch(ctx context.Context, req mcplib.CallToolRequest) (*mcplib.CallToolResult, error) { + args := req.GetArguments() + query, ok := args["query"].(string) + if !ok || query == "" { + return mcplib.NewToolResultError("query is required"), nil + } + + limit := 25 + if v, ok := args["limit"].(float64); ok && v > 0 { + limit = int(v) + } + + db, err := store.OpenReadOnly(dbPath()) + if err != nil { + return mcplib.NewToolResultError(fmt.Sprintf("opening database: %v", err)), nil + } + defer db.Close() + + results, err := db.Search(query, limit) + if err != nil { + return mcplib.NewToolResultError(fmt.Sprintf("search failed: %v", err)), nil + } + + return toolResultJSON(results) +} + +// validateReadOnlyQuery gates the MCP sql tool. The agent contract advertised +// to the host is ReadOnlyHintAnnotation(true); a false annotation on a +// mutating tool lets MCP hosts auto-approve writes and is treated as a real +// bug per the project's agent-native security model. +// +// The gate rejects multi-statement input, then applies an allowlist (SELECT or +// WITH only) AFTER stripping the leading whitespace, line comments, block +// comments, and semicolons that SQLite itself ignores before parsing. A naive +// HasPrefix check on a keyword blocklist is bypassable by prefixing the +// dangerous statement with "/* x */" or "-- x\n"; a naive leading-keyword +// allowlist is bypassable by appending "; ATTACH DATABASE ...". Combined with +// the empirical fact that modernc.org/sqlite's mode=ro does NOT block VACUUM +// INTO (writes a snapshot to a new file) or ATTACH DATABASE (opens a separate +// writable handle), either bypass produces silent exfiltration to an +// attacker-chosen path. +// +// SELECT and WITH are the only allowed leading keywords. WITH supports +// SELECT-form CTEs; CTE-wrapped writes ("WITH x AS (...) INSERT ...") are +// caught by OpenReadOnly's mode=ro one layer down. PRAGMA, ATTACH, VACUUM, +// and every other DDL/DML keyword fail at this gate before reaching SQLite. +func validateReadOnlyQuery(query string) error { + stripped := stripLeadingSQLNoise(query) + if hasTrailingSQLStatement(stripped) { + return fmt.Errorf("only a single SELECT or WITH statement is allowed") + } + upper := strings.ToUpper(stripped) + if !strings.HasPrefix(upper, "SELECT") && !strings.HasPrefix(upper, "WITH") { + return fmt.Errorf("only SELECT queries are allowed") + } + return nil +} + +// stripLeadingSQLNoise removes leading whitespace, SQL line comments +// (-- to end of line), block comments (/* ... */), and statement +// separators (;) from query. SQLite skips these before parsing the first +// keyword, so a security gate that does not strip them mismatches what the +// driver actually executes. +func stripLeadingSQLNoise(query string) string { + for { + query = strings.TrimLeft(query, " \t\r\n;") + switch { + case strings.HasPrefix(query, "--"): + if idx := strings.IndexByte(query, '\n'); idx >= 0 { + query = query[idx+1:] + continue + } + return "" + case strings.HasPrefix(query, "/*"): + if idx := strings.Index(query[2:], "*/"); idx >= 0 { + query = query[2+idx+2:] + continue + } + return "" + default: + return query + } + } +} + +// hasTrailingSQLStatement reports whether query contains a statement +// terminator followed by more executable SQL. A trailing semicolon is allowed; +// a second statement is not. Semicolons inside string literals, quoted +// identifiers, bracket identifiers, and comments are ignored to match SQLite's +// parser shape closely enough for this security gate. +func hasTrailingSQLStatement(query string) bool { + inSingle := false + inDouble := false + inBacktick := false + inBracket := false + inLineComment := false + inBlockComment := false + + for i := 0; i < len(query); i++ { + ch := query[i] + next := byte(0) + if i+1 < len(query) { + next = query[i+1] + } + + switch { + case inLineComment: + if ch == '\n' { + inLineComment = false + } + continue + case inBlockComment: + if ch == '*' && next == '/' { + inBlockComment = false + i++ + } + continue + case inSingle: + if ch == '\'' { + if next == '\'' { + i++ + continue + } + inSingle = false + } + continue + case inDouble: + if ch == '"' { + if next == '"' { + i++ + continue + } + inDouble = false + } + continue + case inBacktick: + if ch == '`' { + if next == '`' { + i++ + continue + } + inBacktick = false + } + continue + case inBracket: + if ch == ']' { + inBracket = false + } + continue + } + + switch { + case ch == '-' && next == '-': + inLineComment = true + i++ + case ch == '/' && next == '*': + inBlockComment = true + i++ + case ch == '\'': + inSingle = true + case ch == '"': + inDouble = true + case ch == '`': + inBacktick = true + case ch == '[': + inBracket = true + case ch == ';': + if stripLeadingSQLNoise(query[i+1:]) != "" { + return true + } + return false + } + } + return false +} + +func handleSQL(ctx context.Context, req mcplib.CallToolRequest) (*mcplib.CallToolResult, error) { + args := req.GetArguments() + query, ok := args["query"].(string) + if !ok || query == "" { + return mcplib.NewToolResultError("query is required"), nil + } + + if err := validateReadOnlyQuery(query); err != nil { + return mcplib.NewToolResultError(err.Error()), nil + } + + db, err := store.OpenReadOnly(dbPath()) + if err != nil { + return mcplib.NewToolResultError(fmt.Sprintf("opening database: %v", err)), nil + } + defer db.Close() + + rows, err := db.Query(query) + if err != nil { + return mcplib.NewToolResultError(fmt.Sprintf("query failed: %v", err)), nil + } + defer rows.Close() + + cols, err := rows.Columns() + if err != nil { + return mcplib.NewToolResultError(fmt.Sprintf("reading columns: %v", err)), nil + } + var results []map[string]any + for rows.Next() { + values := make([]any, len(cols)) + ptrs := make([]any, len(cols)) + for i := range values { + ptrs[i] = &values[i] + } + if err := rows.Scan(ptrs...); err != nil { + return mcplib.NewToolResultError(fmt.Sprintf("scanning row: %v", err)), nil + } + row := make(map[string]any) + for i, col := range cols { + row[col] = values[i] + } + results = append(results, row) + } + // rows.Next() stops on a mid-iteration error without failing the loop, so + // skipping rows.Err() would return a truncated result set as success. + if err := rows.Err(); err != nil { + return mcplib.NewToolResultError(fmt.Sprintf("reading rows: %v", err)), nil + } + + return toolResultJSON(results) +} + +// toolResultJSON renders v as the indented JSON body of an MCP text result, +// surfacing a marshal failure as a tool error instead of empty content. +func toolResultJSON(v any) (*mcplib.CallToolResult, error) { + data, err := json.MarshalIndent(v, "", " ") + if err != nil { + return mcplib.NewToolResultError(fmt.Sprintf("encoding result: %v", err)), nil + } + return mcplib.NewToolResultText(string(data)), nil +} + +func handleContext(_ context.Context, _ mcplib.CallToolRequest) (*mcplib.CallToolResult, error) { + ctx := map[string]any{ + "api": "devonthink", + "description": "Local-first DEVONthink automation with safer shell workflows than raw AppleScript or MCP alone.", + "archetype": "content", + "tool_count": 37, + // tool_surface tells agents which surface a capability lives on. + "tool_surface": "MCP exposes typed endpoint tools plus a runtime mirror of user-facing CLI commands. Endpoint tools keep typed schemas; command-mirror tools shell out to the companion devonthink-pp-cli binary.", + "resources": []map[string]any{ + { + "name": "ai", + "description": "DEVONthink AI and summary helpers", + "endpoints": []string{"ask", "summarize"}, + "searchable": true, + }, + { + "name": "batch", + "description": "Dry-run-first multi-record mutation plans", + "endpoints": []string{"apply", "plan"}, + "searchable": true, + }, + { + "name": "context", + "description": "Agent context bundles", + "endpoints": []string{"pack"}, + "searchable": true, + }, + { + "name": "databases", + "description": "Open DEVONthink databases", + "endpoints": []string{"list"}, + "syncable": true, + }, + { + "name": "graph", + "description": "Links, mentions, and knowledge graph health", + "endpoints": []string{"audit", "links"}, + "searchable": true, + }, + { + "name": "groups", + "description": "DEVONthink groups and folders", + "endpoints": []string{"tree"}, + "searchable": true, + }, + { + "name": "ingest", + "description": "File and URL ingestion", + "endpoints": []string{"file", "url"}, + "searchable": true, + }, + { + "name": "inventory", + "description": "Stable inventory export contracts", + "endpoints": []string{"export"}, + "searchable": true, + }, + { + "name": "ledger", + "description": "Local operation ledger", + "endpoints": []string{"list", "show"}, + "syncable": true, + "searchable": true, + }, + { + "name": "mcp", + "description": "Optional local official MCP passthrough", + "endpoints": []string{"call", "schema", "tools"}, + "syncable": true, + "searchable": true, + }, + { + "name": "media", + "description": "OCR and transcription", + "endpoints": []string{"ocr", "transcribe"}, + "searchable": true, + }, + { + "name": "mirror", + "description": "Local SQLite mirror", + "endpoints": []string{"search", "sync"}, + "syncable": true, + "searchable": true, + }, + { + "name": "privacy", + "description": "Local privacy and exposure reports", + "endpoints": []string{"audit"}, + "searchable": true, + }, + { + "name": "records", + "description": "DEVONthink records", + "endpoints": []string{"content", "create", "get", "highlights", "lookup", "move", "related", "search", "update", "versions"}, + "syncable": true, + "searchable": true, + }, + { + "name": "runtime", + "description": "Local DEVONthink runtime health", + "endpoints": []string{"doctor"}, + }, + { + "name": "selection", + "description": "Current DEVONthink GUI selection", + "endpoints": []string{"get", "snapshot"}, + "syncable": true, + "searchable": true, + }, + { + "name": "sheets", + "description": "DEVONthink sheets", + "endpoints": []string{"get"}, + "searchable": true, + }, + { + "name": "tags", + "description": "Tag taxonomy and hygiene", + "endpoints": []string{"analyze"}, + "searchable": true, + }, + }, + "query_tips": []string{ + "Pagination uses cursor-based paging. Pass after parameter for subsequent pages.", + "Control page size with the limit parameter (default 100).", + "Use since for incremental fetches (filter by modification time).", + "Use the sql tool for ad-hoc analysis on synced data. Run sync first to populate the local database.", + "Use the search tool for full-text search across all synced resources. Faster than iterating list endpoints.", + "Prefer sql/search over repeated API calls when the data is already synced.", + }, + // Command-mirror capabilities are exposed through MCP by shelling out + // to the companion CLI binary. + "command_mirror_capabilities": []map[string]string{ + {"name": "Local context packs", "command": "context pack", "description": "Build a compact evidence packet from records, selections, highlights, links, and related items.", "rationale": "Requires joining live DEVONthink state with a local mirror and explicit token budgets.", "via": "mcp-command-mirror"}, + {"name": "Privacy budget report", "command": "privacy audit", "description": "Preview what a workflow may expose before content leaves the local machine.", "rationale": "Combines database scope, exclusion settings, record size estimates, and planned output shape.", "via": "mcp-command-mirror"}, + {"name": "Batch mutation plans", "command": "batch plan", "description": "Stage multi-record edits as validated dry-run plans before applying them.", "rationale": "Requires a CLI-owned operation ledger and target proof across DEVONthink records.", "via": "mcp-command-mirror"}, + {"name": "Maintenance inventory export", "command": "inventory export", "description": "Export DEVONthink databases, groups, tags, and document metadata for maintenance plugins.", "rationale": "Keeps exporter mechanics in the general CLI while filing policy stays in downstream workflows.", "via": "mcp-command-mirror"}, + {"name": "Knowledge graph audit", "command": "graph audit", "description": "Detect orphans, broken links, unresolved wiki links, weak hubs, and tag-only clusters.", "rationale": "Requires merging DEVONthink item links, wiki links, mentions, tags, parents, and local mirror edges.", "via": "mcp-command-mirror"}, + {"name": "Mirror-backed SQL and FTS", "command": "mirror search", "description": "Query a local SQLite mirror for repeatable fast analysis without repeated app calls.", "rationale": "Requires a CLI-owned local mirror synced from open DEVONthink databases.", "via": "mcp-command-mirror"}, + {"name": "MCP-to-CLI bridge", "command": "mcp call", "description": "Call DEVONthink's official local MCP tools from scripts when the local MCP server is enabled.", "rationale": "Requires a shell-facing MCP client plus local-only transport controls.", "via": "mcp-command-mirror"}, + {"name": "Operation ledger", "command": "ledger list", "description": "Review CLI-driven mutation plans, applies, target proofs, and rollback hints.", "rationale": "Requires durable local evidence for what the CLI changed in DEVONthink.", "via": "mcp-command-mirror"}, + {"name": "Selection workflow capture", "command": "selection snapshot", "description": "Turn the current GUI selection into a reusable JSON workflow seed.", "rationale": "Requires bridging DEVONthink's visual selection into a repeatable terminal artifact.", "via": "mcp-command-mirror"}, + {"name": "Local-only agent mode", "command": "agent-context", "description": "Emit an agent contract that enforces local-machine and own-LAN DEVONthink access only.", "rationale": "Requires the CLI to encode privacy and transport boundaries as machine-readable runtime guidance.", "via": "mcp-command-mirror"}, + }, + "playbook": []map[string]string{ + {"topic": "Local context packs", "insight": "Requires joining live DEVONthink state with a local mirror and explicit token budgets."}, + {"topic": "Privacy budget report", "insight": "Combines database scope, exclusion settings, record size estimates, and planned output shape."}, + {"topic": "Batch mutation plans", "insight": "Requires a CLI-owned operation ledger and target proof across DEVONthink records."}, + {"topic": "Maintenance inventory export", "insight": "Keeps exporter mechanics in the general CLI while filing policy stays in downstream workflows."}, + {"topic": "Knowledge graph audit", "insight": "Requires merging DEVONthink item links, wiki links, mentions, tags, parents, and local mirror edges."}, + {"topic": "Mirror-backed SQL and FTS", "insight": "Requires a CLI-owned local mirror synced from open DEVONthink databases."}, + {"topic": "MCP-to-CLI bridge", "insight": "Requires a shell-facing MCP client plus local-only transport controls."}, + {"topic": "Operation ledger", "insight": "Requires durable local evidence for what the CLI changed in DEVONthink."}, + {"topic": "Selection workflow capture", "insight": "Requires bridging DEVONthink's visual selection into a repeatable terminal artifact."}, + {"topic": "Local-only agent mode", "insight": "Requires the CLI to encode privacy and transport boundaries as machine-readable runtime guidance."}, + }, + } + return toolResultJSON(ctx) +} + +// RegisterNovelFeatureTools is kept as a compatibility no-op for older MCP +// mains. New generated mains call RegisterTools only; RegisterTools now +// includes the runtime Cobra-tree mirror. +func RegisterNovelFeatureTools(s *server.MCPServer) { + _ = s +} diff --git a/library/productivity/devonthink/internal/mcp/tools_test.go b/library/productivity/devonthink/internal/mcp/tools_test.go new file mode 100644 index 0000000000..8dac5d2613 --- /dev/null +++ b/library/productivity/devonthink/internal/mcp/tools_test.go @@ -0,0 +1,320 @@ +// Copyright 2026 rowdy and contributors. Licensed under Apache-2.0. See LICENSE. +// Generated by CLI Printing Press (https://github.com/mvanhorn/cli-printing-press). DO NOT EDIT. + +package mcp + +import ( + "encoding/json" + "strings" + "testing" + + mcplib "github.com/mark3labs/mcp-go/mcp" +) + +// TestValidateReadOnlyQuery_AllowsSelectAndWITH pins the contract: the MCP +// sql tool's allowlist accepts SELECT and WITH-prefix queries, including +// CTEs, mixed case, leading whitespace, leading SQL comments, and leading +// statement separators. SELECT-form CTEs ("WITH x AS (SELECT ...) SELECT") +// must work because novel CLI sql commands in the public library accept +// them as legitimate read-only queries; the MCP surface keeps parity. +func TestValidateReadOnlyQuery_AllowsSelectAndWITH(t *testing.T) { + allowed := []string{ + "SELECT 1", + "select * from resources", + " SELECT 1", + "\tSELECT 1", + "\nSELECT 1", + ";SELECT 1", + "-- comment\nSELECT 1", + "/* comment */ SELECT 1", + "/* comment */SELECT 1", + "/**/SELECT 1", + "-- one\n-- two\nSELECT 1", + "/* a *//* b */ SELECT 1", + "WITH r AS (SELECT 1) SELECT * FROM r", + "with r as (select 1) select * from r", + "SELECT 1;", + "SELECT 1; -- trailing comment", + "SELECT 1; /* trailing comment */", + "SELECT * FROM resources WHERE id = 'a;b'", + `SELECT * FROM "semi;colon"`, + "SELECT * FROM `semi;colon`", + "SELECT * FROM [semi;colon]", + } + for _, q := range allowed { + if err := validateReadOnlyQuery(q); err != nil { + t.Errorf("validateReadOnlyQuery(%q) = %v, want nil", q, err) + } + } +} + +// TestValidateReadOnlyQuery_RejectsBypassVectors covers the comment-prefix +// bypass class that defeated the earlier prefix-blocklist gate. mode=ro on +// modernc.org/sqlite does not block VACUUM INTO (writes a fresh file) or +// ATTACH DATABASE (opens a separate writable handle), so the gate is the +// only defense against those vectors. A successful bypass at this layer +// would let an MCP-trusting agent silently exfiltrate the local database. +func TestValidateReadOnlyQuery_RejectsBypassVectors(t *testing.T) { + rejected := []string{ + "VACUUM INTO '/tmp/x.db'", + "ATTACH DATABASE 'file:/tmp/x.db?mode=rwc' AS evil", + "INSERT INTO resources VALUES ('x', 'y', '{}')", + "UPDATE resources SET resource_type = 'evil'", + "DELETE FROM resources", + "REPLACE INTO resources VALUES ('seed', 'evil', '{}')", + "DROP TABLE resources", + "PRAGMA writable_schema = ON", + "REINDEX", + "DETACH DATABASE x", + "/* x */ VACUUM INTO '/tmp/exfil.db'", + "/* x */VACUUM INTO '/tmp/exfil.db'", + "-- x\nVACUUM INTO '/tmp/exfil.db'", + "/**/VACUUM INTO '/tmp/exfil.db'", + "/* x */ ATTACH DATABASE 'file:/tmp/x.db?mode=rwc' AS evil", + "-- x\nATTACH DATABASE '/tmp/x.db' AS evil", + ";VACUUM INTO '/tmp/x.db'", + "; ; VACUUM INTO '/tmp/x.db'", + "SELECT 1; ATTACH DATABASE 'file:/tmp/x?mode=rwc' AS evil; CREATE TABLE evil.t(x)", + "SELECT 1; DROP TABLE resources", + "SELECT 1; VACUUM INTO '/tmp/x.db'", + "WITH r AS (SELECT 1) SELECT * FROM r; ATTACH DATABASE '/tmp/x.db' AS evil", + "SELECT 1 /* c */ ; ATTACH DATABASE '/tmp/x.db' AS evil", + "SELECT 'a;b'; VACUUM INTO '/tmp/x.db'", + `SELECT * FROM "semi;colon"; ATTACH DATABASE '/tmp/x.db' AS evil`, + "/* a */ /* b */ INSERT INTO t VALUES (1)", + "/* outer /* not nested */ */ SELECT 1", // SQLite doesn't nest, so trailing "*/" closes; second SELECT remains. Reject — the gate must err on the side of caution when the leading shape is suspicious. + "-- only a comment", + "/* only a comment */", + "", + " ", + ";", + } + for _, q := range rejected { + if err := validateReadOnlyQuery(q); err == nil { + t.Errorf("validateReadOnlyQuery(%q) = nil, want error", q) + } + } +} + +func TestHasTrailingSQLStatement(t *testing.T) { + cases := []struct { + name string + query string + want bool + }{ + {name: "single select", query: "SELECT 1", want: false}, + {name: "trailing terminator", query: "SELECT 1;", want: false}, + {name: "trailing terminator and comment", query: "SELECT 1; -- ok", want: false}, + {name: "two statements", query: "SELECT 1; SELECT 2", want: true}, + {name: "attach after select", query: "SELECT 1; ATTACH DATABASE '/tmp/x.db' AS evil", want: true}, + {name: "semicolon in single quote", query: "SELECT 'a;b'", want: false}, + {name: "escaped single quote", query: "SELECT 'a'';b'", want: false}, + {name: "semicolon in double quote", query: `SELECT * FROM "a;b"`, want: false}, + {name: "escaped double quote", query: `SELECT * FROM "a"";b"`, want: false}, + {name: "semicolon in backtick", query: "SELECT * FROM `a;b`", want: false}, + {name: "escaped backtick", query: "SELECT * FROM `a``;b`", want: false}, + {name: "semicolon in bracket", query: "SELECT * FROM [a;b]", want: false}, + {name: "semicolon in line comment", query: "SELECT 1 -- ;\n", want: false}, + {name: "statement after line comment", query: "SELECT 1 -- ;\n; SELECT 2", want: true}, + {name: "semicolon in block comment", query: "SELECT 1 /* ; */", want: false}, + {name: "statement after block comment", query: "SELECT 1 /* ; */; SELECT 2", want: true}, + } + for _, c := range cases { + if got := hasTrailingSQLStatement(c.query); got != c.want { + t.Errorf("hasTrailingSQLStatement(%q) [%s] = %v, want %v", c.query, c.name, got, c.want) + } + } +} + +// TestStripLeadingSQLNoise checks the helper directly so a regression in the +// stripping logic (off-by-one on /* */ length, missing newline handling on +// --) surfaces close to the source rather than only via the integration +// behavior of validateReadOnlyQuery. +func TestStripLeadingSQLNoise(t *testing.T) { + cases := []struct { + in, want string + }{ + {"SELECT 1", "SELECT 1"}, + {" SELECT 1", "SELECT 1"}, + {"\t\nSELECT 1", "SELECT 1"}, + {";SELECT 1", "SELECT 1"}, + {";; ;SELECT 1", "SELECT 1"}, + {"-- x\nSELECT 1", "SELECT 1"}, + {"-- x\n-- y\nSELECT 1", "SELECT 1"}, + {"/* x */SELECT 1", "SELECT 1"}, + {"/**/SELECT 1", "SELECT 1"}, + {"/* x */ /* y */ SELECT 1", "SELECT 1"}, + {"-- only", ""}, + {"/* only", ""}, + {"", ""}, + } + for _, c := range cases { + got := stripLeadingSQLNoise(c.in) + if !strings.EqualFold(got, c.want) { + t.Errorf("stripLeadingSQLNoise(%q) = %q, want %q", c.in, got, c.want) + } + } +} + +func TestMCPToolResultTextBoundsListResponses(t *testing.T) { + items := make([]string, 0, mcpToolResultMaxItems+25) + for i := 0; i < mcpToolResultMaxItems+25; i++ { + items = append(items, strings.Repeat("x", 1600)) + } + data, err := json.Marshal(items) + if err != nil { + t.Fatalf("marshal fixture: %v", err) + } + + text := mcpTextContent(t, mcpToolResultText("GET", data)) + if len(text) > mcpToolResultMaxBytes { + t.Fatalf("bounded result length = %d, want <= %d", len(text), mcpToolResultMaxBytes) + } + + var envelope struct { + Count int `json:"count"` + Items []json.RawMessage `json:"items"` + Truncated bool `json:"truncated"` + ReturnedCount int `json:"returned_count"` + OriginalBytes int `json:"original_bytes"` + } + if err := json.Unmarshal([]byte(text), &envelope); err != nil { + t.Fatalf("bounded list result must remain valid JSON: %v\n%s", err, text) + } + if !envelope.Truncated { + t.Fatalf("bounded list result did not mark truncation: %s", text) + } + if envelope.Count != len(items) { + t.Fatalf("count = %d, want %d", envelope.Count, len(items)) + } + if envelope.ReturnedCount != len(envelope.Items) { + t.Fatalf("returned_count = %d, want item count %d", envelope.ReturnedCount, len(envelope.Items)) + } + if envelope.OriginalBytes != len(data) { + t.Fatalf("original_bytes = %d, want %d", envelope.OriginalBytes, len(data)) + } +} + +func TestMCPToolResultTextBoundsSingleArrayEnvelope(t *testing.T) { + groups := make([]map[string]string, 0, mcpToolResultMaxItems+25) + for i := 0; i < mcpToolResultMaxItems+25; i++ { + groups = append(groups, map[string]string{ + "id": strings.Repeat("g", 8), + "name": strings.Repeat("verbose group name ", 90), + }) + } + data, err := json.Marshal(map[string]any{ + "groups": groups, + "cursor": "next-page", + }) + if err != nil { + t.Fatalf("marshal fixture: %v", err) + } + + text := mcpTextContent(t, mcpToolResultText("GET", data)) + if len(text) > mcpToolResultMaxBytes { + t.Fatalf("bounded result length = %d, want <= %d", len(text), mcpToolResultMaxBytes) + } + + var envelope struct { + Groups []json.RawMessage `json:"groups"` + Cursor string `json:"cursor"` + Truncated bool `json:"_pp_truncated"` + TotalCount int `json:"_pp_total_count"` + ReturnedCount int `json:"_pp_returned_count"` + } + if err := json.Unmarshal([]byte(text), &envelope); err != nil { + t.Fatalf("bounded envelope result must remain valid JSON: %v\n%s", err, text) + } + if !envelope.Truncated { + t.Fatalf("bounded envelope result did not mark truncation: %s", text) + } + if envelope.Cursor != "next-page" { + t.Fatalf("cursor = %q, want preserved metadata", envelope.Cursor) + } + if envelope.TotalCount != len(groups) { + t.Fatalf("total_count = %d, want %d", envelope.TotalCount, len(groups)) + } + if envelope.ReturnedCount != len(envelope.Groups) { + t.Fatalf("returned_count = %d, want group count %d", envelope.ReturnedCount, len(envelope.Groups)) + } +} + +func TestMCPToolResultTextFallsBackToOversizedPreview(t *testing.T) { + data, err := json.Marshal(map[string]string{ + "blob": strings.Repeat("z", mcpToolResultMaxBytes+10000), + }) + if err != nil { + t.Fatalf("marshal fixture: %v", err) + } + + text := mcpTextContent(t, mcpToolResultText("GET", data)) + if len(text) > mcpToolResultMaxBytes { + t.Fatalf("preview result length = %d, want <= %d", len(text), mcpToolResultMaxBytes) + } + + var envelope struct { + Truncated bool `json:"truncated"` + OriginalBytes int `json:"original_bytes"` + Preview string `json:"preview"` + } + if err := json.Unmarshal([]byte(text), &envelope); err != nil { + t.Fatalf("preview result must remain valid JSON: %v\n%s", err, text) + } + if !envelope.Truncated { + t.Fatalf("preview result did not mark truncation: %s", text) + } + if envelope.OriginalBytes != len(data) { + t.Fatalf("original_bytes = %d, want %d", envelope.OriginalBytes, len(data)) + } + if envelope.Preview == "" { + t.Fatalf("preview result should include a bounded preview") + } +} + +func TestMCPToolResultTextBoundsOversizedNonGETResponses(t *testing.T) { + data, err := json.Marshal(map[string]string{ + "blob": strings.Repeat("z", mcpToolResultMaxBytes+10000), + }) + if err != nil { + t.Fatalf("marshal fixture: %v", err) + } + + text := mcpTextContent(t, mcpToolResultText("POST", data)) + if len(text) > mcpToolResultMaxBytes { + t.Fatalf("preview result length = %d, want <= %d", len(text), mcpToolResultMaxBytes) + } + + var envelope struct { + Truncated bool `json:"truncated"` + OriginalBytes int `json:"original_bytes"` + Preview string `json:"preview"` + } + if err := json.Unmarshal([]byte(text), &envelope); err != nil { + t.Fatalf("preview result must remain valid JSON: %v\n%s", err, text) + } + if !envelope.Truncated { + t.Fatalf("preview result did not mark truncation: %s", text) + } + if envelope.OriginalBytes != len(data) { + t.Fatalf("original_bytes = %d, want %d", envelope.OriginalBytes, len(data)) + } + if envelope.Preview == "" { + t.Fatalf("preview result should include a bounded preview") + } +} + +func mcpTextContent(t *testing.T, result *mcplib.CallToolResult) string { + t.Helper() + if result == nil { + t.Fatalf("result is nil") + } + if len(result.Content) != 1 { + t.Fatalf("result content length = %d, want 1", len(result.Content)) + } + content, ok := result.Content[0].(mcplib.TextContent) + if !ok { + t.Fatalf("result content type = %T, want mcp.TextContent", result.Content[0]) + } + return content.Text +} diff --git a/library/productivity/devonthink/internal/store/extras.go b/library/productivity/devonthink/internal/store/extras.go new file mode 100644 index 0000000000..97b8b04178 --- /dev/null +++ b/library/productivity/devonthink/internal/store/extras.go @@ -0,0 +1,28 @@ +// Copyright 2026 rowdy and contributors. Licensed under Apache-2.0. See LICENSE. + +package store + +import ( + "context" + "database/sql" + "fmt" +) + +// migrateExtras runs after the generated store migrations and before the +// schema-version stamp. It is the canonical place for novel-feature auxiliary +// tables that need to live in the local store. +// +// Edit this file when adding tables for novel commands. Keep migrations +// idempotent with CREATE TABLE IF NOT EXISTS / CREATE INDEX IF NOT EXISTS so +// every store open can safely re-run them. +func (s *Store) migrateExtras(ctx context.Context, conn *sql.Conn) error { + migrations := []string{ + // Add CREATE TABLE IF NOT EXISTS statements here. + } + for _, m := range migrations { + if _, err := conn.ExecContext(ctx, m); err != nil { + return fmt.Errorf("extra migration failed: %w", err) + } + } + return nil +} diff --git a/library/productivity/devonthink/internal/store/schema_version_test.go b/library/productivity/devonthink/internal/store/schema_version_test.go new file mode 100644 index 0000000000..87f6b2830d --- /dev/null +++ b/library/productivity/devonthink/internal/store/schema_version_test.go @@ -0,0 +1,1760 @@ +// Copyright 2026 rowdy and contributors. Licensed under Apache-2.0. See LICENSE. +// Generated by CLI Printing Press (https://github.com/mvanhorn/cli-printing-press). DO NOT EDIT. + +package store + +import ( + "context" + "database/sql" + "errors" + "fmt" + "path/filepath" + "strings" + "sync" + "testing" + "time" + + _ "modernc.org/sqlite" +) + +// TestSchemaVersion_StampedOnFreshDB verifies that opening a brand-new +// database stamps the current schema version. This is the contract that +// makes StoreSchemaVersion upgrades safe: every freshly-created DB +// records the version it was built under. +func TestSchemaVersion_StampedOnFreshDB(t *testing.T) { + dbPath := filepath.Join(t.TempDir(), "data.db") + s, err := Open(dbPath) + if err != nil { + t.Fatalf("open fresh db: %v", err) + } + defer s.Close() + + v, err := s.SchemaVersion() + if err != nil { + t.Fatalf("read schema version: %v", err) + } + if v != StoreSchemaVersion { + t.Fatalf("fresh db version = %d, want %d", v, StoreSchemaVersion) + } +} + +// TestOpenAppliesPragmas pins the connection-string contract: the store +// must open in WAL journal mode with a non-zero busy_timeout so a read +// concurrent with a write waits on the lock instead of failing immediately +// with SQLITE_BUSY. It fails the instant the DSN regresses to the mattn- +// style _journal_mode=WAL form, which modernc.org/sqlite silently drops — +// see the OpenReadOnly comment for the driver-syntax detail. +func TestOpenAppliesPragmas(t *testing.T) { + dbPath := filepath.Join(t.TempDir(), "data.db") + s, err := Open(dbPath) + if err != nil { + t.Fatalf("open: %v", err) + } + defer s.Close() + + requirePragma(t, s.DB(), "journal_mode", "wal") + requirePragma(t, s.DB(), "busy_timeout", "5000") + + // The read-only handle (MCP sql/search, analytics) must see the same WAL + // file mode and carry the busy_timeout so it waits on a concurrent writer + // rather than erroring. + ro, err := OpenReadOnly(dbPath) + if err != nil { + t.Fatalf("open read-only: %v", err) + } + defer ro.Close() + + requirePragma(t, ro.DB(), "journal_mode", "wal") + requirePragma(t, ro.DB(), "busy_timeout", "5000") +} + +// requirePragma fails the test unless `PRAGMA <name>` reports want. It reads +// the value as text so one helper covers both string pragmas (journal_mode) +// and integer pragmas (busy_timeout). +func requirePragma(t *testing.T, db *sql.DB, name, want string) { + t.Helper() + var got string + if err := db.QueryRow("PRAGMA " + name).Scan(&got); err != nil { + t.Fatalf("read pragma %s: %v", name, err) + } + if got != want { + t.Fatalf("PRAGMA %s = %q, want %q", name, got, want) + } +} + +// TestOpenReadOnly_DeleteModeDBDoesNotWrite guards the read-only DSN against +// mutating the database. journal_mode is a file-level property set by the +// read-write open, not the connection — issuing PRAGMA journal_mode=WAL on a +// read-only handle to a DB still in the default delete journal mode (a pre-WAL +// database from an older binary, read before its first read-write open) fails +// with "attempt to write a readonly database". OpenReadOnly must read such a +// DB without error, so its DSN carries no journal_mode pragma. +func TestOpenReadOnly_DeleteModeDBDoesNotWrite(t *testing.T) { + dbPath := filepath.Join(t.TempDir(), "data.db") + + // Create a delete-journal-mode DB directly (no WAL conversion), mirroring + // a database written by a binary that predated the WAL DSN. + raw, err := sql.Open("sqlite", dbPath) + if err != nil { + t.Fatalf("open raw: %v", err) + } + if _, err := raw.Exec(`CREATE TABLE resources (id TEXT)`); err != nil { + t.Fatalf("seed table: %v", err) + } + raw.Close() + + ro, err := OpenReadOnly(dbPath) + if err != nil { + t.Fatalf("open read-only: %v", err) + } + defer ro.Close() + + var n int + if err := ro.DB().QueryRow(`SELECT count(*) FROM resources`).Scan(&n); err != nil { + t.Fatalf("read-only query on delete-mode DB: %v", err) + } +} + +// TestSchemaVersion_StampExistingZeroDB verifies the stamp-and-continue +// rule for existing deployed databases. A DB that predates the gate has +// user_version = 0; opening it with this binary should stamp the version +// to StoreSchemaVersion without touching any data. +func TestSchemaVersion_StampExistingZeroDB(t *testing.T) { + dbPath := filepath.Join(t.TempDir(), "data.db") + + // Pre-create the DB with user_version = 0 and no tables, simulating + // a database created by a pre-gate version of the binary before any + // migrations ran. + raw, err := sql.Open("sqlite", dbPath) + if err != nil { + t.Fatalf("open raw: %v", err) + } + if _, err := raw.Exec(`PRAGMA user_version = 0`); err != nil { + t.Fatalf("stamp zero: %v", err) + } + raw.Close() + + s, err := Open(dbPath) + if err != nil { + t.Fatalf("open pre-gate db: %v", err) + } + defer s.Close() + + v, err := s.SchemaVersion() + if err != nil { + t.Fatalf("read schema version: %v", err) + } + if v != StoreSchemaVersion { + t.Fatalf("post-stamp version = %d, want %d", v, StoreSchemaVersion) + } +} + +// TestSchemaVersion_RefusesNewerDB verifies fail-fast when the on-disk +// schema is newer than the binary supports. Without this gate, a user +// who upgrades their library but not their binary would hit silent +// "no such column" errors instead of a clear version mismatch. +func TestSchemaVersion_RefusesNewerDB(t *testing.T) { + dbPath := filepath.Join(t.TempDir(), "data.db") + + raw, err := sql.Open("sqlite", dbPath) + if err != nil { + t.Fatalf("open raw: %v", err) + } + if _, err := raw.Exec(`PRAGMA user_version = 999`); err != nil { + t.Fatalf("stamp future version: %v", err) + } + raw.Close() + + _, err = Open(dbPath) + if err == nil { + t.Fatalf("expected open to fail on newer schema, got nil") + } +} + +// TestMigrate_ConcurrentFreshDB exercises the BEGIN IMMEDIATE migration +// transaction. Without it, N goroutines opening the same fresh DB in +// parallel race per CREATE TABLE statement and trip SQLITE_BUSY despite +// the busy_timeout. With it, they serialize on the RESERVED lock +// acquired at BEGIN time and every Open succeeds. +func TestMigrate_ConcurrentFreshDB(t *testing.T) { + if testing.Short() { + t.Skip("concurrent migration test can take up to migrationLockTimeout under contention") + } + t.Parallel() + + dbPath := filepath.Join(t.TempDir(), "data.db") + + const n = 8 + errs := make(chan error, n) + var wg sync.WaitGroup + wg.Add(n) + for i := 0; i < n; i++ { + go func() { + defer wg.Done() + s, err := Open(dbPath) + if err != nil { + errs <- err + return + } + s.Close() + }() + } + wg.Wait() + close(errs) + + for err := range errs { + t.Fatalf("concurrent Open failed: %v", err) + } +} + +// holdWriteLock takes an exclusive write lock on dbPath that a peer's +// BEGIN IMMEDIATE cannot acquire until the returned cleanup runs. Used +// to construct contention scenarios in the migration tests. +func holdWriteLock(t *testing.T, dbPath string) (cleanup func()) { + t.Helper() + holder, err := sql.Open("sqlite", dbPath+"?_pragma=journal_mode(WAL)&_pragma=busy_timeout(5000)") + if err != nil { + t.Fatalf("open holder: %v", err) + } + htx, err := holder.Begin() + if err != nil { + _ = holder.Close() + t.Fatalf("begin holder tx: %v", err) + } + if _, err := htx.Exec(`CREATE TABLE IF NOT EXISTS holder_lock (id INTEGER)`); err != nil { + _ = htx.Rollback() + _ = holder.Close() + t.Fatalf("seed holder write: %v", err) + } + return func() { + _ = htx.Rollback() + _ = holder.Close() + } +} + +// TestOpenWithContext_RespectsCancellation verifies that a caller that +// cancels its context during a stalled migration sees the cancellation +// surface as the returned error within a short window, instead of +// having to wait out the full migrationLockTimeout. SIGINT in a Cobra +// command's context must interrupt store.Open, not just block on it. +func TestOpenWithContext_RespectsCancellation(t *testing.T) { + t.Parallel() + + dbPath := filepath.Join(t.TempDir(), "data.db") + defer holdWriteLock(t, dbPath)() + + // Pre-cancel the context. The migration's BEGIN IMMEDIATE will BUSY + // against the holder; the very first iteration of retryOnBusy then + // hits the ctx.Done() arm of its select and propagates ctx.Canceled. + // A blocked-then-cancel pattern using time.Sleep would prove the + // same property but cost the sleep interval on every CI run. + ctx, cancel := context.WithCancel(context.Background()) + cancel() + + start := time.Now() + _, err := OpenWithContext(ctx, dbPath) + elapsed := time.Since(start) + + if err == nil { + t.Fatalf("expected OpenWithContext to fail under contention with cancelled ctx") + } + if !errors.Is(err, context.Canceled) { + t.Fatalf("expected context.Canceled in error chain, got: %v", err) + } + // Without ctx threading this would block until migrationLockTimeout + // (default 30s). 5s is generous headroom over the actual return + // time (microseconds for a pre-cancelled ctx) without flaking CI. + if elapsed > 5*time.Second { + t.Fatalf("OpenWithContext returned after %s; pre-cancelled ctx should short-circuit immediately", elapsed) + } +} + +// TestMigrate_RejectsNewerDBImmediately verifies that an old binary +// opening a newer-schema DB rejects fast even when a peer migrator is +// still holding the write lock. The schema-version check runs on the +// pinned connection BEFORE BEGIN IMMEDIATE so the rejection path +// doesn't have to wait out the migration lock. +func TestMigrate_RejectsNewerDBImmediately(t *testing.T) { + t.Parallel() + + dbPath := filepath.Join(t.TempDir(), "data.db") + + // Pre-stamp the DB at a version this binary doesn't support. + raw, err := sql.Open("sqlite", dbPath) + if err != nil { + t.Fatalf("open raw: %v", err) + } + if _, err := raw.Exec(`PRAGMA user_version = 999`); err != nil { + t.Fatalf("stamp future version: %v", err) + } + raw.Close() + + defer holdWriteLock(t, dbPath)() + + start := time.Now() + _, err = Open(dbPath) + elapsed := time.Since(start) + + if err == nil { + t.Fatalf("expected Open to refuse a newer-schema DB") + } + // The fast-path goal: rejection must arrive well under + // migrationLockTimeout. 5s leaves headroom over the WAL init race + // (a few ms in practice) without being so tight CI flakes. + if elapsed > 5*time.Second { + t.Fatalf("Open rejected after %s; fast-path should reject in well under migrationLockTimeout (30s)", elapsed) + } +} + +// TestSchemaVersion_ReopenIsIdempotent verifies that opening an already +// correctly-stamped DB is a no-op — the second open reads the version +// and the migrations are all idempotent (CREATE TABLE IF NOT EXISTS). +func TestSchemaVersion_ReopenIsIdempotent(t *testing.T) { + dbPath := filepath.Join(t.TempDir(), "data.db") + + s1, err := Open(dbPath) + if err != nil { + t.Fatalf("first open: %v", err) + } + s1.Close() + + s2, err := Open(dbPath) + if err != nil { + t.Fatalf("second open: %v", err) + } + defer s2.Close() + + v, err := s2.SchemaVersion() + if err != nil { + t.Fatalf("read schema version: %v", err) + } + if v != StoreSchemaVersion { + t.Fatalf("reopened version = %d, want %d", v, StoreSchemaVersion) + } +} + +func TestResources_CompositeKeyPreservesOverlappingIDs(t *testing.T) { + dbPath := filepath.Join(t.TempDir(), "data.db") + s, err := Open(dbPath) + if err != nil { + t.Fatalf("open db: %v", err) + } + defer s.Close() + + if err := s.Upsert("biz", "shared", []byte(`{"kind":"biz","name":"Pinky restaurant"}`)); err != nil { + t.Fatalf("upsert biz: %v", err) + } + if err := s.Upsert("bookmark", "shared", []byte(`{"kind":"bookmark","note":"anniversary"}`)); err != nil { + t.Fatalf("upsert bookmark: %v", err) + } + + biz, err := s.Get("biz", "shared") + if err != nil { + t.Fatalf("get biz: %v", err) + } + if string(biz) != `{"kind":"biz","name":"Pinky restaurant"}` { + t.Fatalf("biz payload = %s", biz) + } + + bookmark, err := s.Get("bookmark", "shared") + if err != nil { + t.Fatalf("get bookmark: %v", err) + } + if string(bookmark) != `{"kind":"bookmark","note":"anniversary"}` { + t.Fatalf("bookmark payload = %s", bookmark) + } + + var count int + if err := s.DB().QueryRow(`SELECT COUNT(*) FROM resources WHERE id = 'shared'`).Scan(&count); err != nil { + t.Fatalf("count overlapping rows: %v", err) + } + if count != 2 { + t.Fatalf("overlapping row count = %d, want 2", count) + } + + matches, err := s.Search("restaurant", 10) + if err != nil { + t.Fatalf("search restaurant: %v", err) + } + if len(matches) != 1 || string(matches[0]) != `{"kind":"biz","name":"Pinky restaurant"}` { + t.Fatalf("restaurant search = %q, want only biz payload", matches) + } +} + +// Callers detect missing rows via errors.Is(err, sql.ErrNoRows); present +// rows return the JSON payload with a nil error. +func TestGet_MissingRowReturnsErrNoRows(t *testing.T) { + dbPath := filepath.Join(t.TempDir(), "data.db") + s, err := Open(dbPath) + if err != nil { + t.Fatalf("open: %v", err) + } + defer s.Close() + + data, err := s.Get("missing_type", "missing_id") + if !errors.Is(err, sql.ErrNoRows) { + t.Fatalf("Get missing row err = %v, want sql.ErrNoRows", err) + } + if data != nil { + t.Fatalf("Get missing row data = %s, want nil", data) + } + + if err := s.Upsert("present_type", "present_id", []byte(`{"ok":true}`)); err != nil { + t.Fatalf("upsert: %v", err) + } + got, err := s.Get("present_type", "present_id") + if err != nil { + t.Fatalf("Get present row: %v", err) + } + if string(got) != `{"ok":true}` { + t.Fatalf("Get present row data = %s, want {\"ok\":true}", got) + } +} + +func TestMigrate_ResourcesCompositeKeyUpgrade(t *testing.T) { + dbPath := filepath.Join(t.TempDir(), "data.db") + + raw, err := sql.Open("sqlite", dbPath) + if err != nil { + t.Fatalf("open raw: %v", err) + } + if _, err := raw.Exec(`CREATE TABLE resources ( + id TEXT PRIMARY KEY, + resource_type TEXT NOT NULL, + data JSON NOT NULL, + synced_at DATETIME DEFAULT CURRENT_TIMESTAMP, + updated_at DATETIME DEFAULT CURRENT_TIMESTAMP + )`); err != nil { + raw.Close() + t.Fatalf("create v1 resources: %v", err) + } + if _, err := raw.Exec(`CREATE VIRTUAL TABLE resources_fts USING fts5( + id, resource_type, content, tokenize='porter unicode61' + )`); err != nil { + raw.Close() + t.Fatalf("create v1 resources_fts: %v", err) + } + if _, err := raw.Exec(`INSERT INTO resources (id, resource_type, data) VALUES ('shared', 'biz', '{"kind":"biz","name":"legacy restaurant"}')`); err != nil { + raw.Close() + t.Fatalf("insert v1 resource: %v", err) + } + if _, err := raw.Exec(`INSERT INTO resources_fts (rowid, id, resource_type, content) VALUES (1, 'shared', 'biz', '{"kind":"biz","name":"legacy restaurant"}')`); err != nil { + raw.Close() + t.Fatalf("insert v1 fts row: %v", err) + } + if _, err := raw.Exec(`PRAGMA user_version = 1`); err != nil { + raw.Close() + t.Fatalf("stamp v1: %v", err) + } + raw.Close() + + s, err := Open(dbPath) + if err != nil { + t.Fatalf("open upgraded db: %v", err) + } + defer s.Close() + + v, err := s.SchemaVersion() + if err != nil { + t.Fatalf("read schema version: %v", err) + } + if v != StoreSchemaVersion { + t.Fatalf("upgraded version = %d, want %d", v, StoreSchemaVersion) + } + + rows, err := s.DB().Query(`PRAGMA table_info(resources)`) + if err != nil { + t.Fatalf("table_info resources: %v", err) + } + defer rows.Close() + + pk := map[string]int{} + for rows.Next() { + var cid int + var name, typ string + var notnull, pkOrder int + var dflt sql.NullString + if err := rows.Scan(&cid, &name, &typ, ¬null, &dflt, &pkOrder); err != nil { + t.Fatalf("scan table_info: %v", err) + } + pk[name] = pkOrder + } + if err := rows.Err(); err != nil { + t.Fatalf("table_info rows: %v", err) + } + if pk["resource_type"] != 1 || pk["id"] != 2 { + t.Fatalf("resources primary key order = resource_type:%d id:%d, want resource_type:1 id:2", pk["resource_type"], pk["id"]) + } + + if err := s.Upsert("bookmark", "shared", []byte(`{"kind":"bookmark","note":"after upgrade"}`)); err != nil { + t.Fatalf("upsert overlapping resource after upgrade: %v", err) + } + + biz, err := s.Get("biz", "shared") + if err != nil { + t.Fatalf("get migrated biz: %v", err) + } + if string(biz) != `{"kind":"biz","name":"legacy restaurant"}` { + t.Fatalf("migrated biz payload = %s", biz) + } + + bookmark, err := s.Get("bookmark", "shared") + if err != nil { + t.Fatalf("get upgraded bookmark: %v", err) + } + if string(bookmark) != `{"kind":"bookmark","note":"after upgrade"}` { + t.Fatalf("upgraded bookmark payload = %s", bookmark) + } + + matches, err := s.Search("legacy", 10) + if err != nil { + t.Fatalf("search migrated fts: %v", err) + } + if len(matches) != 1 || string(matches[0]) != `{"kind":"biz","name":"legacy restaurant"}` { + t.Fatalf("legacy search = %q, want migrated biz payload", matches) + } +} + +func TestMigrate_V2ResourcesFTSRowIDUpgrade(t *testing.T) { + dbPath := filepath.Join(t.TempDir(), "data.db") + + raw, err := sql.Open("sqlite", dbPath) + if err != nil { + t.Fatalf("open raw: %v", err) + } + if _, err := raw.Exec(`CREATE TABLE resources ( + id TEXT NOT NULL, + resource_type TEXT NOT NULL, + data JSON NOT NULL, + synced_at DATETIME DEFAULT CURRENT_TIMESTAMP, + updated_at DATETIME DEFAULT CURRENT_TIMESTAMP, + PRIMARY KEY (resource_type, id) + )`); err != nil { + raw.Close() + t.Fatalf("create v2 resources: %v", err) + } + if _, err := raw.Exec(`CREATE VIRTUAL TABLE resources_fts USING fts5( + id, resource_type, content, tokenize='porter unicode61' + )`); err != nil { + raw.Close() + t.Fatalf("create stale resources_fts: %v", err) + } + if _, err := raw.Exec(`INSERT INTO resources (id, resource_type, data) VALUES ('shared', 'biz', '{"kind":"biz","name":"legacy restaurant"}')`); err != nil { + raw.Close() + t.Fatalf("seed v2 resource: %v", err) + } + if _, err := raw.Exec(`INSERT INTO resources_fts (rowid, id, resource_type, content) VALUES (1, 'shared', 'biz', '{"kind":"biz","name":"legacy restaurant"}')`); err != nil { + raw.Close() + t.Fatalf("seed stale resources_fts row: %v", err) + } + if _, err := raw.Exec(`PRAGMA user_version = 2`); err != nil { + raw.Close() + t.Fatalf("stamp v2: %v", err) + } + raw.Close() + + s, err := Open(dbPath) + if err != nil { + t.Fatalf("open upgraded db: %v", err) + } + defer s.Close() + + v, err := s.SchemaVersion() + if err != nil { + t.Fatalf("read schema version: %v", err) + } + if v != StoreSchemaVersion { + t.Fatalf("upgraded version = %d, want %d", v, StoreSchemaVersion) + } + + var count int + if err := s.DB().QueryRow(`SELECT COUNT(*) FROM resources_fts WHERE id = 'shared' AND resource_type = 'biz'`).Scan(&count); err != nil { + t.Fatalf("count rebuilt resources_fts rows: %v", err) + } + if count != 1 { + t.Fatalf("resources_fts row count = %d, want 1", count) + } + + var rowid int64 + if err := s.DB().QueryRow(`SELECT rowid FROM resources_fts WHERE id = 'shared' AND resource_type = 'biz'`).Scan(&rowid); err != nil { + t.Fatalf("read rebuilt resources_fts rowid: %v", err) + } + if want := ftsRowID("biz", "shared"); rowid != want { + t.Fatalf("resources_fts rowid = %d, want %d", rowid, want) + } + + data, err := s.Get("biz", "shared") + if err != nil { + t.Fatalf("get preserved v2 resource after rowid migration: %v", err) + } + if string(data) != `{"kind":"biz","name":"legacy restaurant"}` { + t.Fatalf("preserved v2 resource payload = %s, want original", data) + } + + if err := s.Upsert("biz", "shared", []byte(`{"kind":"biz","name":"legacy cafe"}`)); err != nil { + t.Fatalf("upsert after rowid migration: %v", err) + } + matches, err := s.Search("legacy", 10) + if err != nil { + t.Fatalf("search rebuilt fts: %v", err) + } + if len(matches) != 1 || string(matches[0]) != `{"kind":"biz","name":"legacy cafe"}` { + t.Fatalf("legacy search = %q, want exactly one updated payload", matches) + } +} + +func TestMigrate_V3ResourcesFTSRebuildsSearchableContent(t *testing.T) { + dbPath := filepath.Join(t.TempDir(), "data.db") + + raw, err := sql.Open("sqlite", dbPath) + if err != nil { + t.Fatalf("open raw: %v", err) + } + if _, err := raw.Exec(`CREATE TABLE resources ( + id TEXT NOT NULL, + resource_type TEXT NOT NULL, + data JSON NOT NULL, + synced_at DATETIME DEFAULT CURRENT_TIMESTAMP, + updated_at DATETIME DEFAULT CURRENT_TIMESTAMP, + PRIMARY KEY (resource_type, id) + )`); err != nil { + raw.Close() + t.Fatalf("create v3 resources: %v", err) + } + if _, err := raw.Exec(`CREATE VIRTUAL TABLE resources_fts USING fts5( + id, resource_type, content, tokenize='porter unicode61' + )`); err != nil { + raw.Close() + t.Fatalf("create v3 resources_fts: %v", err) + } + if _, err := raw.Exec(`INSERT INTO resources (id, resource_type, data) VALUES ('shared', 'biz', '{"kind":"biz","name":"canonical resource"}')`); err != nil { + raw.Close() + t.Fatalf("seed v3 resource: %v", err) + } + if _, err := raw.Exec(`INSERT INTO resources_fts (rowid, id, resource_type, content) VALUES (?, 'shared', 'biz', '{"kind":"biz","name":"sentinel fts"}')`, ftsRowID("biz", "shared")); err != nil { + raw.Close() + t.Fatalf("seed v3 resources_fts row: %v", err) + } + if _, err := raw.Exec(`PRAGMA user_version = 3`); err != nil { + raw.Close() + t.Fatalf("stamp v3: %v", err) + } + raw.Close() + + s, err := Open(dbPath) + if err != nil { + t.Fatalf("open v3 db: %v", err) + } + defer s.Close() + + v, err := s.SchemaVersion() + if err != nil { + t.Fatalf("read schema version: %v", err) + } + if v != StoreSchemaVersion { + t.Fatalf("schema version = %d, want %d", v, StoreSchemaVersion) + } + + var content string + if err := s.DB().QueryRow(`SELECT content FROM resources_fts WHERE id = 'shared' AND resource_type = 'biz'`).Scan(&content); err != nil { + t.Fatalf("read resources_fts content: %v", err) + } + if strings.Contains(content, "sentinel") || strings.Contains(content, "name") || !strings.Contains(content, "canonical resource") { + t.Fatalf("resources_fts content = %s, want rebuilt searchable values only", content) + } +} + +func TestMigrate_ResourcesFTSContentSchemaVersionNoRebuild(t *testing.T) { + dbPath := filepath.Join(t.TempDir(), "data.db") + + raw, err := sql.Open("sqlite", dbPath) + if err != nil { + t.Fatalf("open raw: %v", err) + } + if _, err := raw.Exec(`CREATE TABLE resources ( + id TEXT NOT NULL, + resource_type TEXT NOT NULL, + data JSON NOT NULL, + synced_at DATETIME DEFAULT CURRENT_TIMESTAMP, + updated_at DATETIME DEFAULT CURRENT_TIMESTAMP, + PRIMARY KEY (resource_type, id) + )`); err != nil { + raw.Close() + t.Fatalf("create resources: %v", err) + } + if _, err := raw.Exec(resourcesFTSCreateSQL); err != nil { + raw.Close() + t.Fatalf("create resources_fts: %v", err) + } + if _, err := raw.Exec(`INSERT INTO resources (id, resource_type, data) VALUES ('shared', 'biz', '{"kind":"biz","name":"canonical resource"}')`); err != nil { + raw.Close() + t.Fatalf("seed resource: %v", err) + } + if _, err := raw.Exec(`INSERT INTO resources_fts (rowid, id, resource_type, content) VALUES (?, 'shared', 'biz', 'sentinel fts')`, ftsRowID("biz", "shared")); err != nil { + raw.Close() + t.Fatalf("seed resources_fts row: %v", err) + } + if _, err := raw.Exec(fmt.Sprintf(`PRAGMA user_version = %d`, resourcesFTSContentSchemaVersion)); err != nil { + raw.Close() + t.Fatalf("stamp resources fts content schema version: %v", err) + } + raw.Close() + + s, err := Open(dbPath) + if err != nil { + t.Fatalf("open db: %v", err) + } + defer s.Close() + + var content string + if err := s.DB().QueryRow(`SELECT content FROM resources_fts WHERE id = 'shared' AND resource_type = 'biz'`).Scan(&content); err != nil { + t.Fatalf("read resources_fts content: %v", err) + } + if content != "sentinel fts" { + t.Fatalf("resources_fts content = %s, want sentinel row preserved", content) + } +} + +// TestOpenReadOnly_RejectsWrites pins the contract: direct and CTE-wrapped +// writes against the main DB fail under mode=ro. Deliberately does not +// assert VACUUM INTO and ATTACH DATABASE — modernc.org/sqlite allows both +// under mode=ro, so those defenses live in the handleSQL keyword blocklist. +func TestOpenReadOnly_RejectsWrites(t *testing.T) { + dbPath := filepath.Join(t.TempDir(), "data.db") + + rw, err := Open(dbPath) + if err != nil { + t.Fatalf("seed open: %v", err) + } + if _, err := rw.DB().Exec(`INSERT INTO resources (id, resource_type, data) VALUES ('seed', 'thing', '{}')`); err != nil { + t.Fatalf("seed insert: %v", err) + } + rw.Close() + + ro, err := OpenReadOnly(dbPath) + if err != nil { + t.Fatalf("open read-only: %v", err) + } + defer ro.Close() + + writes := []struct { + name string + stmt string + }{ + {"insert", `INSERT INTO resources (id, resource_type, data) VALUES ('x', 'y', '{}')`}, + {"update", `UPDATE resources SET resource_type = 'hijacked' WHERE id = 'seed'`}, + {"delete", `DELETE FROM resources WHERE id = 'seed'`}, + {"replace", `REPLACE INTO resources (id, resource_type, data) VALUES ('seed', 'evil', '{}')`}, + // CTE-wrapped INSERT is load-bearing: it justifies leaving WITH + // out of the handleSQL blocklist so SELECT-form CTEs work. + {"cte_insert", `WITH stale AS (SELECT id FROM resources) INSERT INTO resources (id, resource_type, data) SELECT id || '-evil', 'thing', '{}' FROM stale`}, + } + for _, w := range writes { + if _, err := ro.DB().Exec(w.stmt); err == nil { + t.Errorf("%s succeeded under mode=ro; expected rejection. stmt=%q", w.name, w.stmt) + } + } + + var count int + if err := ro.DB().QueryRow(`SELECT COUNT(*) FROM resources`).Scan(&count); err != nil { + t.Fatalf("read-only SELECT failed: %v", err) + } + if count != 1 { + t.Fatalf("SELECT returned %d rows, want 1 (only the seed should remain)", count) + } + if err := ro.DB().QueryRow(`WITH r AS (SELECT id FROM resources WHERE id = 'seed') SELECT COUNT(*) FROM r`).Scan(&count); err != nil { + t.Fatalf("read-only WITH...SELECT CTE failed: %v", err) + } + if count != 1 { + t.Fatalf("CTE SELECT returned %d rows, want 1", count) + } +} + +// TestMigrate_AddsColumnsOnUpgrade_Ai verifies that opening a +// database created by an older binary succeeds and adds newly generated +// columns before CREATE INDEX runs against the pre-existing table. Regression +// coverage for parent_id upgrades and indexed generated columns. +func TestMigrate_AddsColumnsOnUpgrade_Ai(t *testing.T) { + dbPath := filepath.Join(t.TempDir(), "data.db") + + // Pre-create the DB with the older table shape: id, data, synced_at and + // none of the newer generated columns. user_version stays 0 (pre-gate). + raw, err := sql.Open("sqlite", dbPath) + if err != nil { + t.Fatalf("open raw: %v", err) + } + if _, err := raw.Exec(`CREATE TABLE "ai" ( + id TEXT PRIMARY KEY, + data JSON NOT NULL, + synced_at DATETIME DEFAULT CURRENT_TIMESTAMP + )`); err != nil { + raw.Close() + t.Fatalf("create old table: %v", err) + } + raw.Close() + + // Opening with the new binary must run CREATE INDEX statements without + // erroring on missing generated columns. + s, err := Open(dbPath) + if err != nil { + t.Fatalf("open upgraded db: %v", err) + } + defer s.Close() + + // The migration must have added every generated column. + rows, err := s.DB().Query(`PRAGMA table_info("ai")`) + if err != nil { + t.Fatalf("table_info: %v", err) + } + defer rows.Close() + + hasColumn := make(map[string]bool) + for rows.Next() { + var cid int + var name, typ string + var notnull, pk int + var dflt sql.NullString + if err := rows.Scan(&cid, &name, &typ, ¬null, &dflt, &pk); err != nil { + t.Fatalf("scan: %v", err) + } + hasColumn[name] = true + } + if err := rows.Err(); err != nil { + t.Fatalf("rows: %v", err) + } + + for _, want := range []string{ + "text", + "truncated", + } { + if !hasColumn[want] { + t.Fatalf("%s column missing from ai after migrate", want) + } + } +} + +// TestMigrate_AddsColumnsOnUpgrade_Batch verifies that opening a +// database created by an older binary succeeds and adds newly generated +// columns before CREATE INDEX runs against the pre-existing table. Regression +// coverage for parent_id upgrades and indexed generated columns. +func TestMigrate_AddsColumnsOnUpgrade_Batch(t *testing.T) { + dbPath := filepath.Join(t.TempDir(), "data.db") + + // Pre-create the DB with the older table shape: id, data, synced_at and + // none of the newer generated columns. user_version stays 0 (pre-gate). + raw, err := sql.Open("sqlite", dbPath) + if err != nil { + t.Fatalf("open raw: %v", err) + } + if _, err := raw.Exec(`CREATE TABLE "batch" ( + id TEXT PRIMARY KEY, + data JSON NOT NULL, + synced_at DATETIME DEFAULT CURRENT_TIMESTAMP + )`); err != nil { + raw.Close() + t.Fatalf("create old table: %v", err) + } + raw.Close() + + // Opening with the new binary must run CREATE INDEX statements without + // erroring on missing generated columns. + s, err := Open(dbPath) + if err != nil { + t.Fatalf("open upgraded db: %v", err) + } + defer s.Close() + + // The migration must have added every generated column. + rows, err := s.DB().Query(`PRAGMA table_info("batch")`) + if err != nil { + t.Fatalf("table_info: %v", err) + } + defer rows.Close() + + hasColumn := make(map[string]bool) + for rows.Next() { + var cid int + var name, typ string + var notnull, pk int + var dflt sql.NullString + if err := rows.Scan(&cid, &name, &typ, ¬null, &dflt, &pk); err != nil { + t.Fatalf("scan: %v", err) + } + hasColumn[name] = true + } + if err := rows.Err(); err != nil { + t.Fatalf("rows: %v", err) + } + + for _, want := range []string{ + "status", + "dry_run", + "message", + "ledger_id", + "plan_path", + "action_count", + } { + if !hasColumn[want] { + t.Fatalf("%s column missing from batch after migrate", want) + } + } +} + +// TestMigrate_AddsColumnsOnUpgrade_Databases verifies that opening a +// database created by an older binary succeeds and adds newly generated +// columns before CREATE INDEX runs against the pre-existing table. Regression +// coverage for parent_id upgrades and indexed generated columns. +func TestMigrate_AddsColumnsOnUpgrade_Databases(t *testing.T) { + dbPath := filepath.Join(t.TempDir(), "data.db") + + // Pre-create the DB with the older table shape: id, data, synced_at and + // none of the newer generated columns. user_version stays 0 (pre-gate). + raw, err := sql.Open("sqlite", dbPath) + if err != nil { + t.Fatalf("open raw: %v", err) + } + if _, err := raw.Exec(`CREATE TABLE "databases" ( + id TEXT PRIMARY KEY, + data JSON NOT NULL, + synced_at DATETIME DEFAULT CURRENT_TIMESTAMP + )`); err != nil { + raw.Close() + t.Fatalf("create old table: %v", err) + } + raw.Close() + + // Opening with the new binary must run CREATE INDEX statements without + // erroring on missing generated columns. + s, err := Open(dbPath) + if err != nil { + t.Fatalf("open upgraded db: %v", err) + } + defer s.Close() + + // The migration must have added every generated column. + rows, err := s.DB().Query(`PRAGMA table_info("databases")`) + if err != nil { + t.Fatalf("table_info: %v", err) + } + defer rows.Close() + + hasColumn := make(map[string]bool) + for rows.Next() { + var cid int + var name, typ string + var notnull, pk int + var dflt sql.NullString + if err := rows.Scan(&cid, &name, &typ, ¬null, &dflt, &pk); err != nil { + t.Fatalf("scan: %v", err) + } + hasColumn[name] = true + } + if err := rows.Err(); err != nil { + t.Fatalf("rows: %v", err) + } + + for _, want := range []string{ + "uuid", + "name", + "incoming_group_uuid", + "is_open", + } { + if !hasColumn[want] { + t.Fatalf("%s column missing from databases after migrate", want) + } + } +} + +// TestMigrate_AddsColumnsOnUpgrade_Graph verifies that opening a +// database created by an older binary succeeds and adds newly generated +// columns before CREATE INDEX runs against the pre-existing table. Regression +// coverage for parent_id upgrades and indexed generated columns. +func TestMigrate_AddsColumnsOnUpgrade_Graph(t *testing.T) { + dbPath := filepath.Join(t.TempDir(), "data.db") + + // Pre-create the DB with the older table shape: id, data, synced_at and + // none of the newer generated columns. user_version stays 0 (pre-gate). + raw, err := sql.Open("sqlite", dbPath) + if err != nil { + t.Fatalf("open raw: %v", err) + } + if _, err := raw.Exec(`CREATE TABLE "graph" ( + id TEXT PRIMARY KEY, + data JSON NOT NULL, + synced_at DATETIME DEFAULT CURRENT_TIMESTAMP + )`); err != nil { + raw.Close() + t.Fatalf("create old table: %v", err) + } + raw.Close() + + // Opening with the new binary must run CREATE INDEX statements without + // erroring on missing generated columns. + s, err := Open(dbPath) + if err != nil { + t.Fatalf("open upgraded db: %v", err) + } + defer s.Close() + + // The migration must have added every generated column. + rows, err := s.DB().Query(`PRAGMA table_info("graph")`) + if err != nil { + t.Fatalf("table_info: %v", err) + } + defer rows.Close() + + hasColumn := make(map[string]bool) + for rows.Next() { + var cid int + var name, typ string + var notnull, pk int + var dflt sql.NullString + if err := rows.Scan(&cid, &name, &typ, ¬null, &dflt, &pk); err != nil { + t.Fatalf("scan: %v", err) + } + hasColumn[name] = true + } + if err := rows.Err(); err != nil { + t.Fatalf("rows: %v", err) + } + + for _, want := range []string{ + "uuid", + } { + if !hasColumn[want] { + t.Fatalf("%s column missing from graph after migrate", want) + } + } +} + +// TestMigrate_AddsColumnsOnUpgrade_Groups verifies that opening a +// database created by an older binary succeeds and adds newly generated +// columns before CREATE INDEX runs against the pre-existing table. Regression +// coverage for parent_id upgrades and indexed generated columns. +func TestMigrate_AddsColumnsOnUpgrade_Groups(t *testing.T) { + dbPath := filepath.Join(t.TempDir(), "data.db") + + // Pre-create the DB with the older table shape: id, data, synced_at and + // none of the newer generated columns. user_version stays 0 (pre-gate). + raw, err := sql.Open("sqlite", dbPath) + if err != nil { + t.Fatalf("open raw: %v", err) + } + if _, err := raw.Exec(`CREATE TABLE "groups" ( + id TEXT PRIMARY KEY, + data JSON NOT NULL, + synced_at DATETIME DEFAULT CURRENT_TIMESTAMP + )`); err != nil { + raw.Close() + t.Fatalf("create old table: %v", err) + } + raw.Close() + + // Opening with the new binary must run CREATE INDEX statements without + // erroring on missing generated columns. + s, err := Open(dbPath) + if err != nil { + t.Fatalf("open upgraded db: %v", err) + } + defer s.Close() + + // The migration must have added every generated column. + rows, err := s.DB().Query(`PRAGMA table_info("groups")`) + if err != nil { + t.Fatalf("table_info: %v", err) + } + defer rows.Close() + + hasColumn := make(map[string]bool) + for rows.Next() { + var cid int + var name, typ string + var notnull, pk int + var dflt sql.NullString + if err := rows.Scan(&cid, &name, &typ, ¬null, &dflt, &pk); err != nil { + t.Fatalf("scan: %v", err) + } + hasColumn[name] = true + } + if err := rows.Err(); err != nil { + t.Fatalf("rows: %v", err) + } + + for _, want := range []string{ + "uuid", + "name", + "path", + "child_count", + } { + if !hasColumn[want] { + t.Fatalf("%s column missing from groups after migrate", want) + } + } +} + +// TestMigrate_AddsColumnsOnUpgrade_Ingest verifies that opening a +// database created by an older binary succeeds and adds newly generated +// columns before CREATE INDEX runs against the pre-existing table. Regression +// coverage for parent_id upgrades and indexed generated columns. +func TestMigrate_AddsColumnsOnUpgrade_Ingest(t *testing.T) { + dbPath := filepath.Join(t.TempDir(), "data.db") + + // Pre-create the DB with the older table shape: id, data, synced_at and + // none of the newer generated columns. user_version stays 0 (pre-gate). + raw, err := sql.Open("sqlite", dbPath) + if err != nil { + t.Fatalf("open raw: %v", err) + } + if _, err := raw.Exec(`CREATE TABLE "ingest" ( + id TEXT PRIMARY KEY, + data JSON NOT NULL, + synced_at DATETIME DEFAULT CURRENT_TIMESTAMP + )`); err != nil { + raw.Close() + t.Fatalf("create old table: %v", err) + } + raw.Close() + + // Opening with the new binary must run CREATE INDEX statements without + // erroring on missing generated columns. + s, err := Open(dbPath) + if err != nil { + t.Fatalf("open upgraded db: %v", err) + } + defer s.Close() + + // The migration must have added every generated column. + rows, err := s.DB().Query(`PRAGMA table_info("ingest")`) + if err != nil { + t.Fatalf("table_info: %v", err) + } + defer rows.Close() + + hasColumn := make(map[string]bool) + for rows.Next() { + var cid int + var name, typ string + var notnull, pk int + var dflt sql.NullString + if err := rows.Scan(&cid, &name, &typ, ¬null, &dflt, &pk); err != nil { + t.Fatalf("scan: %v", err) + } + hasColumn[name] = true + } + if err := rows.Err(); err != nil { + t.Fatalf("rows: %v", err) + } + + for _, want := range []string{ + "status", + "dry_run", + "message", + "ledger_id", + } { + if !hasColumn[want] { + t.Fatalf("%s column missing from ingest after migrate", want) + } + } +} + +// TestMigrate_AddsColumnsOnUpgrade_Ledger verifies that opening a +// database created by an older binary succeeds and adds newly generated +// columns before CREATE INDEX runs against the pre-existing table. Regression +// coverage for parent_id upgrades and indexed generated columns. +func TestMigrate_AddsColumnsOnUpgrade_Ledger(t *testing.T) { + dbPath := filepath.Join(t.TempDir(), "data.db") + + // Pre-create the DB with the older table shape: id, data, synced_at and + // none of the newer generated columns. user_version stays 0 (pre-gate). + raw, err := sql.Open("sqlite", dbPath) + if err != nil { + t.Fatalf("open raw: %v", err) + } + if _, err := raw.Exec(`CREATE TABLE "ledger" ( + id TEXT PRIMARY KEY, + data JSON NOT NULL, + synced_at DATETIME DEFAULT CURRENT_TIMESTAMP + )`); err != nil { + raw.Close() + t.Fatalf("create old table: %v", err) + } + raw.Close() + + // Opening with the new binary must run CREATE INDEX statements without + // erroring on missing generated columns. + s, err := Open(dbPath) + if err != nil { + t.Fatalf("open upgraded db: %v", err) + } + defer s.Close() + + // The migration must have added every generated column. + rows, err := s.DB().Query(`PRAGMA table_info("ledger")`) + if err != nil { + t.Fatalf("table_info: %v", err) + } + defer rows.Close() + + hasColumn := make(map[string]bool) + for rows.Next() { + var cid int + var name, typ string + var notnull, pk int + var dflt sql.NullString + if err := rows.Scan(&cid, &name, &typ, ¬null, &dflt, &pk); err != nil { + t.Fatalf("scan: %v", err) + } + hasColumn[name] = true + } + if err := rows.Err(); err != nil { + t.Fatalf("rows: %v", err) + } + + for _, want := range []string{ + "time", + "action", + "count", + "status", + } { + if !hasColumn[want] { + t.Fatalf("%s column missing from ledger after migrate", want) + } + } +} + +// TestMigrate_AddsColumnsOnUpgrade_Mcp verifies that opening a +// database created by an older binary succeeds and adds newly generated +// columns before CREATE INDEX runs against the pre-existing table. Regression +// coverage for parent_id upgrades and indexed generated columns. +func TestMigrate_AddsColumnsOnUpgrade_Mcp(t *testing.T) { + dbPath := filepath.Join(t.TempDir(), "data.db") + + // Pre-create the DB with the older table shape: id, data, synced_at and + // none of the newer generated columns. user_version stays 0 (pre-gate). + raw, err := sql.Open("sqlite", dbPath) + if err != nil { + t.Fatalf("open raw: %v", err) + } + if _, err := raw.Exec(`CREATE TABLE "mcp" ( + id TEXT PRIMARY KEY, + data JSON NOT NULL, + synced_at DATETIME DEFAULT CURRENT_TIMESTAMP + )`); err != nil { + raw.Close() + t.Fatalf("create old table: %v", err) + } + raw.Close() + + // Opening with the new binary must run CREATE INDEX statements without + // erroring on missing generated columns. + s, err := Open(dbPath) + if err != nil { + t.Fatalf("open upgraded db: %v", err) + } + defer s.Close() + + // The migration must have added every generated column. + rows, err := s.DB().Query(`PRAGMA table_info("mcp")`) + if err != nil { + t.Fatalf("table_info: %v", err) + } + defer rows.Close() + + hasColumn := make(map[string]bool) + for rows.Next() { + var cid int + var name, typ string + var notnull, pk int + var dflt sql.NullString + if err := rows.Scan(&cid, &name, &typ, ¬null, &dflt, &pk); err != nil { + t.Fatalf("scan: %v", err) + } + hasColumn[name] = true + } + if err := rows.Err(); err != nil { + t.Fatalf("rows: %v", err) + } + + for _, want := range []string{ + "text", + "truncated", + "name", + "description", + } { + if !hasColumn[want] { + t.Fatalf("%s column missing from mcp after migrate", want) + } + } +} + +// TestMigrate_AddsColumnsOnUpgrade_Media verifies that opening a +// database created by an older binary succeeds and adds newly generated +// columns before CREATE INDEX runs against the pre-existing table. Regression +// coverage for parent_id upgrades and indexed generated columns. +func TestMigrate_AddsColumnsOnUpgrade_Media(t *testing.T) { + dbPath := filepath.Join(t.TempDir(), "data.db") + + // Pre-create the DB with the older table shape: id, data, synced_at and + // none of the newer generated columns. user_version stays 0 (pre-gate). + raw, err := sql.Open("sqlite", dbPath) + if err != nil { + t.Fatalf("open raw: %v", err) + } + if _, err := raw.Exec(`CREATE TABLE "media" ( + id TEXT PRIMARY KEY, + data JSON NOT NULL, + synced_at DATETIME DEFAULT CURRENT_TIMESTAMP + )`); err != nil { + raw.Close() + t.Fatalf("create old table: %v", err) + } + raw.Close() + + // Opening with the new binary must run CREATE INDEX statements without + // erroring on missing generated columns. + s, err := Open(dbPath) + if err != nil { + t.Fatalf("open upgraded db: %v", err) + } + defer s.Close() + + // The migration must have added every generated column. + rows, err := s.DB().Query(`PRAGMA table_info("media")`) + if err != nil { + t.Fatalf("table_info: %v", err) + } + defer rows.Close() + + hasColumn := make(map[string]bool) + for rows.Next() { + var cid int + var name, typ string + var notnull, pk int + var dflt sql.NullString + if err := rows.Scan(&cid, &name, &typ, ¬null, &dflt, &pk); err != nil { + t.Fatalf("scan: %v", err) + } + hasColumn[name] = true + } + if err := rows.Err(); err != nil { + t.Fatalf("rows: %v", err) + } + + for _, want := range []string{ + "status", + "dry_run", + "message", + "ledger_id", + "text", + "truncated", + } { + if !hasColumn[want] { + t.Fatalf("%s column missing from media after migrate", want) + } + } +} + +// TestMigrate_AddsColumnsOnUpgrade_Mirror verifies that opening a +// database created by an older binary succeeds and adds newly generated +// columns before CREATE INDEX runs against the pre-existing table. Regression +// coverage for parent_id upgrades and indexed generated columns. +func TestMigrate_AddsColumnsOnUpgrade_Mirror(t *testing.T) { + dbPath := filepath.Join(t.TempDir(), "data.db") + + // Pre-create the DB with the older table shape: id, data, synced_at and + // none of the newer generated columns. user_version stays 0 (pre-gate). + raw, err := sql.Open("sqlite", dbPath) + if err != nil { + t.Fatalf("open raw: %v", err) + } + if _, err := raw.Exec(`CREATE TABLE "mirror" ( + id TEXT PRIMARY KEY, + data JSON NOT NULL, + synced_at DATETIME DEFAULT CURRENT_TIMESTAMP + )`); err != nil { + raw.Close() + t.Fatalf("create old table: %v", err) + } + raw.Close() + + // Opening with the new binary must run CREATE INDEX statements without + // erroring on missing generated columns. + s, err := Open(dbPath) + if err != nil { + t.Fatalf("open upgraded db: %v", err) + } + defer s.Close() + + // The migration must have added every generated column. + rows, err := s.DB().Query(`PRAGMA table_info("mirror")`) + if err != nil { + t.Fatalf("table_info: %v", err) + } + defer rows.Close() + + hasColumn := make(map[string]bool) + for rows.Next() { + var cid int + var name, typ string + var notnull, pk int + var dflt sql.NullString + if err := rows.Scan(&cid, &name, &typ, ¬null, &dflt, &pk); err != nil { + t.Fatalf("scan: %v", err) + } + hasColumn[name] = true + } + if err := rows.Err(); err != nil { + t.Fatalf("rows: %v", err) + } + + for _, want := range []string{ + "uuid", + "name", + "kind", + "path", + "location", + "database", + "item_link", + "database_count", + "record_count", + "mirror_path", + } { + if !hasColumn[want] { + t.Fatalf("%s column missing from mirror after migrate", want) + } + } +} + +// TestMigrate_AddsColumnsOnUpgrade_Records verifies that opening a +// database created by an older binary succeeds and adds newly generated +// columns before CREATE INDEX runs against the pre-existing table. Regression +// coverage for parent_id upgrades and indexed generated columns. +func TestMigrate_AddsColumnsOnUpgrade_Records(t *testing.T) { + dbPath := filepath.Join(t.TempDir(), "data.db") + + // Pre-create the DB with the older table shape: id, data, synced_at and + // none of the newer generated columns. user_version stays 0 (pre-gate). + raw, err := sql.Open("sqlite", dbPath) + if err != nil { + t.Fatalf("open raw: %v", err) + } + if _, err := raw.Exec(`CREATE TABLE "records" ( + id TEXT PRIMARY KEY, + data JSON NOT NULL, + synced_at DATETIME DEFAULT CURRENT_TIMESTAMP + )`); err != nil { + raw.Close() + t.Fatalf("create old table: %v", err) + } + raw.Close() + + // Opening with the new binary must run CREATE INDEX statements without + // erroring on missing generated columns. + s, err := Open(dbPath) + if err != nil { + t.Fatalf("open upgraded db: %v", err) + } + defer s.Close() + + // The migration must have added every generated column. + rows, err := s.DB().Query(`PRAGMA table_info("records")`) + if err != nil { + t.Fatalf("table_info: %v", err) + } + defer rows.Close() + + hasColumn := make(map[string]bool) + for rows.Next() { + var cid int + var name, typ string + var notnull, pk int + var dflt sql.NullString + if err := rows.Scan(&cid, &name, &typ, ¬null, &dflt, &pk); err != nil { + t.Fatalf("scan: %v", err) + } + hasColumn[name] = true + } + if err := rows.Err(); err != nil { + t.Fatalf("rows: %v", err) + } + + for _, want := range []string{ + "uuid", + "content", + "content_format", + "truncated", + "name", + "kind", + "path", + "location", + "database", + "item_link", + "text", + } { + if !hasColumn[want] { + t.Fatalf("%s column missing from records after migrate", want) + } + } +} + +// TestMigrate_AddsColumnsOnUpgrade_Runtime verifies that opening a +// database created by an older binary succeeds and adds newly generated +// columns before CREATE INDEX runs against the pre-existing table. Regression +// coverage for parent_id upgrades and indexed generated columns. +func TestMigrate_AddsColumnsOnUpgrade_Runtime(t *testing.T) { + dbPath := filepath.Join(t.TempDir(), "data.db") + + // Pre-create the DB with the older table shape: id, data, synced_at and + // none of the newer generated columns. user_version stays 0 (pre-gate). + raw, err := sql.Open("sqlite", dbPath) + if err != nil { + t.Fatalf("open raw: %v", err) + } + if _, err := raw.Exec(`CREATE TABLE "runtime" ( + id TEXT PRIMARY KEY, + data JSON NOT NULL, + synced_at DATETIME DEFAULT CURRENT_TIMESTAMP + )`); err != nil { + raw.Close() + t.Fatalf("create old table: %v", err) + } + raw.Close() + + // Opening with the new binary must run CREATE INDEX statements without + // erroring on missing generated columns. + s, err := Open(dbPath) + if err != nil { + t.Fatalf("open upgraded db: %v", err) + } + defer s.Close() + + // The migration must have added every generated column. + rows, err := s.DB().Query(`PRAGMA table_info("runtime")`) + if err != nil { + t.Fatalf("table_info: %v", err) + } + defer rows.Close() + + hasColumn := make(map[string]bool) + for rows.Next() { + var cid int + var name, typ string + var notnull, pk int + var dflt sql.NullString + if err := rows.Scan(&cid, &name, &typ, ¬null, &dflt, &pk); err != nil { + t.Fatalf("scan: %v", err) + } + hasColumn[name] = true + } + if err := rows.Err(); err != nil { + t.Fatalf("rows: %v", err) + } + + for _, want := range []string{ + "running", + "version", + "applescript_ok", + "mcp_url", + "mirror_path", + } { + if !hasColumn[want] { + t.Fatalf("%s column missing from runtime after migrate", want) + } + } +} + +// TestMigrate_AddsColumnsOnUpgrade_Selection verifies that opening a +// database created by an older binary succeeds and adds newly generated +// columns before CREATE INDEX runs against the pre-existing table. Regression +// coverage for parent_id upgrades and indexed generated columns. +func TestMigrate_AddsColumnsOnUpgrade_Selection(t *testing.T) { + dbPath := filepath.Join(t.TempDir(), "data.db") + + // Pre-create the DB with the older table shape: id, data, synced_at and + // none of the newer generated columns. user_version stays 0 (pre-gate). + raw, err := sql.Open("sqlite", dbPath) + if err != nil { + t.Fatalf("open raw: %v", err) + } + if _, err := raw.Exec(`CREATE TABLE "selection" ( + id TEXT PRIMARY KEY, + data JSON NOT NULL, + synced_at DATETIME DEFAULT CURRENT_TIMESTAMP + )`); err != nil { + raw.Close() + t.Fatalf("create old table: %v", err) + } + raw.Close() + + // Opening with the new binary must run CREATE INDEX statements without + // erroring on missing generated columns. + s, err := Open(dbPath) + if err != nil { + t.Fatalf("open upgraded db: %v", err) + } + defer s.Close() + + // The migration must have added every generated column. + rows, err := s.DB().Query(`PRAGMA table_info("selection")`) + if err != nil { + t.Fatalf("table_info: %v", err) + } + defer rows.Close() + + hasColumn := make(map[string]bool) + for rows.Next() { + var cid int + var name, typ string + var notnull, pk int + var dflt sql.NullString + if err := rows.Scan(&cid, &name, &typ, ¬null, &dflt, &pk); err != nil { + t.Fatalf("scan: %v", err) + } + hasColumn[name] = true + } + if err := rows.Err(); err != nil { + t.Fatalf("rows: %v", err) + } + + for _, want := range []string{ + "uuid", + "name", + "kind", + "path", + "location", + "database", + "item_link", + "created_at", + "note", + } { + if !hasColumn[want] { + t.Fatalf("%s column missing from selection after migrate", want) + } + } +} + +// TestMigrate_AddsColumnsOnUpgrade_Tags verifies that opening a +// database created by an older binary succeeds and adds newly generated +// columns before CREATE INDEX runs against the pre-existing table. Regression +// coverage for parent_id upgrades and indexed generated columns. +func TestMigrate_AddsColumnsOnUpgrade_Tags(t *testing.T) { + dbPath := filepath.Join(t.TempDir(), "data.db") + + // Pre-create the DB with the older table shape: id, data, synced_at and + // none of the newer generated columns. user_version stays 0 (pre-gate). + raw, err := sql.Open("sqlite", dbPath) + if err != nil { + t.Fatalf("open raw: %v", err) + } + if _, err := raw.Exec(`CREATE TABLE "tags" ( + id TEXT PRIMARY KEY, + data JSON NOT NULL, + synced_at DATETIME DEFAULT CURRENT_TIMESTAMP + )`); err != nil { + raw.Close() + t.Fatalf("create old table: %v", err) + } + raw.Close() + + // Opening with the new binary must run CREATE INDEX statements without + // erroring on missing generated columns. + s, err := Open(dbPath) + if err != nil { + t.Fatalf("open upgraded db: %v", err) + } + defer s.Close() + + // The migration must have added every generated column. + rows, err := s.DB().Query(`PRAGMA table_info("tags")`) + if err != nil { + t.Fatalf("table_info: %v", err) + } + defer rows.Close() + + hasColumn := make(map[string]bool) + for rows.Next() { + var cid int + var name, typ string + var notnull, pk int + var dflt sql.NullString + if err := rows.Scan(&cid, &name, &typ, ¬null, &dflt, &pk); err != nil { + t.Fatalf("scan: %v", err) + } + hasColumn[name] = true + } + if err := rows.Err(); err != nil { + t.Fatalf("rows: %v", err) + } + + for _, want := range []string{ + "tag_count", + } { + if !hasColumn[want] { + t.Fatalf("%s column missing from tags after migrate", want) + } + } +} + +// TestMigrate_AddsColumnsOnUpgrade_SyncState verifies that opening a +// database created by an older binary succeeds and adds newly generated +// columns before CREATE INDEX runs against the pre-existing table. Regression +// coverage for parent_id upgrades and indexed generated columns. +func TestMigrate_AddsColumnsOnUpgrade_SyncState(t *testing.T) { + dbPath := filepath.Join(t.TempDir(), "data.db") + + // Pre-create the DB with the older table shape: id, data, synced_at and + // none of the newer generated columns. user_version stays 0 (pre-gate). + raw, err := sql.Open("sqlite", dbPath) + if err != nil { + t.Fatalf("open raw: %v", err) + } + if _, err := raw.Exec(`CREATE TABLE "sync_state" ( + id TEXT PRIMARY KEY, + data JSON NOT NULL, + synced_at DATETIME DEFAULT CURRENT_TIMESTAMP + )`); err != nil { + raw.Close() + t.Fatalf("create old table: %v", err) + } + raw.Close() + + // Opening with the new binary must run CREATE INDEX statements without + // erroring on missing generated columns. + s, err := Open(dbPath) + if err != nil { + t.Fatalf("open upgraded db: %v", err) + } + defer s.Close() + + // The migration must have added every generated column. + rows, err := s.DB().Query(`PRAGMA table_info("sync_state")`) + if err != nil { + t.Fatalf("table_info: %v", err) + } + defer rows.Close() + + hasColumn := make(map[string]bool) + for rows.Next() { + var cid int + var name, typ string + var notnull, pk int + var dflt sql.NullString + if err := rows.Scan(&cid, &name, &typ, ¬null, &dflt, &pk); err != nil { + t.Fatalf("scan: %v", err) + } + hasColumn[name] = true + } + if err := rows.Err(); err != nil { + t.Fatalf("rows: %v", err) + } + + for _, want := range []string{ + "last_cursor", + "last_synced_at", + "total_count", + } { + if !hasColumn[want] { + t.Fatalf("%s column missing from sync_state after migrate", want) + } + } +} diff --git a/library/productivity/devonthink/internal/store/store.go b/library/productivity/devonthink/internal/store/store.go new file mode 100644 index 0000000000..36ef537a59 --- /dev/null +++ b/library/productivity/devonthink/internal/store/store.go @@ -0,0 +1,2786 @@ +// Copyright 2026 rowdy and contributors. Licensed under Apache-2.0. See LICENSE. +// Generated by CLI Printing Press (https://github.com/mvanhorn/cli-printing-press). DO NOT EDIT. + +// Package store provides local SQLite persistence for devonthink-pp-cli. +// Uses modernc.org/sqlite (pure Go, no CGO) for zero-dependency cross-compilation. +// FTS5 full-text search indexes are created for searchable content. +package store + +import ( + "bytes" + "context" + "database/sql" + "encoding/json" + "fmt" + "hash/fnv" + "math" + "os" + "path/filepath" + "regexp" + "strconv" + "strings" + "sync" + "time" + + _ "modernc.org/sqlite" +) + +var uuidPattern = regexp.MustCompile(`^[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12}$`) +var isoDatePattern = regexp.MustCompile(`^\d{4}-\d{2}-\d{2}(?:[T ][0-9:.+-Zz]+)?$`) +var ftsQueryTokenRE = regexp.MustCompile(`[\pL\pN_]+`) + +var sqliteDriverInit struct { + mu sync.Mutex + done bool +} + +// validIdentifierRE pins ListField's `field` argument to a safe SQL +// identifier shape before any Sprintf interpolation. Matches what +// pragma_table_info implicitly enforces on the primary path, so the +// fallback path inherits the same defense without depending on whether +// the parent's typed domain table exists at the moment of the lookup. +var validIdentifierRE = regexp.MustCompile(`^[a-zA-Z_][a-zA-Z0-9_]*$`) + +// IsUUID returns true if the input looks like a UUID. +func IsUUID(s string) bool { + return uuidPattern.MatchString(s) +} + +// StoreSchemaVersion is the on-disk schema version this binary understands. +// It is stamped into SQLite's PRAGMA user_version on fresh databases and +// checked on every open. Non-learn CLIs advance to v4 for the +// resources_fts content extraction. +const StoreSchemaVersion = 4 + +// resourcesFTSContentSchemaVersion pins the schema bump that rewrote +// resources_fts content from raw JSON to searchable leaf values. Keep this +// separate from StoreSchemaVersion so future unrelated migrations do not +// trigger an expensive full FTS rebuild. +const resourcesFTSContentSchemaVersion = 4 + +const resourcesFTSCreateSQL = `CREATE VIRTUAL TABLE IF NOT EXISTS resources_fts USING fts5( + id, resource_type, content, tokenize='porter unicode61' +)` + +type Store struct { + db *sql.DB + // writeMu serializes all DB writes. Read paths bypass the lock and run + // concurrently against WAL. Resource-level concurrency in sync.go.tmpl + // is 1 (one goroutine per resource via len(resources)-sized work channel) + // — read-then-write sequences (e.g., GetSyncCursor → SaveSyncState) are + // race-free by construction within a resource. + writeMu sync.Mutex + path string +} + +// Open opens or creates the SQLite store at dbPath using the background +// context. Prefer OpenWithContext from a Cobra command so SIGINT during +// a slow migration interrupts the open instead of stranding the caller. +func Open(dbPath string) (*Store, error) { + return OpenWithContext(context.Background(), dbPath) +} + +// OpenReadOnly opens an existing SQLite store at dbPath in read-only mode. +// mode=ro rejects direct and CTE-wrapped writes (INSERT, UPDATE, DELETE, +// REPLACE, "WITH x AS (...) INSERT ...") at the driver level. Skips +// MkdirAll and migrate; the file is expected to exist. +// +// The file: URI prefix is load-bearing: modernc.org/sqlite only honors +// SQLite's URI query parameters (mode, cache, etc.) when the DSN starts +// with "file:". Without the prefix, "?mode=ro" is silently dropped and +// the connection opens read-write. Pragmas use the driver's _pragma= +// name(value) syntax — modernc.org/sqlite does NOT recognize the +// mattn/go-sqlite3 _journal_mode=WAL / _busy_timeout=5000 form and drops +// those keys silently, so the busy_timeout below is what keeps a read +// concurrent with a writer from failing immediately with SQLITE_BUSY. +// +// Deliberately no journal_mode pragma here: journal mode is a property of +// the database file, set by the read-write open, not the connection. Issuing +// PRAGMA journal_mode=WAL on a read-only handle to a DB still in the default +// delete mode (e.g. a pre-WAL database opened by an old binary before its +// first read-write open) errors with "attempt to write a readonly database". +func OpenReadOnly(dbPath string) (*Store, error) { + dsn := "file:" + dbPath + "?mode=ro&_pragma=busy_timeout(5000)&_pragma=foreign_keys(ON)&_pragma=temp_store(MEMORY)&_pragma=mmap_size(268435456)" + if err := ensureSQLiteDriverInitialized(context.Background(), dsn); err != nil { + return nil, err + } + + db, err := sql.Open("sqlite", dsn) + if err != nil { + return nil, fmt.Errorf("opening database (read-only): %w", err) + } + db.SetMaxOpenConns(2) + return &Store{db: db, path: dbPath}, nil +} + +// OpenWithContext opens or creates the SQLite store at dbPath. The +// context is honored by the migration path: cancellation interrupts the +// retry-on-SQLITE_BUSY loop and propagates ctx.Err() back to the caller +// instead of waiting out the full migrationLockTimeout. +func OpenWithContext(ctx context.Context, dbPath string) (*Store, error) { + if err := os.MkdirAll(filepath.Dir(dbPath), 0o700); err != nil { + return nil, fmt.Errorf("creating db directory: %w", err) + } + + dsn := dbPath + "?_pragma=journal_mode(WAL)&_pragma=synchronous(NORMAL)&_pragma=busy_timeout(5000)&_pragma=foreign_keys(ON)&_pragma=temp_store(MEMORY)&_pragma=mmap_size(268435456)" + if err := ensureSQLiteDriverInitialized(ctx, dsn); err != nil { + return nil, err + } + + db, err := sql.Open("sqlite", dsn) + if err != nil { + return nil, fmt.Errorf("opening database: %w", err) + } + + // WAL mode + 2 connections allows one read cursor open while a second + // query executes (e.g., analytics commands calling helpers during row + // iteration). Writes are still serialized by SQLite's WAL lock. + db.SetMaxOpenConns(2) + + s := &Store{db: db, path: dbPath} + if err := s.migrate(ctx); err != nil { + _ = db.Close() + return nil, fmt.Errorf("running migrations: %w", err) + } + + return s, nil +} + +func ensureSQLiteDriverInitialized(ctx context.Context, dsn string) error { + sqliteDriverInit.mu.Lock() + defer sqliteDriverInit.mu.Unlock() + + if sqliteDriverInit.done { + return nil + } + + db, err := sql.Open("sqlite", dsn) + if err != nil { + return fmt.Errorf("opening database for driver initialization: %w", err) + } + defer db.Close() + + conn, err := db.Conn(ctx) + if err != nil { + return fmt.Errorf("initializing sqlite driver: %w", err) + } + if err := conn.Close(); err != nil { + return fmt.Errorf("closing sqlite initialization connection: %w", err) + } + + sqliteDriverInit.done = true + return nil +} + +func (s *Store) Close() error { + return s.db.Close() +} + +// Path returns the on-disk path of the backing SQLite file. +func (s *Store) Path() string { + return s.path +} + +// DB exposes the underlying *sql.DB for callers that need to run ad-hoc +// queries (e.g., doctor's cache inspection, share snapshot import). +// Callers must not call Close on the returned handle. +func (s *Store) DB() *sql.DB { + return s.db +} + +// SchemaVersion reads PRAGMA user_version, which is stamped by migrate(). +// A zero value means the database predates the schema-version gate — not +// a bug, but the caller may want to warn. +func (s *Store) SchemaVersion() (int, error) { + var v int + if err := s.db.QueryRow(`PRAGMA user_version`).Scan(&v); err != nil { + return 0, fmt.Errorf("read user_version: %w", err) + } + return v, nil +} + +// ensureColumn adds a column to an existing table if it isn't already +// present. It is the upgrade-path safety valve for schema additions: +// CREATE TABLE IF NOT EXISTS is a no-op when the table already exists, so +// columns added by newer binaries (e.g. parent_id from the dependent- +// resources work) never land on databases created by older binaries — +// which then trip "no such column" when a follow-on CREATE INDEX runs. +// +// Skips silently if the table doesn't yet exist (fresh install — the +// CREATE TABLE migration will create it with the column already declared) +// or if the column already exists. Runs on the pinned migration +// connection so it sees the writes performed by the in-flight BEGIN +// IMMEDIATE transaction; using s.db here would route through the pool +// and BUSY against the holding writer under concurrent migrators. +func (s *Store) ensureColumn(ctx context.Context, conn *sql.Conn, table, column, decl string) error { + var name string + err := conn.QueryRowContext(ctx, + `SELECT name FROM sqlite_master WHERE type='table' AND name=?`, table, + ).Scan(&name) + if err == sql.ErrNoRows { + return nil + } + if err != nil { + return fmt.Errorf("checking table %s: %w", table, err) + } + + rows, err := conn.QueryContext(ctx, fmt.Sprintf(`PRAGMA table_info("%s")`, table)) + if err != nil { + return fmt.Errorf("table_info %s: %w", table, err) + } + defer rows.Close() + for rows.Next() { + var cid int + var n, typ string + var notnull, pk int + var dflt sql.NullString + if err := rows.Scan(&cid, &n, &typ, ¬null, &dflt, &pk); err != nil { + return fmt.Errorf("scan table_info %s: %w", table, err) + } + if n == column { + return nil + } + } + if err := rows.Err(); err != nil { + return fmt.Errorf("iterating table_info %s: %w", table, err) + } + + if _, err := conn.ExecContext(ctx, fmt.Sprintf(`ALTER TABLE "%s" ADD COLUMN "%s" %s`, table, column, decl)); err != nil { + // A concurrent Open() may have added the column between our + // PRAGMA check and this ALTER. SQLite returns SQLITE_ERROR with + // "duplicate column name", which busy_timeout does not retry. + // The DB is now in the desired state regardless of who won. + if strings.Contains(err.Error(), "duplicate column name") { + return nil + } + return fmt.Errorf("add column %s.%s: %w", table, column, err) + } + return nil +} + +// backfillColumns adds columns that newer binaries declare but that +// pre-existing databases (created before those columns were added) lack. +// Must run before the migrations slice so that subsequent CREATE INDEX +// statements referencing the column can succeed against the upgraded +// table. Idempotent: safe to call on fresh DBs (table-not-found short- +// circuit) and on already-current DBs (column-exists short-circuit). +// +// Table names are emitted bare (no safeName) — ensureColumn double-quotes +// them at SQL emit time and uses parameter binding for the sqlite_master +// lookup, so the values flow as Go string literals first and SQL +// identifiers second. Wrapping with safeName here would embed literal +// double-quote characters into the Go string and break compilation for +// any spec whose dependent-resource snake_cased name is a SQL reserved +// word. +func (s *Store) backfillColumns(ctx context.Context, conn *sql.Conn) error { + for _, c := range []struct{ table, column, decl string }{ + {table: "ai", column: "text", decl: "TEXT"}, + {table: "ai", column: "truncated", decl: "INTEGER"}, + {table: "batch", column: "status", decl: "TEXT"}, + {table: "batch", column: "dry_run", decl: "INTEGER"}, + {table: "batch", column: "message", decl: "TEXT"}, + {table: "batch", column: "ledger_id", decl: "TEXT"}, + {table: "batch", column: "plan_path", decl: "TEXT"}, + {table: "batch", column: "action_count", decl: "INTEGER"}, + {table: "databases", column: "uuid", decl: "TEXT"}, + {table: "databases", column: "name", decl: "TEXT"}, + {table: "databases", column: "incoming_group_uuid", decl: "TEXT"}, + {table: "databases", column: "is_open", decl: "INTEGER"}, + {table: "graph", column: "uuid", decl: "TEXT"}, + {table: "groups", column: "uuid", decl: "TEXT"}, + {table: "groups", column: "name", decl: "TEXT"}, + {table: "groups", column: "path", decl: "TEXT"}, + {table: "groups", column: "child_count", decl: "INTEGER"}, + {table: "ingest", column: "status", decl: "TEXT"}, + {table: "ingest", column: "dry_run", decl: "INTEGER"}, + {table: "ingest", column: "message", decl: "TEXT"}, + {table: "ingest", column: "ledger_id", decl: "TEXT"}, + {table: "ledger", column: "time", decl: "TEXT"}, + {table: "ledger", column: "action", decl: "TEXT"}, + {table: "ledger", column: "count", decl: "INTEGER"}, + {table: "ledger", column: "status", decl: "TEXT"}, + {table: "mcp", column: "text", decl: "TEXT"}, + {table: "mcp", column: "truncated", decl: "INTEGER"}, + {table: "mcp", column: "name", decl: "TEXT"}, + {table: "mcp", column: "description", decl: "TEXT"}, + {table: "media", column: "status", decl: "TEXT"}, + {table: "media", column: "dry_run", decl: "INTEGER"}, + {table: "media", column: "message", decl: "TEXT"}, + {table: "media", column: "ledger_id", decl: "TEXT"}, + {table: "media", column: "text", decl: "TEXT"}, + {table: "media", column: "truncated", decl: "INTEGER"}, + {table: "mirror", column: "uuid", decl: "TEXT"}, + {table: "mirror", column: "name", decl: "TEXT"}, + {table: "mirror", column: "kind", decl: "TEXT"}, + {table: "mirror", column: "path", decl: "TEXT"}, + {table: "mirror", column: "location", decl: "TEXT"}, + {table: "mirror", column: "database", decl: "TEXT"}, + {table: "mirror", column: "item_link", decl: "TEXT"}, + {table: "mirror", column: "database_count", decl: "INTEGER"}, + {table: "mirror", column: "record_count", decl: "INTEGER"}, + {table: "mirror", column: "mirror_path", decl: "TEXT"}, + {table: "records", column: "uuid", decl: "TEXT"}, + {table: "records", column: "content", decl: "TEXT"}, + {table: "records", column: "content_format", decl: "TEXT"}, + {table: "records", column: "truncated", decl: "INTEGER"}, + {table: "records", column: "name", decl: "TEXT"}, + {table: "records", column: "kind", decl: "TEXT"}, + {table: "records", column: "path", decl: "TEXT"}, + {table: "records", column: "location", decl: "TEXT"}, + {table: "records", column: "database", decl: "TEXT"}, + {table: "records", column: "item_link", decl: "TEXT"}, + {table: "records", column: "text", decl: "TEXT"}, + {table: "runtime", column: "running", decl: "INTEGER"}, + {table: "runtime", column: "version", decl: "TEXT"}, + {table: "runtime", column: "applescript_ok", decl: "INTEGER"}, + {table: "runtime", column: "mcp_url", decl: "TEXT"}, + {table: "runtime", column: "mirror_path", decl: "TEXT"}, + {table: "selection", column: "uuid", decl: "TEXT"}, + {table: "selection", column: "name", decl: "TEXT"}, + {table: "selection", column: "kind", decl: "TEXT"}, + {table: "selection", column: "path", decl: "TEXT"}, + {table: "selection", column: "location", decl: "TEXT"}, + {table: "selection", column: "database", decl: "TEXT"}, + {table: "selection", column: "item_link", decl: "TEXT"}, + {table: "selection", column: "created_at", decl: "TEXT"}, + {table: "selection", column: "note", decl: "TEXT"}, + {table: "tags", column: "tag_count", decl: "INTEGER"}, + {table: "sync_state", column: "last_cursor", decl: "TEXT"}, + {table: "sync_state", column: "last_synced_at", decl: "DATETIME"}, + {table: "sync_state", column: "total_count", decl: "INTEGER DEFAULT 0"}, + } { + if err := s.ensureColumn(ctx, conn, c.table, c.column, c.decl); err != nil { + return err + } + } + return nil +} + +func (s *Store) migrate(ctx context.Context) error { + // Acquiring the migration connection establishes a physical SQLite + // connection, which runs the DSN _pragma directives — including the + // journal_mode(WAL) conversion. On a fresh DB opened by several + // processes at once, that conversion briefly needs an exclusive lock + // and can return SQLITE_BUSY before any statement-level busy handler + // applies, so retry the acquisition against the shared deadline. + deadline := time.Now().Add(migrationLockTimeout) + var conn *sql.Conn + if err := retryOnBusy(ctx, deadline, "acquiring migration connection", func() error { + c, err := s.db.Conn(ctx) + if err != nil { + return err + } + conn = c + return nil + }); err != nil { + return err + } + defer conn.Close() + + // Read user_version before the migration lock so an old binary + // opening a newer-schema DB rejects immediately. WAL readers don't + // normally block on writers, but the fresh-DB WAL-init race can BUSY + // a SELECT — share the lock's deadline so total budget stays bounded. + var current int + if err := retryOnBusy(ctx, deadline, "reading schema version", func() error { + return conn.QueryRowContext(ctx, `PRAGMA user_version`).Scan(¤t) + }); err != nil { + return err + } + if current > StoreSchemaVersion { + return fmt.Errorf("database schema version %d is newer than supported version %d; upgrade the CLI binary or open an older database", current, StoreSchemaVersion) + } + + migrations := []string{ + `CREATE TABLE IF NOT EXISTS resources ( + id TEXT NOT NULL, + resource_type TEXT NOT NULL, + data JSON NOT NULL, + synced_at DATETIME DEFAULT CURRENT_TIMESTAMP, + updated_at DATETIME DEFAULT CURRENT_TIMESTAMP, + PRIMARY KEY (resource_type, id) + )`, + `CREATE INDEX IF NOT EXISTS idx_resources_type ON resources(resource_type)`, + `CREATE INDEX IF NOT EXISTS idx_resources_synced ON resources(synced_at)`, + `CREATE TABLE IF NOT EXISTS sync_state ( + resource_type TEXT PRIMARY KEY, + last_cursor TEXT, + last_synced_at DATETIME, + total_count INTEGER DEFAULT 0 + )`, + resourcesFTSCreateSQL, + `CREATE TABLE IF NOT EXISTS "ai" ( + "id" TEXT PRIMARY KEY, + "data" JSON NOT NULL, + "synced_at" DATETIME DEFAULT CURRENT_TIMESTAMP, + "text" TEXT, + "truncated" INTEGER + )`, + `CREATE TABLE IF NOT EXISTS "databases" ( + "id" TEXT PRIMARY KEY, + "data" JSON NOT NULL, + "synced_at" DATETIME DEFAULT CURRENT_TIMESTAMP, + "uuid" TEXT, + "name" TEXT, + "incoming_group_uuid" TEXT, + "is_open" INTEGER + )`, + `CREATE TABLE IF NOT EXISTS "graph" ( + "id" TEXT PRIMARY KEY, + "data" JSON NOT NULL, + "synced_at" DATETIME DEFAULT CURRENT_TIMESTAMP, + "uuid" TEXT + )`, + `CREATE TABLE IF NOT EXISTS "groups" ( + "id" TEXT PRIMARY KEY, + "data" JSON NOT NULL, + "synced_at" DATETIME DEFAULT CURRENT_TIMESTAMP, + "uuid" TEXT, + "name" TEXT, + "path" TEXT, + "child_count" INTEGER + )`, + `CREATE TABLE IF NOT EXISTS "ingest" ( + "id" TEXT PRIMARY KEY, + "data" JSON NOT NULL, + "synced_at" DATETIME DEFAULT CURRENT_TIMESTAMP, + "status" TEXT, + "dry_run" INTEGER, + "message" TEXT, + "ledger_id" TEXT + )`, + `CREATE INDEX IF NOT EXISTS "idx_ingest_ledger_id" ON "ingest"("ledger_id")`, + `CREATE TABLE IF NOT EXISTS "ledger" ( + "id" TEXT PRIMARY KEY, + "data" JSON NOT NULL, + "synced_at" DATETIME DEFAULT CURRENT_TIMESTAMP, + "time" TEXT, + "action" TEXT, + "count" INTEGER, + "status" TEXT + )`, + `CREATE TABLE IF NOT EXISTS "mcp" ( + "id" TEXT PRIMARY KEY, + "data" JSON NOT NULL, + "synced_at" DATETIME DEFAULT CURRENT_TIMESTAMP, + "text" TEXT, + "truncated" INTEGER, + "name" TEXT, + "description" TEXT + )`, + `CREATE VIRTUAL TABLE IF NOT EXISTS "mcp_fts" USING fts5( + "text", + "name", + "description", + content='mcp', + content_rowid='rowid' + )`, + `CREATE TRIGGER IF NOT EXISTS "mcp_ai" AFTER INSERT ON "mcp" BEGIN + INSERT INTO "mcp_fts"(rowid, "text", "name", "description") + VALUES (new.rowid,new."text", new."name", new."description"); + END`, + `CREATE TRIGGER IF NOT EXISTS "mcp_ad" AFTER DELETE ON "mcp" BEGIN + INSERT INTO "mcp_fts"("mcp_fts", rowid, "text", "name", "description") + VALUES ('delete', old.rowid,old."text", old."name", old."description"); + END`, + `CREATE TRIGGER IF NOT EXISTS "mcp_au" AFTER UPDATE ON "mcp" BEGIN + INSERT INTO "mcp_fts"("mcp_fts", rowid, "text", "name", "description") + VALUES ('delete', old.rowid,old."text", old."name", old."description"); + INSERT INTO "mcp_fts"(rowid, "text", "name", "description") + VALUES (new.rowid,new."text", new."name", new."description"); + END`, + `CREATE TABLE IF NOT EXISTS "media" ( + "id" TEXT PRIMARY KEY, + "data" JSON NOT NULL, + "synced_at" DATETIME DEFAULT CURRENT_TIMESTAMP, + "status" TEXT, + "dry_run" INTEGER, + "message" TEXT, + "ledger_id" TEXT, + "text" TEXT, + "truncated" INTEGER + )`, + `CREATE INDEX IF NOT EXISTS "idx_media_ledger_id" ON "media"("ledger_id")`, + `CREATE VIRTUAL TABLE IF NOT EXISTS "media_fts" USING fts5( + "message", + "text", + content='media', + content_rowid='rowid' + )`, + `CREATE TRIGGER IF NOT EXISTS "media_ai" AFTER INSERT ON "media" BEGIN + INSERT INTO "media_fts"(rowid, "message", "text") + VALUES (new.rowid,new."message", new."text"); + END`, + `CREATE TRIGGER IF NOT EXISTS "media_ad" AFTER DELETE ON "media" BEGIN + INSERT INTO "media_fts"("media_fts", rowid, "message", "text") + VALUES ('delete', old.rowid,old."message", old."text"); + END`, + `CREATE TRIGGER IF NOT EXISTS "media_au" AFTER UPDATE ON "media" BEGIN + INSERT INTO "media_fts"("media_fts", rowid, "message", "text") + VALUES ('delete', old.rowid,old."message", old."text"); + INSERT INTO "media_fts"(rowid, "message", "text") + VALUES (new.rowid,new."message", new."text"); + END`, + `CREATE TABLE IF NOT EXISTS "mirror" ( + "id" TEXT PRIMARY KEY, + "data" JSON NOT NULL, + "synced_at" DATETIME DEFAULT CURRENT_TIMESTAMP, + "uuid" TEXT, + "name" TEXT, + "kind" TEXT, + "path" TEXT, + "location" TEXT, + "database" TEXT, + "item_link" TEXT, + "database_count" INTEGER, + "record_count" INTEGER, + "mirror_path" TEXT + )`, + `CREATE TABLE IF NOT EXISTS "records" ( + "id" TEXT PRIMARY KEY, + "data" JSON NOT NULL, + "synced_at" DATETIME DEFAULT CURRENT_TIMESTAMP, + "uuid" TEXT, + "content" TEXT, + "content_format" TEXT, + "truncated" INTEGER, + "name" TEXT, + "kind" TEXT, + "path" TEXT, + "location" TEXT, + "database" TEXT, + "item_link" TEXT, + "text" TEXT + )`, + `CREATE VIRTUAL TABLE IF NOT EXISTS "records_fts" USING fts5( + "content", + "name", + "text", + content='records', + content_rowid='rowid' + )`, + `CREATE TRIGGER IF NOT EXISTS "records_ai" AFTER INSERT ON "records" BEGIN + INSERT INTO "records_fts"(rowid, "content", "name", "text") + VALUES (new.rowid,new."content", new."name", new."text"); + END`, + `CREATE TRIGGER IF NOT EXISTS "records_ad" AFTER DELETE ON "records" BEGIN + INSERT INTO "records_fts"("records_fts", rowid, "content", "name", "text") + VALUES ('delete', old.rowid,old."content", old."name", old."text"); + END`, + `CREATE TRIGGER IF NOT EXISTS "records_au" AFTER UPDATE ON "records" BEGIN + INSERT INTO "records_fts"("records_fts", rowid, "content", "name", "text") + VALUES ('delete', old.rowid,old."content", old."name", old."text"); + INSERT INTO "records_fts"(rowid, "content", "name", "text") + VALUES (new.rowid,new."content", new."name", new."text"); + END`, + `CREATE TABLE IF NOT EXISTS "runtime" ( + "id" TEXT PRIMARY KEY, + "data" JSON NOT NULL, + "synced_at" DATETIME DEFAULT CURRENT_TIMESTAMP, + "running" INTEGER, + "version" TEXT, + "applescript_ok" INTEGER, + "mcp_url" TEXT, + "mirror_path" TEXT + )`, + `CREATE TABLE IF NOT EXISTS "selection" ( + "id" TEXT PRIMARY KEY, + "data" JSON NOT NULL, + "synced_at" DATETIME DEFAULT CURRENT_TIMESTAMP, + "uuid" TEXT, + "name" TEXT, + "kind" TEXT, + "path" TEXT, + "location" TEXT, + "database" TEXT, + "item_link" TEXT, + "created_at" TEXT, + "note" TEXT + )`, + `CREATE INDEX IF NOT EXISTS "idx_selection_created_at" ON "selection"("created_at")`, + `CREATE VIRTUAL TABLE IF NOT EXISTS "selection_fts" USING fts5( + "name", + "note", + content='selection', + content_rowid='rowid' + )`, + `CREATE TRIGGER IF NOT EXISTS "selection_ai" AFTER INSERT ON "selection" BEGIN + INSERT INTO "selection_fts"(rowid, "name", "note") + VALUES (new.rowid,new."name", new."note"); + END`, + `CREATE TRIGGER IF NOT EXISTS "selection_ad" AFTER DELETE ON "selection" BEGIN + INSERT INTO "selection_fts"("selection_fts", rowid, "name", "note") + VALUES ('delete', old.rowid,old."name", old."note"); + END`, + `CREATE TRIGGER IF NOT EXISTS "selection_au" AFTER UPDATE ON "selection" BEGIN + INSERT INTO "selection_fts"("selection_fts", rowid, "name", "note") + VALUES ('delete', old.rowid,old."name", old."note"); + INSERT INTO "selection_fts"(rowid, "name", "note") + VALUES (new.rowid,new."name", new."note"); + END`, + `CREATE TABLE IF NOT EXISTS "tags" ( + "id" TEXT PRIMARY KEY, + "data" JSON NOT NULL, + "synced_at" DATETIME DEFAULT CURRENT_TIMESTAMP, + "tag_count" INTEGER + )`, + } + + // Run every migration — including the column backfill and the + // schema-version stamp — inside a single BEGIN IMMEDIATE transaction + // pinned to one connection. IMMEDIATE acquires SQLite's RESERVED lock + // at BEGIN time so concurrent migrators serialize on it instead of + // racing per-statement and tripping SQLITE_BUSY despite busy_timeout. + // modernc.org/sqlite's busy_timeout does not always cover write-write + // contention at BEGIN/COMMIT time, so we retry both explicitly on + // SQLITE_BUSY for up to migrationLockTimeout. + return withMigrationLock(ctx, conn, deadline, func() error { + // Re-read user_version inside the lock. This is load-bearing, + // not paranoid: between the pre-lock read above and our + // successful BEGIN IMMEDIATE, a newer-binary peer may have + // committed a higher version stamp. Without this re-read, an + // older binary (smaller StoreSchemaVersion) would proceed to + // stamp its own lower version at the end of the closure, + // silently downgrading user_version on a schema that's already + // at the newer level. Future maintainers: leave this read in. + var current int + if err := conn.QueryRowContext(ctx, `PRAGMA user_version`).Scan(¤t); err != nil { + return fmt.Errorf("reading schema version: %w", err) + } + if current > StoreSchemaVersion { + return fmt.Errorf("database schema version %d is newer than supported version %d; upgrade the CLI binary or open an older database", current, StoreSchemaVersion) + } + + if current < 2 { + if err := s.migrateResourcesCompositeKey(ctx, conn); err != nil { + return fmt.Errorf("migrating resources composite key: %w", err) + } + } + if current == 2 { + if err := s.migrateResourcesFTSRowIDs(ctx, conn); err != nil { + return fmt.Errorf("migrating resources FTS rowids: %w", err) + } + } + + if err := s.backfillColumns(ctx, conn); err != nil { + return fmt.Errorf("backfilling columns: %w", err) + } + for _, m := range migrations { + if _, err := conn.ExecContext(ctx, m); err != nil { + return fmt.Errorf("migration failed: %w", err) + } + } + if err := s.migrateExtras(ctx, conn); err != nil { + return fmt.Errorf("running extra migrations: %w", err) + } + if current < resourcesFTSContentSchemaVersion { + if err := s.migrateResourcesFTSContent(ctx, conn); err != nil { + return fmt.Errorf("migrating resources FTS content: %w", err) + } + } + // Stamp the schema version. On a fresh DB this writes the current + // StoreSchemaVersion; on an already-stamped DB this is a no-op + // write of the same value. + // An older DB with user_version = 0 and pre-existing tables gets + // stamped here after any version-gated rewrites and idempotent + // CREATE TABLE IF NOT EXISTS statements have completed. + if _, err := conn.ExecContext(ctx, fmt.Sprintf(`PRAGMA user_version = %d`, StoreSchemaVersion)); err != nil { + return fmt.Errorf("stamp user_version: %w", err) + } + return nil + }) +} + +func (s *Store) migrateResourcesCompositeKey(ctx context.Context, conn *sql.Conn) error { + exists, err := tableExists(ctx, conn, "resources") + if err != nil { + return err + } + if !exists { + return nil + } + + composite, err := resourcesTableHasCompositeKey(ctx, conn) + if err != nil { + return err + } + if !composite { + if _, err := conn.ExecContext(ctx, `CREATE TABLE resources_v2 ( + id TEXT NOT NULL, + resource_type TEXT NOT NULL, + data JSON NOT NULL, + synced_at DATETIME DEFAULT CURRENT_TIMESTAMP, + updated_at DATETIME DEFAULT CURRENT_TIMESTAMP, + PRIMARY KEY (resource_type, id) + )`); err != nil { + return fmt.Errorf("creating resources_v2: %w", err) + } + if _, err := conn.ExecContext(ctx, `INSERT INTO resources_v2 (id, resource_type, data, synced_at, updated_at) + SELECT id, resource_type, data, synced_at, updated_at FROM resources`); err != nil { + return fmt.Errorf("copying resources rows: %w", err) + } + if _, err := conn.ExecContext(ctx, `DROP TABLE resources`); err != nil { + return fmt.Errorf("dropping old resources table: %w", err) + } + if _, err := conn.ExecContext(ctx, `ALTER TABLE resources_v2 RENAME TO resources`); err != nil { + return fmt.Errorf("renaming resources_v2: %w", err) + } + } + + // Always rebuild FTS during the v2 transition. The resources table may + // already have the composite key, but v1 FTS rowids were scoped by id + // alone and must be replaced with resource_type + id rowids. + if _, err := conn.ExecContext(ctx, `DROP TABLE IF EXISTS resources_fts`); err != nil { + return fmt.Errorf("dropping resources_fts: %w", err) + } + if _, err := conn.ExecContext(ctx, resourcesFTSCreateSQL); err != nil { + return fmt.Errorf("creating resources_fts: %w", err) + } + if err := rebuildResourcesFTS(ctx, conn); err != nil { + return fmt.Errorf("rebuilding resources_fts: %w", err) + } + return nil +} + +func (s *Store) migrateResourcesFTSRowIDs(ctx context.Context, conn *sql.Conn) error { + exists, err := tableExists(ctx, conn, "resources") + if err != nil { + return err + } + if !exists { + return nil + } + + if _, err := conn.ExecContext(ctx, `DROP TABLE IF EXISTS resources_fts`); err != nil { + return fmt.Errorf("dropping resources_fts: %w", err) + } + if _, err := conn.ExecContext(ctx, resourcesFTSCreateSQL); err != nil { + return fmt.Errorf("creating resources_fts: %w", err) + } + if err := rebuildResourcesFTS(ctx, conn); err != nil { + return fmt.Errorf("rebuilding resources_fts: %w", err) + } + return nil +} + +func (s *Store) migrateResourcesFTSContent(ctx context.Context, conn *sql.Conn) error { + exists, err := tableExists(ctx, conn, "resources") + if err != nil { + return err + } + if !exists { + return nil + } + + if _, err := conn.ExecContext(ctx, `DROP TABLE IF EXISTS resources_fts`); err != nil { + return fmt.Errorf("dropping resources_fts: %w", err) + } + if _, err := conn.ExecContext(ctx, resourcesFTSCreateSQL); err != nil { + return fmt.Errorf("creating resources_fts: %w", err) + } + if err := rebuildResourcesFTS(ctx, conn); err != nil { + return fmt.Errorf("rebuilding resources_fts: %w", err) + } + return nil +} + +func tableExists(ctx context.Context, conn *sql.Conn, name string) (bool, error) { + var count int + if err := conn.QueryRowContext(ctx, `SELECT COUNT(*) FROM sqlite_master WHERE type = 'table' AND name = ?`, name).Scan(&count); err != nil { + return false, fmt.Errorf("checking table %s: %w", name, err) + } + return count > 0, nil +} + +func resourcesTableHasCompositeKey(ctx context.Context, conn *sql.Conn) (bool, error) { + rows, err := conn.QueryContext(ctx, `PRAGMA table_info(resources)`) + if err != nil { + return false, fmt.Errorf("reading resources table info: %w", err) + } + defer rows.Close() + + pk := map[string]int{} + for rows.Next() { + var cid int + var name, typ string + var notnull, pkOrder int + var dflt sql.NullString + if err := rows.Scan(&cid, &name, &typ, ¬null, &dflt, &pkOrder); err != nil { + return false, fmt.Errorf("scanning resources table info: %w", err) + } + pk[name] = pkOrder + } + if err := rows.Err(); err != nil { + return false, fmt.Errorf("reading resources table info rows: %w", err) + } + return pk["resource_type"] == 1 && pk["id"] == 2, nil +} + +func rebuildResourcesFTS(ctx context.Context, conn *sql.Conn) error { + rows, err := conn.QueryContext(ctx, `SELECT id, resource_type, data FROM resources`) + if err != nil { + return fmt.Errorf("querying resources: %w", err) + } + + type resourceRow struct { + id string + resourceType string + data string + } + var resources []resourceRow + for rows.Next() { + var r resourceRow + if err := rows.Scan(&r.id, &r.resourceType, &r.data); err != nil { + _ = rows.Close() + return fmt.Errorf("scanning resource: %w", err) + } + resources = append(resources, r) + } + if err := rows.Err(); err != nil { + _ = rows.Close() + return fmt.Errorf("reading resource rows: %w", err) + } + if err := rows.Close(); err != nil { + return fmt.Errorf("closing resource rows: %w", err) + } + + for _, r := range resources { + if _, err := conn.ExecContext(ctx, + `INSERT INTO resources_fts (rowid, id, resource_type, content) VALUES (?, ?, ?, ?)`, + ftsRowID(r.resourceType, r.id), r.id, r.resourceType, searchableResourceContent(json.RawMessage(r.data)), + ); err != nil { + return fmt.Errorf("indexing resource %s/%s: %w", r.resourceType, r.id, err) + } + } + return nil +} + +const ( + migrationLockTimeout = 30 * time.Second + migrationLockBackoffMin = 5 * time.Millisecond + migrationLockBackoffMax = 100 * time.Millisecond +) + +// withMigrationLock runs fn inside a BEGIN IMMEDIATE / COMMIT pair on +// conn, retrying both BEGIN and COMMIT on SQLITE_BUSY against the +// caller-provided deadline. Sharing the deadline with the pre-lock +// version read keeps total Open() latency bounded by a single budget. +// The real upper bound is deadline + one trailing backoff interval +// (≤100ms) + the driver's busy_timeout for the in-flight Exec, since +// the deadline is checked after each failed attempt rather than as a +// hard wall-clock cutoff. fn must use conn (not s.db) so its writes +// participate in the held transaction. +func withMigrationLock(ctx context.Context, conn *sql.Conn, deadline time.Time, fn func() error) error { + if err := execWithBusyRetry(ctx, conn, "BEGIN IMMEDIATE", "begin migration transaction", deadline); err != nil { + return err + } + committed := false + defer func() { + if committed { + return + } + // ROLLBACK uses context.Background() so caller-context cancellation + // can't strand the connection in an open transaction. A failed + // rollback is rare on local SQLite (broken file handle, fatal + // driver error) but worth surfacing — silent swallow leaves a + // pinned connection returned to the pool with state that will + // confuse later queries. + if _, rerr := conn.ExecContext(context.Background(), "ROLLBACK"); rerr != nil { + fmt.Fprintf(os.Stderr, "warning: store migration rollback failed: %v\n", rerr) + } + }() + + if err := fn(); err != nil { + return err + } + + if err := execWithBusyRetry(ctx, conn, "COMMIT", "commit migration transaction", deadline); err != nil { + return err + } + committed = true + return nil +} + +// execWithBusyRetry runs stmt on conn and retries on SQLITE_BUSY until +// deadline. It covers BEGIN IMMEDIATE and COMMIT contention; +// modernc.org/sqlite's busy_timeout does not reliably cover either when +// multiple connections race for the WAL write lock. +func execWithBusyRetry(ctx context.Context, conn *sql.Conn, stmt, label string, deadline time.Time) error { + return retryOnBusy(ctx, deadline, label, func() error { + _, err := conn.ExecContext(ctx, stmt) + return err + }) +} + +// retryOnBusy runs op and retries it on SQLITE_BUSY/LOCKED until +// deadline. The same retry shape covers Exec, Query, and any other +// SQLite call that can race the WAL writer lock — including the +// pre-lock user_version read, where the WAL initialization race on a +// fresh DB can BUSY a SELECT that should otherwise succeed under WAL +// reader/writer concurrency. +func retryOnBusy(ctx context.Context, deadline time.Time, label string, op func() error) error { + backoff := migrationLockBackoffMin + for { + err := op() + if err == nil { + return nil + } + if !isSQLiteBusy(err) { + return fmt.Errorf("%s: %w", label, err) + } + if time.Now().After(deadline) { + // The label carries the operation context (e.g. "begin + // migration transaction", "reading schema version") — we + // don't hardcode "waiting for write lock" because pre-lock + // reads also flow through this helper. + return fmt.Errorf("%s: timed out after %s under SQLite contention: %w", label, migrationLockTimeout, err) + } + select { + case <-ctx.Done(): + return fmt.Errorf("%s: %w", label, ctx.Err()) + case <-time.After(backoff): + } + backoff = min(backoff*2, migrationLockBackoffMax) + } +} + +// isSQLiteBusy reports whether err is a retryable SQLite lock condition. +// Covers both the file-level WAL writer race (SQLITE_BUSY / "database is +// locked") and the table-level shared-cache contention (SQLITE_LOCKED / +// "database table is locked"). The match is on the error string because +// modernc.org/sqlite does not export an error type the generated code +// can switch on without dragging the driver package into every store +// consumer. +func isSQLiteBusy(err error) bool { + if err == nil { + return false + } + msg := err.Error() + return strings.Contains(msg, "SQLITE_BUSY") || + strings.Contains(msg, "SQLITE_LOCKED") || + strings.Contains(msg, "database is locked") || + strings.Contains(msg, "database table is locked") +} + +func (s *Store) upsertGenericResourceTx(tx *sql.Tx, resourceType, id string, data json.RawMessage) error { + _, err := tx.Exec( + `INSERT INTO resources (id, resource_type, data, synced_at, updated_at) + VALUES (?, ?, ?, ?, ?) + ON CONFLICT(resource_type, id) DO UPDATE SET data = excluded.data, synced_at = excluded.synced_at, updated_at = excluded.updated_at`, + id, resourceType, string(data), time.Now().UTC().Format(time.RFC3339), time.Now().UTC().Format(time.RFC3339), + ) + if err != nil { + return err + } + + ftsRowid := ftsRowID(resourceType, id) + // Use explicit rowid for FTS5 compatibility with modernc.org/sqlite. + // Standard DELETE WHERE column=? may not work on FTS5 virtual tables. + if _, err = tx.Exec(`DELETE FROM resources_fts WHERE rowid = ?`, ftsRowid); err != nil { + fmt.Fprintf(os.Stderr, "warning: FTS index cleanup failed: %v\n", err) + } + + if _, err = tx.Exec( + `INSERT INTO resources_fts (rowid, id, resource_type, content) + VALUES (?, ?, ?, ?)`, + ftsRowid, id, resourceType, searchableResourceContent(data), + ); err != nil { + // FTS insert failure is non-fatal + fmt.Fprintf(os.Stderr, "warning: FTS index update failed: %v\n", err) + } + + return nil +} + +func (s *Store) Upsert(resourceType, id string, data json.RawMessage) error { + s.writeMu.Lock() + defer s.writeMu.Unlock() + tx, err := s.db.Begin() + if err != nil { + return err + } + defer tx.Rollback() + + if err := s.upsertGenericResourceTx(tx, resourceType, id, data); err != nil { + return err + } + + return tx.Commit() +} + +// Propagates sql.ErrNoRows on a miss so callers can distinguish absence from +// other scan errors via errors.Is. +func (s *Store) Get(resourceType, id string) (json.RawMessage, error) { + var data string + err := s.db.QueryRow( + `SELECT data FROM resources WHERE resource_type = ? AND id = ?`, + resourceType, id, + ).Scan(&data) + if err != nil { + return nil, err + } + return json.RawMessage(data), nil +} + +// List returns resources of the given type. A positive limit caps the result +// count; zero or negative means no limit. +func (s *Store) List(resourceType string, limit int) ([]json.RawMessage, error) { + query := `SELECT data FROM resources WHERE resource_type = ? ORDER BY updated_at DESC` + args := []any{resourceType} + if limit > 0 { + query += ` LIMIT ?` + args = append(args, limit) + } + rows, err := s.db.Query(query, args...) + if err != nil { + return nil, err + } + defer rows.Close() + + var results []json.RawMessage + for rows.Next() { + var data string + if err := rows.Scan(&data); err != nil { + return nil, err + } + results = append(results, json.RawMessage(data)) + } + return results, rows.Err() +} + +func (s *Store) Search(query string, limit int, resourceTypes ...string) ([]json.RawMessage, error) { + if limit <= 0 { + limit = 50 + } + matchQuery := ftsMatchQuery(query) + if matchQuery == "" { + return nil, nil + } + resourceType := "" + if len(resourceTypes) > 0 { + resourceType = strings.TrimSpace(resourceTypes[0]) + } + if resourceType != "" { + rows, err := s.db.Query( + `SELECT r.data FROM resources r + JOIN resources_fts f ON r.id = f.id AND r.resource_type = f.resource_type + WHERE resources_fts MATCH ? + AND r.resource_type = ? + ORDER BY rank + LIMIT ?`, + matchQuery, resourceType, limit, + ) + if err != nil { + return nil, err + } + defer rows.Close() + + var results []json.RawMessage + for rows.Next() { + var data string + if err := rows.Scan(&data); err != nil { + return nil, err + } + results = append(results, json.RawMessage(data)) + } + return results, rows.Err() + } + rows, err := s.db.Query( + `SELECT r.data FROM resources r + JOIN resources_fts f ON r.id = f.id AND r.resource_type = f.resource_type + WHERE resources_fts MATCH ? + ORDER BY rank + LIMIT ?`, + matchQuery, limit, + ) + if err != nil { + return nil, err + } + defer rows.Close() + + var results []json.RawMessage + for rows.Next() { + var data string + if err := rows.Scan(&data); err != nil { + return nil, err + } + results = append(results, json.RawMessage(data)) + } + return results, rows.Err() +} + +func searchableResourceContent(data json.RawMessage) string { + dec := json.NewDecoder(bytes.NewReader(data)) + dec.UseNumber() + var value any + if err := dec.Decode(&value); err != nil { + return "" + } + var parts []string + collectSearchableStrings(&parts, "", value) + return strings.Join(parts, " ") +} + +func collectSearchableStrings(parts *[]string, key string, value any) { + switch v := value.(type) { + case map[string]any: + for childKey, child := range v { + collectSearchableStrings(parts, childKey, child) + } + case []any: + for _, child := range v { + collectSearchableStrings(parts, key, child) + } + case string: + if shouldIndexSearchString(key, v) { + *parts = append(*parts, strings.TrimSpace(v)) + } + } +} + +func shouldIndexSearchString(key, value string) bool { + s := strings.TrimSpace(value) + if len(s) < 2 { + return false + } + if isIdentifierKey(key) { + return false + } + lower := strings.ToLower(s) + upper := strings.ToUpper(s) + switch { + case IsUUID(s): + return false + case isoDatePattern.MatchString(s): + return false + case strings.HasPrefix(lower, "http://") || strings.HasPrefix(lower, "https://"): + return false + case upper == s && len(s) == 3 && strings.IndexFunc(s, func(r rune) bool { return r < 'A' || r > 'Z' }) == -1: + return false + } + tokens := ftsQueryTokenRE.FindAllString(s, -1) + return len(tokens) > 0 +} + +func isIdentifierKey(key string) bool { + if key == "" { + return false + } + lower := strings.ToLower(key) + return lower == "id" || + lower == "uuid" || + strings.HasSuffix(lower, "_id") || + strings.HasSuffix(lower, "-id") || + strings.HasSuffix(key, "Id") || + strings.HasSuffix(key, "ID") +} + +func ftsMatchQuery(query string) string { + tokens := ftsQueryTokenRE.FindAllString(query, -1) + if len(tokens) == 0 { + return "" + } + quoted := make([]string, 0, len(tokens)) + for _, token := range tokens { + quoted = append(quoted, `"`+token+`"`) + } + return strings.Join(quoted, " ") +} + +func extractObjectID(obj map[string]any) string { + for _, key := range []string{"id", "Id", "ID", "uuid", "slug", "name"} { + if v, ok := obj[key]; ok { + return ResourceIDString(v) + } + } + return "" +} + +// ftsRowID derives a deterministic rowid from a string ID for use with FTS5. +// Any change to this derivation requires a StoreSchemaVersion bump and a +// resources_fts rebuild migration for already-stamped databases. +// modernc.org/sqlite's FTS5 implementation may not support DELETE WHERE column=? +// on virtual tables, so we use explicit rowids and DELETE WHERE rowid=? instead. +func ftsRowID(scope, id string) int64 { + h := fnv.New64a() + _, _ = h.Write([]byte(scope)) + _, _ = h.Write([]byte{0}) // separator so ("ab","c") != ("a","bc") + _, _ = h.Write([]byte(id)) + return int64(h.Sum64() & 0x7FFFFFFFFFFFFFFF) // ensure positive +} + +// LookupFieldValue resolves a field value from a JSON object map, trying the +// snake_case key first, then the camelCase rendering, then the PascalCase +// rendering. Exported so the sync command's extractID and the upsert path +// resolve fields the same way — a divergence here produces silent drops on +// heterogeneous payloads. The PascalCase pass handles .NET-shaped responses +// (`Id`, `Name`, `OrderId`) without forcing each spec to declare casing. +func LookupFieldValue(obj map[string]any, snakeKey string) any { + if v, ok := obj[snakeKey]; ok { + return sqliteFieldValue(v) + } + parts := strings.Split(snakeKey, "_") + for i := 1; i < len(parts); i++ { + if parts[i] == "" { + continue + } + parts[i] = strings.ToUpper(parts[i][:1]) + parts[i][1:] + } + camel := strings.Join(parts, "") + if v, ok := obj[camel]; ok { + return sqliteFieldValue(v) + } + if parts[0] != "" { + pascal := strings.ToUpper(parts[0][:1]) + parts[0][1:] + strings.Join(parts[1:], "") + if v, ok := obj[pascal]; ok { + return sqliteFieldValue(v) + } + } + return nil +} + +func sqliteFieldValue(v any) any { + switch t := v.(type) { + case nil, string, bool, int, int64, float64, []byte: + return v + case json.Number: + return strings.TrimSpace(t.String()) + default: + data, err := json.Marshal(v) + if err != nil { + return fmt.Sprint(v) + } + return string(data) + } +} + +// lookupFieldValue is kept as an unexported alias for in-package callers so +// the existing UpsertBatch code reads naturally without prefixing every call +// with the package name. +func lookupFieldValue(obj map[string]any, snakeKey string) any { + return LookupFieldValue(obj, snakeKey) +} + +// DecodeJSONObject decodes data into an object while preserving JSON numbers. +// Plain json.Unmarshal turns numbers into float64, and fmt on those values can +// render large integer IDs as scientific notation before they reach resources.id. +func DecodeJSONObject(data json.RawMessage) (map[string]any, error) { + var obj map[string]any + dec := json.NewDecoder(bytes.NewReader(data)) + dec.UseNumber() + if err := dec.Decode(&obj); err != nil { + return nil, err + } + return obj, nil +} + +// ResourceIDString returns the stable text form used for resources.id. +func ResourceIDString(v any) string { + switch t := v.(type) { + case nil: + return "" + case json.Number: + return strings.TrimSpace(t.String()) + case float64: + if math.IsNaN(t) || math.IsInf(t, 0) { + return "" + } + return strconv.FormatFloat(t, 'f', -1, 64) + case float32: + f := float64(t) + if math.IsNaN(f) || math.IsInf(f, 0) { + return "" + } + return strconv.FormatFloat(f, 'f', -1, 32) + default: + // fmt.Sprint on typed nil pointers returns "<nil>"; callers still guard + // that sentinel so unresolved IDs do not become stored resource keys. + return strings.TrimSpace(fmt.Sprint(t)) + } +} + +// upsertAiTx writes the per-resource domain-table portion of a +// ai upsert inside an existing transaction. The caller is +// responsible for the generic resources insert (via upsertGenericResourceTx) +// and for committing the tx. Splitting this out lets UpsertBatch dispatch +// domain inserts per item without opening a per-item transaction. +func (s *Store) upsertAiTx(tx *sql.Tx, id string, obj map[string]any, data json.RawMessage) error { + if _, err := tx.Exec( + `INSERT INTO "ai" ("id", "data", "synced_at", "text", "truncated") + VALUES (?, ?, ?, ?, ?) + ON CONFLICT("id") DO UPDATE SET "data" = excluded."data", "synced_at" = excluded."synced_at", "text" = excluded."text", "truncated" = excluded."truncated"`, + id, + string(data), + time.Now().UTC().Format(time.RFC3339), + lookupFieldValue(obj, "text"), + lookupFieldValue(obj, "truncated"), + ); err != nil { + return fmt.Errorf("insert into ai: %w", err) + } + + return nil +} + +// UpsertAi inserts or updates a ai record with domain-specific columns. +func (s *Store) UpsertAi(data json.RawMessage) error { + obj, err := DecodeJSONObject(data) + if err != nil { + return fmt.Errorf("unmarshaling ai: %w", err) + } + + id := extractObjectID(obj) + if id == "" { + return fmt.Errorf("missing id for ai") + } + storageID := resourceStorageID("ai", id, obj) + + s.writeMu.Lock() + defer s.writeMu.Unlock() + tx, err := s.db.Begin() + if err != nil { + return err + } + defer tx.Rollback() + + if err := s.upsertGenericResourceTx(tx, "ai", storageID, data); err != nil { + return err + } + if err := s.upsertAiTx(tx, storageID, obj, data); err != nil { + return err + } + + return tx.Commit() +} + +// upsertDatabasesTx writes the per-resource domain-table portion of a +// databases upsert inside an existing transaction. The caller is +// responsible for the generic resources insert (via upsertGenericResourceTx) +// and for committing the tx. Splitting this out lets UpsertBatch dispatch +// domain inserts per item without opening a per-item transaction. +func (s *Store) upsertDatabasesTx(tx *sql.Tx, id string, obj map[string]any, data json.RawMessage) error { + if _, err := tx.Exec( + `INSERT INTO "databases" ("id", "data", "synced_at", "uuid", "name", "incoming_group_uuid", "is_open") + VALUES (?, ?, ?, ?, ?, ?, ?) + ON CONFLICT("id") DO UPDATE SET "data" = excluded."data", "synced_at" = excluded."synced_at", "uuid" = excluded."uuid", "name" = excluded."name", "incoming_group_uuid" = excluded."incoming_group_uuid", "is_open" = excluded."is_open"`, + id, + string(data), + time.Now().UTC().Format(time.RFC3339), + lookupFieldValue(obj, "uuid"), + lookupFieldValue(obj, "name"), + lookupFieldValue(obj, "incoming_group_uuid"), + lookupFieldValue(obj, "is_open"), + ); err != nil { + return fmt.Errorf("insert into databases: %w", err) + } + + return nil +} + +// UpsertDatabases inserts or updates a databases record with domain-specific columns. +func (s *Store) UpsertDatabases(data json.RawMessage) error { + obj, err := DecodeJSONObject(data) + if err != nil { + return fmt.Errorf("unmarshaling databases: %w", err) + } + + id := extractObjectID(obj) + if id == "" { + return fmt.Errorf("missing id for databases") + } + storageID := resourceStorageID("databases", id, obj) + + s.writeMu.Lock() + defer s.writeMu.Unlock() + tx, err := s.db.Begin() + if err != nil { + return err + } + defer tx.Rollback() + + if err := s.upsertGenericResourceTx(tx, "databases", storageID, data); err != nil { + return err + } + if err := s.upsertDatabasesTx(tx, storageID, obj, data); err != nil { + return err + } + + return tx.Commit() +} + +// upsertGraphTx writes the per-resource domain-table portion of a +// graph upsert inside an existing transaction. The caller is +// responsible for the generic resources insert (via upsertGenericResourceTx) +// and for committing the tx. Splitting this out lets UpsertBatch dispatch +// domain inserts per item without opening a per-item transaction. +func (s *Store) upsertGraphTx(tx *sql.Tx, id string, obj map[string]any, data json.RawMessage) error { + if _, err := tx.Exec( + `INSERT INTO "graph" ("id", "data", "synced_at", "uuid") + VALUES (?, ?, ?, ?) + ON CONFLICT("id") DO UPDATE SET "data" = excluded."data", "synced_at" = excluded."synced_at", "uuid" = excluded."uuid"`, + id, + string(data), + time.Now().UTC().Format(time.RFC3339), + lookupFieldValue(obj, "uuid"), + ); err != nil { + return fmt.Errorf("insert into graph: %w", err) + } + + return nil +} + +// UpsertGraph inserts or updates a graph record with domain-specific columns. +func (s *Store) UpsertGraph(data json.RawMessage) error { + obj, err := DecodeJSONObject(data) + if err != nil { + return fmt.Errorf("unmarshaling graph: %w", err) + } + + id := extractObjectID(obj) + if id == "" { + return fmt.Errorf("missing id for graph") + } + storageID := resourceStorageID("graph", id, obj) + + s.writeMu.Lock() + defer s.writeMu.Unlock() + tx, err := s.db.Begin() + if err != nil { + return err + } + defer tx.Rollback() + + if err := s.upsertGenericResourceTx(tx, "graph", storageID, data); err != nil { + return err + } + if err := s.upsertGraphTx(tx, storageID, obj, data); err != nil { + return err + } + + return tx.Commit() +} + +// upsertGroupsTx writes the per-resource domain-table portion of a +// groups upsert inside an existing transaction. The caller is +// responsible for the generic resources insert (via upsertGenericResourceTx) +// and for committing the tx. Splitting this out lets UpsertBatch dispatch +// domain inserts per item without opening a per-item transaction. +func (s *Store) upsertGroupsTx(tx *sql.Tx, id string, obj map[string]any, data json.RawMessage) error { + if _, err := tx.Exec( + `INSERT INTO "groups" ("id", "data", "synced_at", "uuid", "name", "path", "child_count") + VALUES (?, ?, ?, ?, ?, ?, ?) + ON CONFLICT("id") DO UPDATE SET "data" = excluded."data", "synced_at" = excluded."synced_at", "uuid" = excluded."uuid", "name" = excluded."name", "path" = excluded."path", "child_count" = excluded."child_count"`, + id, + string(data), + time.Now().UTC().Format(time.RFC3339), + lookupFieldValue(obj, "uuid"), + lookupFieldValue(obj, "name"), + lookupFieldValue(obj, "path"), + lookupFieldValue(obj, "child_count"), + ); err != nil { + return fmt.Errorf("insert into groups: %w", err) + } + + return nil +} + +// UpsertGroups inserts or updates a groups record with domain-specific columns. +func (s *Store) UpsertGroups(data json.RawMessage) error { + obj, err := DecodeJSONObject(data) + if err != nil { + return fmt.Errorf("unmarshaling groups: %w", err) + } + + id := extractObjectID(obj) + if id == "" { + return fmt.Errorf("missing id for groups") + } + storageID := resourceStorageID("groups", id, obj) + + s.writeMu.Lock() + defer s.writeMu.Unlock() + tx, err := s.db.Begin() + if err != nil { + return err + } + defer tx.Rollback() + + if err := s.upsertGenericResourceTx(tx, "groups", storageID, data); err != nil { + return err + } + if err := s.upsertGroupsTx(tx, storageID, obj, data); err != nil { + return err + } + + return tx.Commit() +} + +// upsertIngestTx writes the per-resource domain-table portion of a +// ingest upsert inside an existing transaction. The caller is +// responsible for the generic resources insert (via upsertGenericResourceTx) +// and for committing the tx. Splitting this out lets UpsertBatch dispatch +// domain inserts per item without opening a per-item transaction. +func (s *Store) upsertIngestTx(tx *sql.Tx, id string, obj map[string]any, data json.RawMessage) error { + if _, err := tx.Exec( + `INSERT INTO "ingest" ("id", "data", "synced_at", "status", "dry_run", "message", "ledger_id") + VALUES (?, ?, ?, ?, ?, ?, ?) + ON CONFLICT("id") DO UPDATE SET "data" = excluded."data", "synced_at" = excluded."synced_at", "status" = excluded."status", "dry_run" = excluded."dry_run", "message" = excluded."message", "ledger_id" = excluded."ledger_id"`, + id, + string(data), + time.Now().UTC().Format(time.RFC3339), + lookupFieldValue(obj, "status"), + lookupFieldValue(obj, "dry_run"), + lookupFieldValue(obj, "message"), + lookupFieldValue(obj, "ledger_id"), + ); err != nil { + return fmt.Errorf("insert into ingest: %w", err) + } + + return nil +} + +// UpsertIngest inserts or updates a ingest record with domain-specific columns. +func (s *Store) UpsertIngest(data json.RawMessage) error { + obj, err := DecodeJSONObject(data) + if err != nil { + return fmt.Errorf("unmarshaling ingest: %w", err) + } + + id := extractObjectID(obj) + if id == "" { + return fmt.Errorf("missing id for ingest") + } + storageID := resourceStorageID("ingest", id, obj) + + s.writeMu.Lock() + defer s.writeMu.Unlock() + tx, err := s.db.Begin() + if err != nil { + return err + } + defer tx.Rollback() + + if err := s.upsertGenericResourceTx(tx, "ingest", storageID, data); err != nil { + return err + } + if err := s.upsertIngestTx(tx, storageID, obj, data); err != nil { + return err + } + + return tx.Commit() +} + +// upsertLedgerTx writes the per-resource domain-table portion of a +// ledger upsert inside an existing transaction. The caller is +// responsible for the generic resources insert (via upsertGenericResourceTx) +// and for committing the tx. Splitting this out lets UpsertBatch dispatch +// domain inserts per item without opening a per-item transaction. +func (s *Store) upsertLedgerTx(tx *sql.Tx, id string, obj map[string]any, data json.RawMessage) error { + if _, err := tx.Exec( + `INSERT INTO "ledger" ("id", "data", "synced_at", "time", "action", "count", "status") + VALUES (?, ?, ?, ?, ?, ?, ?) + ON CONFLICT("id") DO UPDATE SET "data" = excluded."data", "synced_at" = excluded."synced_at", "time" = excluded."time", "action" = excluded."action", "count" = excluded."count", "status" = excluded."status"`, + id, + string(data), + time.Now().UTC().Format(time.RFC3339), + lookupFieldValue(obj, "time"), + lookupFieldValue(obj, "action"), + lookupFieldValue(obj, "count"), + lookupFieldValue(obj, "status"), + ); err != nil { + return fmt.Errorf("insert into ledger: %w", err) + } + + return nil +} + +// UpsertLedger inserts or updates a ledger record with domain-specific columns. +func (s *Store) UpsertLedger(data json.RawMessage) error { + obj, err := DecodeJSONObject(data) + if err != nil { + return fmt.Errorf("unmarshaling ledger: %w", err) + } + + id := extractObjectID(obj) + if id == "" { + return fmt.Errorf("missing id for ledger") + } + storageID := resourceStorageID("ledger", id, obj) + + s.writeMu.Lock() + defer s.writeMu.Unlock() + tx, err := s.db.Begin() + if err != nil { + return err + } + defer tx.Rollback() + + if err := s.upsertGenericResourceTx(tx, "ledger", storageID, data); err != nil { + return err + } + if err := s.upsertLedgerTx(tx, storageID, obj, data); err != nil { + return err + } + + return tx.Commit() +} + +// upsertMcpTx writes the per-resource domain-table portion of a +// mcp upsert inside an existing transaction. The caller is +// responsible for the generic resources insert (via upsertGenericResourceTx) +// and for committing the tx. Splitting this out lets UpsertBatch dispatch +// domain inserts per item without opening a per-item transaction. +func (s *Store) upsertMcpTx(tx *sql.Tx, id string, obj map[string]any, data json.RawMessage) error { + if _, err := tx.Exec( + `INSERT INTO "mcp" ("id", "data", "synced_at", "text", "truncated", "name", "description") + VALUES (?, ?, ?, ?, ?, ?, ?) + ON CONFLICT("id") DO UPDATE SET "data" = excluded."data", "synced_at" = excluded."synced_at", "text" = excluded."text", "truncated" = excluded."truncated", "name" = excluded."name", "description" = excluded."description"`, + id, + string(data), + time.Now().UTC().Format(time.RFC3339), + lookupFieldValue(obj, "text"), + lookupFieldValue(obj, "truncated"), + lookupFieldValue(obj, "name"), + lookupFieldValue(obj, "description"), + ); err != nil { + return fmt.Errorf("insert into mcp: %w", err) + } + + return nil +} + +// UpsertMcp inserts or updates a mcp record with domain-specific columns. +func (s *Store) UpsertMcp(data json.RawMessage) error { + obj, err := DecodeJSONObject(data) + if err != nil { + return fmt.Errorf("unmarshaling mcp: %w", err) + } + + id := extractObjectID(obj) + if id == "" { + return fmt.Errorf("missing id for mcp") + } + storageID := resourceStorageID("mcp", id, obj) + + s.writeMu.Lock() + defer s.writeMu.Unlock() + tx, err := s.db.Begin() + if err != nil { + return err + } + defer tx.Rollback() + + if err := s.upsertGenericResourceTx(tx, "mcp", storageID, data); err != nil { + return err + } + if err := s.upsertMcpTx(tx, storageID, obj, data); err != nil { + return err + } + + return tx.Commit() +} + +// upsertMediaTx writes the per-resource domain-table portion of a +// media upsert inside an existing transaction. The caller is +// responsible for the generic resources insert (via upsertGenericResourceTx) +// and for committing the tx. Splitting this out lets UpsertBatch dispatch +// domain inserts per item without opening a per-item transaction. +func (s *Store) upsertMediaTx(tx *sql.Tx, id string, obj map[string]any, data json.RawMessage) error { + if _, err := tx.Exec( + `INSERT INTO "media" ("id", "data", "synced_at", "status", "dry_run", "message", "ledger_id", "text", "truncated") + VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?) + ON CONFLICT("id") DO UPDATE SET "data" = excluded."data", "synced_at" = excluded."synced_at", "status" = excluded."status", "dry_run" = excluded."dry_run", "message" = excluded."message", "ledger_id" = excluded."ledger_id", "text" = excluded."text", "truncated" = excluded."truncated"`, + id, + string(data), + time.Now().UTC().Format(time.RFC3339), + lookupFieldValue(obj, "status"), + lookupFieldValue(obj, "dry_run"), + lookupFieldValue(obj, "message"), + lookupFieldValue(obj, "ledger_id"), + lookupFieldValue(obj, "text"), + lookupFieldValue(obj, "truncated"), + ); err != nil { + return fmt.Errorf("insert into media: %w", err) + } + + return nil +} + +// UpsertMedia inserts or updates a media record with domain-specific columns. +func (s *Store) UpsertMedia(data json.RawMessage) error { + obj, err := DecodeJSONObject(data) + if err != nil { + return fmt.Errorf("unmarshaling media: %w", err) + } + + id := extractObjectID(obj) + if id == "" { + return fmt.Errorf("missing id for media") + } + storageID := resourceStorageID("media", id, obj) + + s.writeMu.Lock() + defer s.writeMu.Unlock() + tx, err := s.db.Begin() + if err != nil { + return err + } + defer tx.Rollback() + + if err := s.upsertGenericResourceTx(tx, "media", storageID, data); err != nil { + return err + } + if err := s.upsertMediaTx(tx, storageID, obj, data); err != nil { + return err + } + + return tx.Commit() +} + +// upsertMirrorTx writes the per-resource domain-table portion of a +// mirror upsert inside an existing transaction. The caller is +// responsible for the generic resources insert (via upsertGenericResourceTx) +// and for committing the tx. Splitting this out lets UpsertBatch dispatch +// domain inserts per item without opening a per-item transaction. +func (s *Store) upsertMirrorTx(tx *sql.Tx, id string, obj map[string]any, data json.RawMessage) error { + if _, err := tx.Exec( + `INSERT INTO "mirror" ("id", "data", "synced_at", "uuid", "name", "kind", "path", "location", "database", "item_link", "database_count", "record_count", "mirror_path") + VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) + ON CONFLICT("id") DO UPDATE SET "data" = excluded."data", "synced_at" = excluded."synced_at", "uuid" = excluded."uuid", "name" = excluded."name", "kind" = excluded."kind", "path" = excluded."path", "location" = excluded."location", "database" = excluded."database", "item_link" = excluded."item_link", "database_count" = excluded."database_count", "record_count" = excluded."record_count", "mirror_path" = excluded."mirror_path"`, + id, + string(data), + time.Now().UTC().Format(time.RFC3339), + lookupFieldValue(obj, "uuid"), + lookupFieldValue(obj, "name"), + lookupFieldValue(obj, "kind"), + lookupFieldValue(obj, "path"), + lookupFieldValue(obj, "location"), + lookupFieldValue(obj, "database"), + lookupFieldValue(obj, "item_link"), + lookupFieldValue(obj, "database_count"), + lookupFieldValue(obj, "record_count"), + lookupFieldValue(obj, "mirror_path"), + ); err != nil { + return fmt.Errorf("insert into mirror: %w", err) + } + + return nil +} + +// UpsertMirror inserts or updates a mirror record with domain-specific columns. +func (s *Store) UpsertMirror(data json.RawMessage) error { + obj, err := DecodeJSONObject(data) + if err != nil { + return fmt.Errorf("unmarshaling mirror: %w", err) + } + + id := extractObjectID(obj) + if id == "" { + return fmt.Errorf("missing id for mirror") + } + storageID := resourceStorageID("mirror", id, obj) + + s.writeMu.Lock() + defer s.writeMu.Unlock() + tx, err := s.db.Begin() + if err != nil { + return err + } + defer tx.Rollback() + + if err := s.upsertGenericResourceTx(tx, "mirror", storageID, data); err != nil { + return err + } + if err := s.upsertMirrorTx(tx, storageID, obj, data); err != nil { + return err + } + + return tx.Commit() +} + +// upsertRecordsTx writes the per-resource domain-table portion of a +// records upsert inside an existing transaction. The caller is +// responsible for the generic resources insert (via upsertGenericResourceTx) +// and for committing the tx. Splitting this out lets UpsertBatch dispatch +// domain inserts per item without opening a per-item transaction. +func (s *Store) upsertRecordsTx(tx *sql.Tx, id string, obj map[string]any, data json.RawMessage) error { + if _, err := tx.Exec( + `INSERT INTO "records" ("id", "data", "synced_at", "uuid", "content", "content_format", "truncated", "name", "kind", "path", "location", "database", "item_link", "text") + VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) + ON CONFLICT("id") DO UPDATE SET "data" = excluded."data", "synced_at" = excluded."synced_at", "uuid" = excluded."uuid", "content" = excluded."content", "content_format" = excluded."content_format", "truncated" = excluded."truncated", "name" = excluded."name", "kind" = excluded."kind", "path" = excluded."path", "location" = excluded."location", "database" = excluded."database", "item_link" = excluded."item_link", "text" = excluded."text"`, + id, + string(data), + time.Now().UTC().Format(time.RFC3339), + lookupFieldValue(obj, "uuid"), + lookupFieldValue(obj, "content"), + lookupFieldValue(obj, "content_format"), + lookupFieldValue(obj, "truncated"), + lookupFieldValue(obj, "name"), + lookupFieldValue(obj, "kind"), + lookupFieldValue(obj, "path"), + lookupFieldValue(obj, "location"), + lookupFieldValue(obj, "database"), + lookupFieldValue(obj, "item_link"), + lookupFieldValue(obj, "text"), + ); err != nil { + return fmt.Errorf("insert into records: %w", err) + } + + return nil +} + +// UpsertRecords inserts or updates a records record with domain-specific columns. +func (s *Store) UpsertRecords(data json.RawMessage) error { + obj, err := DecodeJSONObject(data) + if err != nil { + return fmt.Errorf("unmarshaling records: %w", err) + } + + id := extractObjectID(obj) + if id == "" { + return fmt.Errorf("missing id for records") + } + storageID := resourceStorageID("records", id, obj) + + s.writeMu.Lock() + defer s.writeMu.Unlock() + tx, err := s.db.Begin() + if err != nil { + return err + } + defer tx.Rollback() + + if err := s.upsertGenericResourceTx(tx, "records", storageID, data); err != nil { + return err + } + if err := s.upsertRecordsTx(tx, storageID, obj, data); err != nil { + return err + } + + return tx.Commit() +} + +// upsertRuntimeTx writes the per-resource domain-table portion of a +// runtime upsert inside an existing transaction. The caller is +// responsible for the generic resources insert (via upsertGenericResourceTx) +// and for committing the tx. Splitting this out lets UpsertBatch dispatch +// domain inserts per item without opening a per-item transaction. +func (s *Store) upsertRuntimeTx(tx *sql.Tx, id string, obj map[string]any, data json.RawMessage) error { + if _, err := tx.Exec( + `INSERT INTO "runtime" ("id", "data", "synced_at", "running", "version", "applescript_ok", "mcp_url", "mirror_path") + VALUES (?, ?, ?, ?, ?, ?, ?, ?) + ON CONFLICT("id") DO UPDATE SET "data" = excluded."data", "synced_at" = excluded."synced_at", "running" = excluded."running", "version" = excluded."version", "applescript_ok" = excluded."applescript_ok", "mcp_url" = excluded."mcp_url", "mirror_path" = excluded."mirror_path"`, + id, + string(data), + time.Now().UTC().Format(time.RFC3339), + lookupFieldValue(obj, "running"), + lookupFieldValue(obj, "version"), + lookupFieldValue(obj, "applescript_ok"), + lookupFieldValue(obj, "mcp_url"), + lookupFieldValue(obj, "mirror_path"), + ); err != nil { + return fmt.Errorf("insert into runtime: %w", err) + } + + return nil +} + +// UpsertRuntime inserts or updates a runtime record with domain-specific columns. +func (s *Store) UpsertRuntime(data json.RawMessage) error { + obj, err := DecodeJSONObject(data) + if err != nil { + return fmt.Errorf("unmarshaling runtime: %w", err) + } + + id := extractObjectID(obj) + if id == "" { + return fmt.Errorf("missing id for runtime") + } + storageID := resourceStorageID("runtime", id, obj) + + s.writeMu.Lock() + defer s.writeMu.Unlock() + tx, err := s.db.Begin() + if err != nil { + return err + } + defer tx.Rollback() + + if err := s.upsertGenericResourceTx(tx, "runtime", storageID, data); err != nil { + return err + } + if err := s.upsertRuntimeTx(tx, storageID, obj, data); err != nil { + return err + } + + return tx.Commit() +} + +// upsertSelectionTx writes the per-resource domain-table portion of a +// selection upsert inside an existing transaction. The caller is +// responsible for the generic resources insert (via upsertGenericResourceTx) +// and for committing the tx. Splitting this out lets UpsertBatch dispatch +// domain inserts per item without opening a per-item transaction. +func (s *Store) upsertSelectionTx(tx *sql.Tx, id string, obj map[string]any, data json.RawMessage) error { + if _, err := tx.Exec( + `INSERT INTO "selection" ("id", "data", "synced_at", "uuid", "name", "kind", "path", "location", "database", "item_link", "created_at", "note") + VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) + ON CONFLICT("id") DO UPDATE SET "data" = excluded."data", "synced_at" = excluded."synced_at", "uuid" = excluded."uuid", "name" = excluded."name", "kind" = excluded."kind", "path" = excluded."path", "location" = excluded."location", "database" = excluded."database", "item_link" = excluded."item_link", "created_at" = excluded."created_at", "note" = excluded."note"`, + id, + string(data), + time.Now().UTC().Format(time.RFC3339), + lookupFieldValue(obj, "uuid"), + lookupFieldValue(obj, "name"), + lookupFieldValue(obj, "kind"), + lookupFieldValue(obj, "path"), + lookupFieldValue(obj, "location"), + lookupFieldValue(obj, "database"), + lookupFieldValue(obj, "item_link"), + lookupFieldValue(obj, "created_at"), + lookupFieldValue(obj, "note"), + ); err != nil { + return fmt.Errorf("insert into selection: %w", err) + } + + return nil +} + +// UpsertSelection inserts or updates a selection record with domain-specific columns. +func (s *Store) UpsertSelection(data json.RawMessage) error { + obj, err := DecodeJSONObject(data) + if err != nil { + return fmt.Errorf("unmarshaling selection: %w", err) + } + + id := extractObjectID(obj) + if id == "" { + return fmt.Errorf("missing id for selection") + } + storageID := resourceStorageID("selection", id, obj) + + s.writeMu.Lock() + defer s.writeMu.Unlock() + tx, err := s.db.Begin() + if err != nil { + return err + } + defer tx.Rollback() + + if err := s.upsertGenericResourceTx(tx, "selection", storageID, data); err != nil { + return err + } + if err := s.upsertSelectionTx(tx, storageID, obj, data); err != nil { + return err + } + + return tx.Commit() +} + +// upsertTagsTx writes the per-resource domain-table portion of a +// tags upsert inside an existing transaction. The caller is +// responsible for the generic resources insert (via upsertGenericResourceTx) +// and for committing the tx. Splitting this out lets UpsertBatch dispatch +// domain inserts per item without opening a per-item transaction. +func (s *Store) upsertTagsTx(tx *sql.Tx, id string, obj map[string]any, data json.RawMessage) error { + if _, err := tx.Exec( + `INSERT INTO "tags" ("id", "data", "synced_at", "tag_count") + VALUES (?, ?, ?, ?) + ON CONFLICT("id") DO UPDATE SET "data" = excluded."data", "synced_at" = excluded."synced_at", "tag_count" = excluded."tag_count"`, + id, + string(data), + time.Now().UTC().Format(time.RFC3339), + lookupFieldValue(obj, "tag_count"), + ); err != nil { + return fmt.Errorf("insert into tags: %w", err) + } + + return nil +} + +// UpsertTags inserts or updates a tags record with domain-specific columns. +func (s *Store) UpsertTags(data json.RawMessage) error { + obj, err := DecodeJSONObject(data) + if err != nil { + return fmt.Errorf("unmarshaling tags: %w", err) + } + + id := extractObjectID(obj) + if id == "" { + return fmt.Errorf("missing id for tags") + } + storageID := resourceStorageID("tags", id, obj) + + s.writeMu.Lock() + defer s.writeMu.Unlock() + tx, err := s.db.Begin() + if err != nil { + return err + } + defer tx.Rollback() + + if err := s.upsertGenericResourceTx(tx, "tags", storageID, data); err != nil { + return err + } + if err := s.upsertTagsTx(tx, storageID, obj, data); err != nil { + return err + } + + return tx.Commit() +} + +// resourceIDFieldOverrides projects per-resource IDField (set by the profiler +// from x-resource-id or response-schema fallback) into a runtime lookup map. +// UpsertBatch consults this first so the templated path wins over the +// generic fallback list. Empty when no resource declared an override; the +// runtime fallback list still applies. +// +// Includes both flat resources and dependent (parent-child) resources so a +// child path-item annotated with x-resource-id resolves the same as a flat +// path-item. +var resourceIDFieldOverrides = map[string]string{} + +// genericIDFieldFallbacks is the runtime safety net for resources that did +// NOT receive a templated IDField. API-specific names belong in spec +// annotations (x-resource-id), not this list. Order matters: vendor +// identifier names (gid, sid, uid, uuid, guid) take precedence over `name` +// so APIs like Asana (gid) and Twilio (sid) don't fall through to a display +// field and upsert on names — see #1394. +var genericIDFieldFallbacks = []string{"id", "ID", "gid", "sid", "uid", "uuid", "guid", "name", "slug", "key", "code"} + +// resourceParentKeyColumns identifies generated dependent resources whose +// local mirror rows need the parent context in the storage key. Without this, +// many-to-many sub-collections collapse every parent association onto the +// child's bare id and silently keep only the last synced parent. +var resourceParentKeyColumns = map[string]string{} + +// ExtractResourceID resolves the bare resource id field that UpsertBatch +// extracts from a resource item. For dependent resource types, UpsertBatch +// derives the actual storage key by combining this id with the parent value; +// use resourceStorageID if you need the key as it appears in the database. +// Callers that need to gate best-effort writes can use this to avoid passing +// non-entity envelopes into the batch path. +func ExtractResourceID(resourceType string, obj map[string]any) string { + if override, ok := resourceIDFieldOverrides[resourceType]; ok && override != "" { + if v := lookupFieldValue(obj, override); v != nil { + s := ResourceIDString(v) + if s != "" && s != "<nil>" { + return s + } + } + } + for _, key := range genericIDFieldFallbacks { + if v := lookupFieldValue(obj, key); v != nil { + s := ResourceIDString(v) + if s != "" && s != "<nil>" { + return s + } + } + } + if s := suffixIDFieldFallback(resourceType, obj); s != "" { + return s + } + return "" +} + +// suffixIDFieldFallback resolves an id-less resource that keys on its own +// "<name>_code" / "<name>_id" / "<name>_key" / "<name>_slug" field (e.g. the +// "currencies" resource keying on "currency_code" — see #2327). It is scoped to +// the resource's OWN name so a foreign key like account_id/parent_id is never +// promoted to the primary key, and it uses direct map lookups in a fixed suffix +// order so the chosen id is deterministic. +func suffixIDFieldFallback(resourceType string, obj map[string]any) string { + for _, base := range resourceIDBaseNames(resourceType) { + for _, suffix := range []string{"_id", "_code", "_key", "_slug"} { + if v, ok := obj[base+suffix]; ok { + if s := scalarIDString(v); s != "" && s != "<nil>" { + return s + } + } + } + } + return "" +} + +// resourceIDBaseNames returns lowercase candidate singular/plural stems of a +// resource name to build "<base>_id"-style key probes from (e.g. "currencies" +// -> ["currencies","currency"]). OpenAPI-/path-derived names can carry a +// leading verb token ("get-currencies"), so the same probes are also attempted +// on the de-verbed stem. Minimal English depluralization; the raw name is +// always included so already-singular names work too. +func resourceIDBaseNames(resourceType string) []string { + r := strings.ToLower(strings.TrimSpace(resourceType)) + if r == "" { + return nil + } + stems := []string{r} + if d := stripLeadingResourceVerb(r); d != "" && d != r { + stems = append(stems, d) + } + var bases []string + seen := map[string]bool{} + add := func(s string) { + if s != "" && !seen[s] { + seen[s] = true + bases = append(bases, s) + } + } + for _, stem := range stems { + add(stem) + add(depluralizeResourceStem(stem)) + } + return bases +} + +func stripLeadingResourceVerb(r string) string { + for _, verb := range []string{"get", "list", "fetch", "find", "retrieve", "read", "show", "all"} { + for _, sep := range []string{"-", "_"} { + prefix := verb + sep + if strings.HasPrefix(r, prefix) && len(r) > len(prefix) { + return r[len(prefix):] + } + } + } + return "" +} + +func depluralizeResourceStem(r string) string { + switch { + case strings.HasSuffix(r, "ies") && len(r) > 3: + return strings.TrimSuffix(r, "ies") + "y" // currencies -> currency + // Plurals formed by adding "es" to a base ending in s/x/z/ch/sh. The + // double-s "sses" guard (not bare "ses") keeps soft-e plurals — where the + // singular already ends in a silent "e" (cases, databases, licenses, + // purchases) — out of this branch; they fall through to the "-s" case below + // (cases -> case, not cas). Trade-off: a genuine "-es" plural of an s-ending + // singular (buses, statuses) depluralizes imperfectly, but those are rare as + // resource names and this stem only feeds best-effort id-field probing. + case strings.HasSuffix(r, "sses") || strings.HasSuffix(r, "xes") || + strings.HasSuffix(r, "zes") || strings.HasSuffix(r, "ches") || + strings.HasSuffix(r, "shes"): + return strings.TrimSuffix(r, "es") // classes -> class, boxes -> box, dishes -> dish + case strings.HasSuffix(r, "s") && !strings.HasSuffix(r, "ss") && len(r) > 1: + return strings.TrimSuffix(r, "s") // languages -> language, cases -> case + } + return r +} + +func scalarIDString(value any) string { + switch value.(type) { + case string, bool, int, int8, int16, int32, int64, + uint, uint8, uint16, uint32, uint64, + float32, float64, json.Number, []byte: + return ResourceIDString(value) + default: + return "" + } +} + +func resourceStorageID(resourceType, id string, obj map[string]any) string { + parentKey := resourceParentKeyColumns[resourceType] + if parentKey == "" { + return id + } + parentValue := ResourceIDString(lookupFieldValue(obj, parentKey)) + if parentValue == "" || parentValue == "<nil>" { + return id + } + return id + string([]byte{0}) + parentValue +} + +// UpsertBatch inserts or replaces multiple records in a single transaction +// and returns (stored, extractFailures, err). stored counts rows landed in +// the generic resources table; extractFailures counts items that survived +// JSON unmarshal but had no extractable primary key (templated IDField AND +// generic fallback both missed). callers (sync.go.tmpl) compare these +// against len(items) to emit the per-item primary_key_unresolved warning +// and the F4b stored_count_zero_after_extraction probe. +// +// For resource types that have a domain-specific typed table, the per-item +// generic insert is followed by a dispatch to the matching upsert<Pascal>Tx +// inside the same transaction. Without that dispatch, paginated syncs would +// only populate the generic resources table — typed tables (and indexed +// columns like parent_id added by dependent-resource sync) would stay empty. +// +// Each typed-table dispatch runs inside a per-item SAVEPOINT so a constraint +// failure in the typed insert (e.g. NOT NULL parent FK when the generator +// didn't populate the parent path placeholder) rolls back only that typed +// upsert. The generic resources row inserted just above it survives the +// rollback, so successful API fetches never strand in memory because one +// downstream typed table is misconfigured. Failures are surfaced via a +// trailing stderr warning rather than aborting the batch. +func (s *Store) UpsertBatch(resourceType string, items []json.RawMessage) (int, int, error) { + s.writeMu.Lock() + defer s.writeMu.Unlock() + tx, err := s.db.Begin() + if err != nil { + return 0, 0, fmt.Errorf("starting batch transaction: %w", err) + } + defer tx.Rollback() + + var stored, skippedCount, extractFailures, typedFailures int + for i, item := range items { + obj, err := DecodeJSONObject(item) + if err != nil { + skippedCount++ + continue + } + // Templated IDField wins; generic fallback list runs second when + // the override is empty OR the override field is absent on this + // particular item (response shape mismatches happen even when the + // spec declares x-resource-id). + id := ExtractResourceID(resourceType, obj) + if id == "" { + skippedCount++ + extractFailures++ + continue + } + storageID := resourceStorageID(resourceType, id, obj) + + if err := s.upsertGenericResourceTx(tx, resourceType, storageID, item); err != nil { + // Return the running stored count rather than zero so callers + // inspecting partial progress on failure see what already + // landed in earlier loop iterations. + return stored, extractFailures, fmt.Errorf("upserting %s/%s: %w", resourceType, storageID, err) + } + stored++ + + savepoint := fmt.Sprintf("pp_typed_%d", i) + if _, err := tx.Exec("SAVEPOINT " + savepoint); err != nil { + return stored, extractFailures, fmt.Errorf("savepoint begin for %s/%s: %w", resourceType, storageID, err) + } + + var typedErr error + switch resourceType { + case "ai": + typedErr = s.upsertAiTx(tx, storageID, obj, item) + case "databases": + typedErr = s.upsertDatabasesTx(tx, storageID, obj, item) + case "graph": + typedErr = s.upsertGraphTx(tx, storageID, obj, item) + case "groups": + typedErr = s.upsertGroupsTx(tx, storageID, obj, item) + case "ingest": + typedErr = s.upsertIngestTx(tx, storageID, obj, item) + case "ledger": + typedErr = s.upsertLedgerTx(tx, storageID, obj, item) + case "mcp": + typedErr = s.upsertMcpTx(tx, storageID, obj, item) + case "media": + typedErr = s.upsertMediaTx(tx, storageID, obj, item) + case "mirror": + typedErr = s.upsertMirrorTx(tx, storageID, obj, item) + case "records": + typedErr = s.upsertRecordsTx(tx, storageID, obj, item) + case "runtime": + typedErr = s.upsertRuntimeTx(tx, storageID, obj, item) + case "selection": + typedErr = s.upsertSelectionTx(tx, storageID, obj, item) + case "tags": + typedErr = s.upsertTagsTx(tx, storageID, obj, item) + } + + if typedErr != nil { + if _, rbErr := tx.Exec("ROLLBACK TO SAVEPOINT " + savepoint); rbErr != nil { + return stored, extractFailures, fmt.Errorf("rollback to savepoint for %s/%s (typed err: %v): %w", resourceType, storageID, typedErr, rbErr) + } + if _, relErr := tx.Exec("RELEASE SAVEPOINT " + savepoint); relErr != nil { + return stored, extractFailures, fmt.Errorf("release savepoint after rollback for %s/%s: %w", resourceType, storageID, relErr) + } + typedFailures++ + continue + } + if _, err := tx.Exec("RELEASE SAVEPOINT " + savepoint); err != nil { + return stored, extractFailures, fmt.Errorf("release savepoint for %s/%s: %w", resourceType, storageID, err) + } + } + + // Warn when most items in a batch lack an extractable ID — this likely + // means the API uses a primary key field we don't recognize yet. + if skippedCount > 0 && len(items) > 0 && skippedCount*2 > len(items) { + fmt.Fprintf(os.Stderr, "warning: %d/%d %s items returned but not cached locally (no extractable ID field; offline lookup against these rows will be incomplete; live queries unaffected)\n", skippedCount, len(items), resourceType) + } + // Surface typed-table failures without aborting the batch. Generic rows + // already committed; only the typed projection failed. + if typedFailures > 0 { + fmt.Fprintf(os.Stderr, "warning: %d/%d %s items: typed-table upsert failed; generic resources rows preserved\n", typedFailures, len(items), resourceType) + } + + if err := tx.Commit(); err != nil { + return 0, extractFailures, err + } + return stored, extractFailures, nil +} + +// SearchMcp searches the mcp_fts index with optional filters. +func (s *Store) SearchMcp(query string, limit int) ([]json.RawMessage, error) { + if limit <= 0 { + limit = 50 + } + matchQuery := ftsMatchQuery(query) + if matchQuery == "" { + return nil, nil + } + rows, err := s.db.Query( + `SELECT t.data FROM "mcp" t + JOIN "mcp_fts" ON "mcp_fts".rowid = t.rowid + WHERE "mcp_fts" MATCH ? + ORDER BY rank LIMIT ?`, + matchQuery, limit, + ) + if err != nil { + return nil, err + } + defer rows.Close() + + var results []json.RawMessage + for rows.Next() { + var data string + if err := rows.Scan(&data); err != nil { + return nil, err + } + results = append(results, json.RawMessage(data)) + } + return results, rows.Err() +} + +// SearchMedia searches the media_fts index with optional filters. +func (s *Store) SearchMedia(query string, limit int) ([]json.RawMessage, error) { + if limit <= 0 { + limit = 50 + } + matchQuery := ftsMatchQuery(query) + if matchQuery == "" { + return nil, nil + } + rows, err := s.db.Query( + `SELECT t.data FROM "media" t + JOIN "media_fts" ON "media_fts".rowid = t.rowid + WHERE "media_fts" MATCH ? + ORDER BY rank LIMIT ?`, + matchQuery, limit, + ) + if err != nil { + return nil, err + } + defer rows.Close() + + var results []json.RawMessage + for rows.Next() { + var data string + if err := rows.Scan(&data); err != nil { + return nil, err + } + results = append(results, json.RawMessage(data)) + } + return results, rows.Err() +} + +// SearchRecords searches the records_fts index with optional filters. +func (s *Store) SearchRecords(query string, limit int) ([]json.RawMessage, error) { + if limit <= 0 { + limit = 50 + } + matchQuery := ftsMatchQuery(query) + if matchQuery == "" { + return nil, nil + } + rows, err := s.db.Query( + `SELECT t.data FROM "records" t + JOIN "records_fts" ON "records_fts".rowid = t.rowid + WHERE "records_fts" MATCH ? + ORDER BY rank LIMIT ?`, + matchQuery, limit, + ) + if err != nil { + return nil, err + } + defer rows.Close() + + var results []json.RawMessage + for rows.Next() { + var data string + if err := rows.Scan(&data); err != nil { + return nil, err + } + results = append(results, json.RawMessage(data)) + } + return results, rows.Err() +} + +// SearchSelection searches the selection_fts index with optional filters. +func (s *Store) SearchSelection(query string, limit int) ([]json.RawMessage, error) { + if limit <= 0 { + limit = 50 + } + matchQuery := ftsMatchQuery(query) + if matchQuery == "" { + return nil, nil + } + rows, err := s.db.Query( + `SELECT t.data FROM "selection" t + JOIN "selection_fts" ON "selection_fts".rowid = t.rowid + WHERE "selection_fts" MATCH ? + ORDER BY rank LIMIT ?`, + matchQuery, limit, + ) + if err != nil { + return nil, err + } + defer rows.Close() + + var results []json.RawMessage + for rows.Next() { + var data string + if err := rows.Scan(&data); err != nil { + return nil, err + } + results = append(results, json.RawMessage(data)) + } + return results, rows.Err() +} + +func (s *Store) SaveSyncState(resourceType, cursor string, count int) error { + s.writeMu.Lock() + defer s.writeMu.Unlock() + _, err := s.db.Exec( + `INSERT INTO sync_state (resource_type, last_cursor, last_synced_at, total_count) + VALUES (?, ?, ?, ?) + ON CONFLICT(resource_type) DO UPDATE SET last_cursor = excluded.last_cursor, + last_synced_at = excluded.last_synced_at, total_count = excluded.total_count`, + resourceType, cursor, time.Now().UTC().Format(time.RFC3339), count, + ) + return err +} + +func (s *Store) GetSyncState(resourceType string) (cursor string, lastSynced time.Time, count int, err error) { + err = s.db.QueryRow( + `SELECT last_cursor, last_synced_at, total_count FROM sync_state WHERE resource_type = ?`, + resourceType, + ).Scan(&cursor, &lastSynced, &count) + if err == sql.ErrNoRows { + return "", time.Time{}, 0, nil + } + return +} + +// SaveSyncCursor stores the pagination cursor for a resource type. +func (s *Store) SaveSyncCursor(resourceType, cursor string) error { + s.writeMu.Lock() + defer s.writeMu.Unlock() + _, err := s.db.Exec( + `INSERT INTO sync_state (resource_type, last_cursor, last_synced_at, total_count) + VALUES (?, ?, CURRENT_TIMESTAMP, 0) + ON CONFLICT(resource_type) DO UPDATE SET last_cursor = ?, last_synced_at = CURRENT_TIMESTAMP`, + resourceType, cursor, cursor, + ) + return err +} + +// GetSyncCursor returns the last pagination cursor for a resource type. +func (s *Store) GetSyncCursor(resourceType string) string { + var cursor sql.NullString + if err := s.db.QueryRow("SELECT last_cursor FROM sync_state WHERE resource_type = ?", resourceType).Scan(&cursor); err != nil { + return "" + } + if cursor.Valid { + return cursor.String + } + return "" +} + +// ListIDs returns all IDs from a resource's domain table, or from the generic +// resources table if no domain table exists. Used by dependent sync to iterate parents. +// +// resourceType is never interpolated into SQL directly. We resolve it to a real +// table name via a parameterized sqlite_master lookup; only that trusted name is +// substituted (double-quoted) into the SELECT. Callers may pass any string. +func (s *Store) ListIDs(resourceType string) ([]string, error) { + var table string + err := s.db.QueryRow( + `SELECT name FROM sqlite_master WHERE type='table' AND name=?`, + resourceType, + ).Scan(&table) + var rows *sql.Rows + if err == nil && table != "" { + rows, err = s.db.Query(fmt.Sprintf(`SELECT id FROM "%s"`, strings.ReplaceAll(table, `"`, `""`))) + } + if err != nil || table == "" { + // Fall back to generic resources table + rows, err = s.db.Query("SELECT id FROM resources WHERE resource_type = ?", resourceType) + if err != nil { + return nil, err + } + } + defer rows.Close() + + var ids []string + for rows.Next() { + var id string + if err := rows.Scan(&id); err != nil { + continue + } + ids = append(ids, id) + } + return ids, rows.Err() +} + +// ListField returns values of a named field from a resource's domain table, +// or from the generic resources table via json_extract when no typed column +// exists. Used by dependent sync to iterate parents when a spec-declared +// walker extracts a non-PK field (Endpoint.Walker.KeyField in the upstream +// printing-press repo) for the child path's placeholder. +// +// Defense in depth: field is validated against validIdentifierRE at entry +// — the regex pins it to SQL-safe identifier shape covering both the +// typed-column primary path AND the json_extract fallback (where +// pragma_table_info validation would never run if the parent's domain +// table doesn't exist yet). resourceType is never interpolated into SQL +// directly; we resolve it to a real table name via a parameterized +// sqlite_master lookup. Only validated names are substituted +// (double-quoted) into the SELECT. Mirrors ListIDs's defense pattern so +// callers may pass any string. +func (s *Store) ListField(resourceType, field string) ([]string, error) { + if !validIdentifierRE.MatchString(field) { + return nil, fmt.Errorf("ListField: invalid field name %q (must match %s)", field, validIdentifierRE.String()) + } + var table string + err := s.db.QueryRow( + `SELECT name FROM sqlite_master WHERE type='table' AND name=?`, + resourceType, + ).Scan(&table) + var rows *sql.Rows + if err == nil && table != "" { + // Validate the column exists on the resolved table before splicing + // it into the SELECT. pragma_table_info is parameterizable. + var colName string + colErr := s.db.QueryRow( + `SELECT name FROM pragma_table_info(?) WHERE name=?`, + table, field, + ).Scan(&colName) + if colErr == nil && colName != "" { + qTable := strings.ReplaceAll(table, `"`, `""`) + qCol := strings.ReplaceAll(colName, `"`, `""`) + // DISTINCT: callers iterate the returned values as parent keys + // for child-resource fan-out. Multiple parent rows sharing a + // key_field value (legal for non-PK fields) would otherwise + // cause the child endpoint to be fetched once per duplicate row. + rows, err = s.db.Query(fmt.Sprintf( + `SELECT DISTINCT "%s" FROM "%s" WHERE "%s" IS NOT NULL AND "%s" != ''`, + qCol, qTable, qCol, qCol, + )) + } else { + err = colErr + } + } + if err != nil || rows == nil { + // Fall back to generic resources table via json_extract. Path is + // Sprintf'd into the SQL string (matches ResolveByName below). + // DISTINCT for the same reason as the typed-column path above. + fallback := fmt.Sprintf( // #nosec G201 -- field is validated against validIdentifierRE before interpolation. + `SELECT DISTINCT json_extract(data, '$.%s') FROM resources WHERE resource_type = ? AND json_extract(data, '$.%s') IS NOT NULL`, + field, field, + ) + rows, err = s.db.Query(fallback, resourceType) + if err != nil { + return nil, err + } + } + defer rows.Close() + + var values []string + for rows.Next() { + var v sql.NullString + if err := rows.Scan(&v); err == nil && v.Valid && v.String != "" { + values = append(values, v.String) + } + } + return values, rows.Err() +} + +// ListFieldSets returns row-correlated values from the generic resources +// table. Dependent sync uses this for multi-placeholder paths where values +// such as owner/repo or server/webapp must stay paired per parent row. +func (s *Store) ListFieldSets(resourceType string, fields []string) ([]map[string]string, error) { + if len(fields) == 0 { + return nil, nil + } + for _, field := range fields { + if !validIdentifierRE.MatchString(field) { + return nil, fmt.Errorf("ListFieldSets: invalid field name %q (must match %s)", field, validIdentifierRE.String()) + } + } + + rows, err := s.db.Query(`SELECT id, data FROM resources WHERE resource_type = ?`, resourceType) + if err != nil { + return nil, err + } + defer rows.Close() + + var out []map[string]string + seenRows := map[string]bool{} + for rows.Next() { + var id string + var data []byte + if err := rows.Scan(&id, &data); err != nil { + return nil, err + } + var obj map[string]any + if len(data) > 0 { + var err error + obj, err = DecodeJSONObject(data) + if err != nil { + return nil, fmt.Errorf("decode %s parent row %s: %w", resourceType, id, err) + } + } + values := make(map[string]string, len(fields)) + complete := true + for _, field := range fields { + var value any + if field == "id" { + value = id + } else { + value = LookupFieldValue(obj, field) + } + valueString := ResourceIDString(value) + if value == nil || valueString == "" { + complete = false + break + } + values[field] = valueString + } + if complete { + keyParts := make([]string, 0, len(fields)) + for _, field := range fields { + keyParts = append(keyParts, values[field]) + } + key := strings.Join(keyParts, "\x00") + if seenRows[key] { + continue + } + seenRows[key] = true + out = append(out, values) + } + } + return out, rows.Err() +} + +// GetLastSyncedAt returns the last sync timestamp for a resource type. +func (s *Store) GetLastSyncedAt(resourceType string) string { + var ts sql.NullString + if err := s.db.QueryRow("SELECT last_synced_at FROM sync_state WHERE resource_type = ?", resourceType).Scan(&ts); err != nil { + return "" + } + if ts.Valid { + return ts.String + } + return "" +} + +// ClearSyncCursors resets all sync state for a full resync. +func (s *Store) ClearSyncCursors() error { + s.writeMu.Lock() + defer s.writeMu.Unlock() + _, err := s.db.Exec("DELETE FROM sync_state") + return err +} + +// Query executes a raw SQL query and returns the rows. +// Used by workflow commands that need custom queries against the local store. +func (s *Store) Query(query string, args ...any) (*sql.Rows, error) { + return s.db.Query(query, args...) +} + +func (s *Store) Count(resourceType string) (int, error) { + var count int + err := s.db.QueryRow( + `SELECT COUNT(*) FROM resources WHERE resource_type = ?`, + resourceType, + ).Scan(&count) + return count, err +} + +func (s *Store) Status() (map[string]int, error) { + rows, err := s.db.Query( + `SELECT resource_type, COUNT(*) FROM resources GROUP BY resource_type ORDER BY resource_type`, + ) + if err != nil { + return nil, err + } + defer rows.Close() + + status := make(map[string]int) + for rows.Next() { + var rt string + var count int + if err := rows.Scan(&rt, &count); err != nil { + return nil, err + } + status[rt] = count + } + return status, rows.Err() +} + +// ResolveByName resolves a human-readable name to a UUID from synced data. +// If the input is already a UUID, it is returned as-is. +// matchFields are JSON field names to search against (e.g., "name", "key", "email"). +// +// json_extract path components cannot be bound as SQL parameters, so each +// field is validated against validIdentifierRE before being spliced into +// the query. +func (s *Store) ResolveByName(resourceType string, input string, matchFields ...string) (string, error) { + if IsUUID(input) { + return input, nil + } + + var matches []string + for _, field := range matchFields { + if !validIdentifierRE.MatchString(field) { + continue + } + query := fmt.Sprintf( // #nosec G201 -- field is validated against validIdentifierRE before interpolation. + `SELECT id FROM resources WHERE resource_type = ? AND LOWER(json_extract(data, '$.%s')) = LOWER(?)`, + field, + ) + rows, err := s.db.Query(query, resourceType, input) + if err != nil { + return "", err + } + for rows.Next() { + var id string + if rows.Scan(&id) == nil { + // Deduplicate + found := false + for _, m := range matches { + if m == id { + found = true + break + } + } + if !found { + matches = append(matches, id) + } + } + } + if err := rows.Err(); err != nil { + _ = rows.Close() + return "", err + } + if err := rows.Close(); err != nil { + return "", err + } + } + + switch len(matches) { + case 0: + return "", fmt.Errorf("%s %q not found in local store. Run 'sync' first, or use the UUID directly", resourceType, input) + case 1: + return matches[0], nil + default: + hint := matches[0] + if len(matches) > 5 { + hint = strings.Join(matches[:5], ", ") + "..." + } else { + hint = strings.Join(matches, ", ") + } + return "", fmt.Errorf("ambiguous: %q matches %d %s entries (%s). Use the exact UUID instead", input, len(matches), resourceType, hint) + } +} diff --git a/library/productivity/devonthink/internal/store/upsert_batch_test.go b/library/productivity/devonthink/internal/store/upsert_batch_test.go new file mode 100644 index 0000000000..fa491a2af1 --- /dev/null +++ b/library/productivity/devonthink/internal/store/upsert_batch_test.go @@ -0,0 +1,1042 @@ +// Copyright 2026 rowdy and contributors. Licensed under Apache-2.0. See LICENSE. +// Generated by CLI Printing Press (https://github.com/mvanhorn/cli-printing-press). DO NOT EDIT. + +package store + +import ( + "encoding/json" + "fmt" + "path/filepath" + "reflect" + "strings" + "sync" + "testing" + + _ "modernc.org/sqlite" +) + +// TestStoreWrite_NoSQLITE_BUSY_HighConcurrency exercises the writeMu serialization +// guarantee: 16 fetcher-style goroutines hammer the store with a mix of +// UpsertBatch, SaveSyncState, and SaveSyncCursor calls. Before the mutex +// fix, this test reproduces SQLITE_BUSY at default sync concurrency on +// pure-Go SQLite (modernc.org/sqlite + WAL) because multiple writers +// race for the WAL lock and busy_timeout retries are not exhaustive. +// +// Run under `go test -race` to catch any data races on Store fields. +func TestStoreWrite_NoSQLITE_BUSY_HighConcurrency(t *testing.T) { + dbPath := filepath.Join(t.TempDir(), "data.db") + s, err := Open(dbPath) + if err != nil { + t.Fatalf("open: %v", err) + } + defer s.Close() + + const goroutines = 16 + const itemsPerBatch = 5 + + var wg sync.WaitGroup + errCh := make(chan error, goroutines*3) + + for g := 0; g < goroutines; g++ { + wg.Add(1) + go func(gid int) { + defer wg.Done() + rt := fmt.Sprintf("rt_%d", gid) + items := make([]json.RawMessage, 0, itemsPerBatch) + for i := 0; i < itemsPerBatch; i++ { + items = append(items, json.RawMessage(fmt.Sprintf(`{"id": "g%d-i%d"}`, gid, i))) + } + if _, _, err := s.UpsertBatch(rt, items); err != nil { + errCh <- fmt.Errorf("UpsertBatch goroutine %d: %w", gid, err) + return + } + if err := s.SaveSyncState(rt, fmt.Sprintf("cursor-%d", gid), itemsPerBatch); err != nil { + errCh <- fmt.Errorf("SaveSyncState goroutine %d: %w", gid, err) + return + } + if err := s.SaveSyncCursor(rt, fmt.Sprintf("cursor2-%d", gid)); err != nil { + errCh <- fmt.Errorf("SaveSyncCursor goroutine %d: %w", gid, err) + return + } + }(g) + } + wg.Wait() + close(errCh) + + for err := range errCh { + if err == nil { + continue + } + // SQLITE_BUSY surfaces as "database is locked" or "SQLITE_BUSY" + // in the error message — assert neither occurs. + msg := err.Error() + if strings.Contains(msg, "SQLITE_BUSY") || strings.Contains(strings.ToLower(msg), "database is locked") { + t.Fatalf("got SQLITE_BUSY-class error under concurrent writers: %v", err) + } + t.Fatalf("unexpected error under concurrent writers: %v", err) + } + + // Verify all rows persisted: goroutines * itemsPerBatch in the generic + // resources table. + db := s.DB() + var total int + if err := db.QueryRow(`SELECT COUNT(*) FROM resources`).Scan(&total); err != nil { + t.Fatalf("count resources: %v", err) + } + if total != goroutines*itemsPerBatch { + t.Fatalf("resources total = %d, want %d", total, goroutines*itemsPerBatch) + } +} + +// TestStoreWrite_PanicReleasesLock confirms that a panic inside a locked +// section unwinds via defer s.writeMu.Unlock() so subsequent writers can +// proceed. A leaked lock would deadlock the second call indefinitely. +func TestStoreWrite_PanicReleasesLock(t *testing.T) { + dbPath := filepath.Join(t.TempDir(), "data.db") + s, err := Open(dbPath) + if err != nil { + t.Fatalf("open: %v", err) + } + defer s.Close() + + // Trigger panic by passing a nil *Store method receiver indirectly: + // we call UpsertBatch with malformed JSON that survives Unmarshal + // (it's wrapped in skipped-count handling) — there's no easy panic + // path inside a locked section that doesn't also corrupt state, so + // we instead simulate the post-panic state by manually locking and + // unlocking, then assert subsequent calls succeed. + func() { + defer func() { + recover() + }() + s.writeMu.Lock() + defer s.writeMu.Unlock() + panic("simulated writer panic") + }() + + // Subsequent writer must not block. + done := make(chan struct{}) + go func() { + if _, _, err := s.UpsertBatch("post_panic", []json.RawMessage{json.RawMessage(`{"id": "x"}`)}); err != nil { + t.Errorf("post-panic UpsertBatch: %v", err) + } + close(done) + }() + <-done +} + +// TestUpsertBatch_TemplatedIDFieldOverrideWins exercises the +// per-resource ID-field override. When the spec author annotates a +// path-item with x-resource-id, the profiler emits SyncableResource.IDField, +// the generator templates this into resourceIDFieldOverrides, and +// UpsertBatch consults that map first. This test seeds the override map +// at runtime (since the generated table here may or may not declare any +// override) to assert the lookup path itself works. +func TestUpsertBatch_TemplatedIDFieldOverrideWins(t *testing.T) { + dbPath := filepath.Join(t.TempDir(), "data.db") + s, err := Open(dbPath) + if err != nil { + t.Fatalf("open: %v", err) + } + defer s.Close() + + // Inject a runtime override for a synthetic resource. Item carries + // no generic-fallback field (no id/name/uuid/...) — only a custom + // "ticker" field. Without the override, all 3 items would be + // dropped as PK-unresolved; with it, all 3 land. + prev, hadPrev := resourceIDFieldOverrides["overrideTest"] + resourceIDFieldOverrides["overrideTest"] = "ticker" + defer func() { + if hadPrev { + resourceIDFieldOverrides["overrideTest"] = prev + } else { + delete(resourceIDFieldOverrides, "overrideTest") + } + }() + + items := []json.RawMessage{ + json.RawMessage(`{"ticker": "AAPL", "price": 100}`), + json.RawMessage(`{"ticker": "GOOG", "price": 200}`), + json.RawMessage(`{"ticker": "MSFT", "price": 300}`), + } + stored, extractFailures, err := s.UpsertBatch("overrideTest", items) + if err != nil { + t.Fatalf("UpsertBatch: %v", err) + } + if stored != 3 { + t.Fatalf("stored = %d, want 3 (templated override should resolve all PKs)", stored) + } + if extractFailures != 0 { + t.Fatalf("extractFailures = %d, want 0", extractFailures) + } +} + +// TestUpsertBatch_GenericFallbackList covers each name in the reduced +// fallback list. The kalshi-accreted names (ticker/event_ticker/series_ticker) +// were dropped because the user owns kalshi and will regenerate +// it with x-resource-id annotations; this test pins what the generic list +// is now responsible for so a future trim doesn't silently break unannotated +// specs. +func TestUpsertBatch_GenericFallbackList(t *testing.T) { + dbPath := filepath.Join(t.TempDir(), "data.db") + s, err := Open(dbPath) + if err != nil { + t.Fatalf("open: %v", err) + } + defer s.Close() + + for _, key := range []string{"id", "ID", "gid", "sid", "uid", "uuid", "guid", "name", "slug", "key", "code"} { + t.Run(key, func(t *testing.T) { + rt := "fallback_" + key + items := []json.RawMessage{ + json.RawMessage(fmt.Sprintf(`{%q: %q}`, key, "value-1")), + json.RawMessage(fmt.Sprintf(`{%q: %q}`, key, "value-2")), + } + stored, extractFailures, err := s.UpsertBatch(rt, items) + if err != nil { + t.Fatalf("UpsertBatch(%q): %v", key, err) + } + if stored != 2 { + t.Fatalf("stored = %d, want 2 (fallback %q must resolve)", stored, key) + } + if extractFailures != 0 { + t.Fatalf("extractFailures = %d, want 0", extractFailures) + } + }) + } + + // Negative: API-specific names dropped must NOT resolve. + // Spec authors annotate these via x-resource-id instead. + for _, key := range []string{"ticker", "event_ticker", "series_ticker"} { + t.Run("dropped_"+key, func(t *testing.T) { + rt := "dropped_" + key + items := []json.RawMessage{ + json.RawMessage(fmt.Sprintf(`{%q: %q}`, key, "v1")), + } + stored, extractFailures, err := s.UpsertBatch(rt, items) + if err != nil { + t.Fatalf("UpsertBatch(%q): %v", key, err) + } + if stored != 0 { + t.Fatalf("stored = %d, want 0 (%q must NOT be in the generic fallback list)", stored, key) + } + if extractFailures != 1 { + t.Fatalf("extractFailures = %d, want 1 (%q drop must surface as extract failure)", extractFailures, key) + } + }) + } +} + +func TestUpsertBatch_PreservesLargeIntegerResourceIDs(t *testing.T) { + dbPath := filepath.Join(t.TempDir(), "data.db") + s, err := Open(dbPath) + if err != nil { + t.Fatalf("open: %v", err) + } + defer s.Close() + + items := []json.RawMessage{ + json.RawMessage(`{"id": 55043301, "name": "large"}`), + json.RawMessage(`{"id": 100, "name": "small"}`), + json.RawMessage(`{"id": 7, "name": "tiny"}`), + } + stored, extractFailures, err := s.UpsertBatch("numeric_ids", items) + if err != nil { + t.Fatalf("UpsertBatch: %v", err) + } + if stored != len(items) { + t.Fatalf("stored = %d, want %d", stored, len(items)) + } + if extractFailures != 0 { + t.Fatalf("extractFailures = %d, want 0", extractFailures) + } + + rows, err := s.DB().Query(`SELECT id FROM resources WHERE resource_type = ? ORDER BY CAST(id AS INTEGER)`, "numeric_ids") + if err != nil { + t.Fatalf("query resources: %v", err) + } + defer rows.Close() + + var got []string + for rows.Next() { + var id string + if err := rows.Scan(&id); err != nil { + t.Fatalf("scan id: %v", err) + } + got = append(got, id) + } + if err := rows.Err(); err != nil { + t.Fatalf("rows: %v", err) + } + want := []string{"7", "100", "55043301"} + if !reflect.DeepEqual(got, want) { + t.Fatalf("resource ids = %v, want %v", got, want) + } + + var literalMatches int + if err := s.DB().QueryRow( + `SELECT COUNT(*) FROM resources WHERE resource_type = ? AND id IN ('55043301', '100', '7')`, + "numeric_ids", + ).Scan(&literalMatches); err != nil { + t.Fatalf("count literal id matches: %v", err) + } + if literalMatches != len(items) { + t.Fatalf("literal id matches = %d, want %d", literalMatches, len(items)) + } +} + +// TestUpsertBatch_ExtractFailuresReturnedForPerItemMisses pins the third +// return value: items that survive JSON unmarshal but have no extractable +// PK (templated override AND generic fallback both miss) bump +// extractFailures. The sync.go.tmpl call site uses this to emit the +// per-resource primary_key_unresolved sync_anomaly the first time silent +// drops occur. +func TestUpsertBatch_ExtractFailuresReturnedForPerItemMisses(t *testing.T) { + dbPath := filepath.Join(t.TempDir(), "data.db") + s, err := Open(dbPath) + if err != nil { + t.Fatalf("open: %v", err) + } + defer s.Close() + + items := []json.RawMessage{ + json.RawMessage(`{"id": "ok-1"}`), + json.RawMessage(`{"some_random_field": "no-pk-here"}`), + json.RawMessage(`{"id": "ok-2"}`), + json.RawMessage(`{"another_field": 42}`), + } + stored, extractFailures, err := s.UpsertBatch("mixed_extraction", items) + if err != nil { + t.Fatalf("UpsertBatch: %v", err) + } + if stored != 2 { + t.Fatalf("stored = %d, want 2 (only items with id should land)", stored) + } + if extractFailures != 2 { + t.Fatalf("extractFailures = %d, want 2 (two items have no extractable PK)", extractFailures) + } +} + +func TestSearchQuotesFTSQuerySyntax(t *testing.T) { + dbPath := filepath.Join(t.TempDir(), "data.db") + s, err := Open(dbPath) + if err != nil { + t.Fatalf("open: %v", err) + } + defer s.Close() + + items := []json.RawMessage{ + json.RawMessage(`{"id": "ip", "value": "10.0.0.1"}`), + json.RawMessage(`{"id": "cidr", "value": "172.16.192.0/18"}`), + json.RawMessage(`{"id": "host", "value": "host.example.com"}`), + json.RawMessage(`{"id": "email", "value": "user@example.com"}`), + json.RawMessage(`{"id": "mac", "value": "aa:bb:cc:dd:ee:ff"}`), + json.RawMessage(`{"id": "hyphen", "value": "some-name"}`), + json.RawMessage(`{"id": "multi", "value": "error with extra words before timeout"}`), + } + if stored, failed, err := s.UpsertBatch("search-regression", items); err != nil { + t.Fatalf("UpsertBatch: %v", err) + } else if failed != 0 || stored != len(items) { + t.Fatalf("UpsertBatch stored=%d failed=%d, want stored=%d failed=0", stored, failed, len(items)) + } + + for _, query := range []string{ + "10.0.0.1", + "172.16.192.0/18", + "host.example.com", + "user@example.com", + "aa:bb:cc:dd:ee:ff", + "some-name", + "error timeout", + } { + results, err := s.Search(query, 10) + if err != nil { + t.Fatalf("Search(%q): %v", query, err) + } + if len(results) == 0 { + t.Fatalf("Search(%q) returned no results", query) + } + } +} + +// TestUpsertBatch_PopulatesAiTable verifies that UpsertBatch +// dispatches paginated items into both the generic resources table AND the +// typed ai table. Regression for issue #268: before the fix, paginated +// syncs only filled the generic resources table, so domain commands that +// query the typed table saw zero rows. +func TestUpsertBatch_PopulatesAiTable(t *testing.T) { + dbPath := filepath.Join(t.TempDir(), "data.db") + s, err := Open(dbPath) + if err != nil { + t.Fatalf("open: %v", err) + } + defer s.Close() + + items := []json.RawMessage{ + json.RawMessage(`{"id": "test-001"}`), + json.RawMessage(`{"id": "test-002"}`), + json.RawMessage(`{"id": "test-003"}`), + } + if _, _, err := s.UpsertBatch("ai", items); err != nil { + t.Fatalf("UpsertBatch: %v", err) + } + + db := s.DB() + + var generic int + if err := db.QueryRow(`SELECT COUNT(*) FROM resources WHERE resource_type = ?`, "ai").Scan(&generic); err != nil { + t.Fatalf("count resources: %v", err) + } + if generic != len(items) { + t.Fatalf("resources count = %d, want %d", generic, len(items)) + } + + var typed int + typedQuery := fmt.Sprintf(`SELECT COUNT(*) FROM "%s"`, "ai") + if err := db.QueryRow(typedQuery).Scan(&typed); err != nil { + t.Fatalf("count ai: %v", err) + } + if typed != len(items) { + t.Fatalf("ai count = %d, want %d (typed table not populated by UpsertBatch)", typed, len(items)) + } +} + +// TestUpsertBatch_PopulatesDatabasesTable verifies that UpsertBatch +// dispatches paginated items into both the generic resources table AND the +// typed databases table. Regression for issue #268: before the fix, paginated +// syncs only filled the generic resources table, so domain commands that +// query the typed table saw zero rows. +func TestUpsertBatch_PopulatesDatabasesTable(t *testing.T) { + dbPath := filepath.Join(t.TempDir(), "data.db") + s, err := Open(dbPath) + if err != nil { + t.Fatalf("open: %v", err) + } + defer s.Close() + + items := []json.RawMessage{ + json.RawMessage(`{"id": "test-001"}`), + json.RawMessage(`{"id": "test-002"}`), + json.RawMessage(`{"id": "test-003"}`), + } + if _, _, err := s.UpsertBatch("databases", items); err != nil { + t.Fatalf("UpsertBatch: %v", err) + } + + db := s.DB() + + var generic int + if err := db.QueryRow(`SELECT COUNT(*) FROM resources WHERE resource_type = ?`, "databases").Scan(&generic); err != nil { + t.Fatalf("count resources: %v", err) + } + if generic != len(items) { + t.Fatalf("resources count = %d, want %d", generic, len(items)) + } + + var typed int + typedQuery := fmt.Sprintf(`SELECT COUNT(*) FROM "%s"`, "databases") + if err := db.QueryRow(typedQuery).Scan(&typed); err != nil { + t.Fatalf("count databases: %v", err) + } + if typed != len(items) { + t.Fatalf("databases count = %d, want %d (typed table not populated by UpsertBatch)", typed, len(items)) + } +} + +// TestUpsertBatch_PopulatesGraphTable verifies that UpsertBatch +// dispatches paginated items into both the generic resources table AND the +// typed graph table. Regression for issue #268: before the fix, paginated +// syncs only filled the generic resources table, so domain commands that +// query the typed table saw zero rows. +func TestUpsertBatch_PopulatesGraphTable(t *testing.T) { + dbPath := filepath.Join(t.TempDir(), "data.db") + s, err := Open(dbPath) + if err != nil { + t.Fatalf("open: %v", err) + } + defer s.Close() + + items := []json.RawMessage{ + json.RawMessage(`{"id": "test-001"}`), + json.RawMessage(`{"id": "test-002"}`), + json.RawMessage(`{"id": "test-003"}`), + } + if _, _, err := s.UpsertBatch("graph", items); err != nil { + t.Fatalf("UpsertBatch: %v", err) + } + + db := s.DB() + + var generic int + if err := db.QueryRow(`SELECT COUNT(*) FROM resources WHERE resource_type = ?`, "graph").Scan(&generic); err != nil { + t.Fatalf("count resources: %v", err) + } + if generic != len(items) { + t.Fatalf("resources count = %d, want %d", generic, len(items)) + } + + var typed int + typedQuery := fmt.Sprintf(`SELECT COUNT(*) FROM "%s"`, "graph") + if err := db.QueryRow(typedQuery).Scan(&typed); err != nil { + t.Fatalf("count graph: %v", err) + } + if typed != len(items) { + t.Fatalf("graph count = %d, want %d (typed table not populated by UpsertBatch)", typed, len(items)) + } +} + +// TestUpsertBatch_PopulatesGroupsTable verifies that UpsertBatch +// dispatches paginated items into both the generic resources table AND the +// typed groups table. Regression for issue #268: before the fix, paginated +// syncs only filled the generic resources table, so domain commands that +// query the typed table saw zero rows. +func TestUpsertBatch_PopulatesGroupsTable(t *testing.T) { + dbPath := filepath.Join(t.TempDir(), "data.db") + s, err := Open(dbPath) + if err != nil { + t.Fatalf("open: %v", err) + } + defer s.Close() + + items := []json.RawMessage{ + json.RawMessage(`{"id": "test-001"}`), + json.RawMessage(`{"id": "test-002"}`), + json.RawMessage(`{"id": "test-003"}`), + } + if _, _, err := s.UpsertBatch("groups", items); err != nil { + t.Fatalf("UpsertBatch: %v", err) + } + + db := s.DB() + + var generic int + if err := db.QueryRow(`SELECT COUNT(*) FROM resources WHERE resource_type = ?`, "groups").Scan(&generic); err != nil { + t.Fatalf("count resources: %v", err) + } + if generic != len(items) { + t.Fatalf("resources count = %d, want %d", generic, len(items)) + } + + var typed int + typedQuery := fmt.Sprintf(`SELECT COUNT(*) FROM "%s"`, "groups") + if err := db.QueryRow(typedQuery).Scan(&typed); err != nil { + t.Fatalf("count groups: %v", err) + } + if typed != len(items) { + t.Fatalf("groups count = %d, want %d (typed table not populated by UpsertBatch)", typed, len(items)) + } +} + +// TestUpsertBatch_PopulatesIngestTable verifies that UpsertBatch +// dispatches paginated items into both the generic resources table AND the +// typed ingest table. Regression for issue #268: before the fix, paginated +// syncs only filled the generic resources table, so domain commands that +// query the typed table saw zero rows. +func TestUpsertBatch_PopulatesIngestTable(t *testing.T) { + dbPath := filepath.Join(t.TempDir(), "data.db") + s, err := Open(dbPath) + if err != nil { + t.Fatalf("open: %v", err) + } + defer s.Close() + + items := []json.RawMessage{ + json.RawMessage(`{"id": "test-001"}`), + json.RawMessage(`{"id": "test-002"}`), + json.RawMessage(`{"id": "test-003"}`), + } + if _, _, err := s.UpsertBatch("ingest", items); err != nil { + t.Fatalf("UpsertBatch: %v", err) + } + + db := s.DB() + + var generic int + if err := db.QueryRow(`SELECT COUNT(*) FROM resources WHERE resource_type = ?`, "ingest").Scan(&generic); err != nil { + t.Fatalf("count resources: %v", err) + } + if generic != len(items) { + t.Fatalf("resources count = %d, want %d", generic, len(items)) + } + + var typed int + typedQuery := fmt.Sprintf(`SELECT COUNT(*) FROM "%s"`, "ingest") + if err := db.QueryRow(typedQuery).Scan(&typed); err != nil { + t.Fatalf("count ingest: %v", err) + } + if typed != len(items) { + t.Fatalf("ingest count = %d, want %d (typed table not populated by UpsertBatch)", typed, len(items)) + } +} + +// TestUpsertBatch_PopulatesLedgerTable verifies that UpsertBatch +// dispatches paginated items into both the generic resources table AND the +// typed ledger table. Regression for issue #268: before the fix, paginated +// syncs only filled the generic resources table, so domain commands that +// query the typed table saw zero rows. +func TestUpsertBatch_PopulatesLedgerTable(t *testing.T) { + dbPath := filepath.Join(t.TempDir(), "data.db") + s, err := Open(dbPath) + if err != nil { + t.Fatalf("open: %v", err) + } + defer s.Close() + + items := []json.RawMessage{ + json.RawMessage(`{"id": "test-001"}`), + json.RawMessage(`{"id": "test-002"}`), + json.RawMessage(`{"id": "test-003"}`), + } + if _, _, err := s.UpsertBatch("ledger", items); err != nil { + t.Fatalf("UpsertBatch: %v", err) + } + + db := s.DB() + + var generic int + if err := db.QueryRow(`SELECT COUNT(*) FROM resources WHERE resource_type = ?`, "ledger").Scan(&generic); err != nil { + t.Fatalf("count resources: %v", err) + } + if generic != len(items) { + t.Fatalf("resources count = %d, want %d", generic, len(items)) + } + + var typed int + typedQuery := fmt.Sprintf(`SELECT COUNT(*) FROM "%s"`, "ledger") + if err := db.QueryRow(typedQuery).Scan(&typed); err != nil { + t.Fatalf("count ledger: %v", err) + } + if typed != len(items) { + t.Fatalf("ledger count = %d, want %d (typed table not populated by UpsertBatch)", typed, len(items)) + } +} + +// TestUpsertBatch_PopulatesMcpTable verifies that UpsertBatch +// dispatches paginated items into both the generic resources table AND the +// typed mcp table. Regression for issue #268: before the fix, paginated +// syncs only filled the generic resources table, so domain commands that +// query the typed table saw zero rows. +func TestUpsertBatch_PopulatesMcpTable(t *testing.T) { + dbPath := filepath.Join(t.TempDir(), "data.db") + s, err := Open(dbPath) + if err != nil { + t.Fatalf("open: %v", err) + } + defer s.Close() + + items := []json.RawMessage{ + json.RawMessage(`{"id": "test-001"}`), + json.RawMessage(`{"id": "test-002"}`), + json.RawMessage(`{"id": "test-003"}`), + } + if _, _, err := s.UpsertBatch("mcp", items); err != nil { + t.Fatalf("UpsertBatch: %v", err) + } + + db := s.DB() + + var generic int + if err := db.QueryRow(`SELECT COUNT(*) FROM resources WHERE resource_type = ?`, "mcp").Scan(&generic); err != nil { + t.Fatalf("count resources: %v", err) + } + if generic != len(items) { + t.Fatalf("resources count = %d, want %d", generic, len(items)) + } + + var typed int + typedQuery := fmt.Sprintf(`SELECT COUNT(*) FROM "%s"`, "mcp") + if err := db.QueryRow(typedQuery).Scan(&typed); err != nil { + t.Fatalf("count mcp: %v", err) + } + if typed != len(items) { + t.Fatalf("mcp count = %d, want %d (typed table not populated by UpsertBatch)", typed, len(items)) + } +} + +func TestSearchMcpQuotesFTSQuerySyntax(t *testing.T) { + dbPath := filepath.Join(t.TempDir(), "data.db") + s, err := Open(dbPath) + if err != nil { + t.Fatalf("open: %v", err) + } + defer s.Close() + + items := []json.RawMessage{ + json.RawMessage(`{"id": "typed-ip", "text": "10.0.0.1"}`), + json.RawMessage(`{"id": "typed-host", "text": "host.example.com"}`), + json.RawMessage(`{"id": "typed-multi", "text": "error with extra words before timeout"}`), + } + if stored, failed, err := s.UpsertBatch("mcp", items); err != nil { + t.Fatalf("UpsertBatch: %v", err) + } else if failed != 0 || stored != len(items) { + t.Fatalf("UpsertBatch stored=%d failed=%d, want stored=%d failed=0", stored, failed, len(items)) + } + + for _, query := range []string{ + "10.0.0.1", + "host.example.com", + "error timeout", + } { + results, err := s.SearchMcp(query, 10) + if err != nil { + t.Fatalf("SearchMcp(%q): %v", query, err) + } + if len(results) == 0 { + t.Fatalf("SearchMcp(%q) returned no results", query) + } + } +} + +// TestUpsertBatch_PopulatesMediaTable verifies that UpsertBatch +// dispatches paginated items into both the generic resources table AND the +// typed media table. Regression for issue #268: before the fix, paginated +// syncs only filled the generic resources table, so domain commands that +// query the typed table saw zero rows. +func TestUpsertBatch_PopulatesMediaTable(t *testing.T) { + dbPath := filepath.Join(t.TempDir(), "data.db") + s, err := Open(dbPath) + if err != nil { + t.Fatalf("open: %v", err) + } + defer s.Close() + + items := []json.RawMessage{ + json.RawMessage(`{"id": "test-001"}`), + json.RawMessage(`{"id": "test-002"}`), + json.RawMessage(`{"id": "test-003"}`), + } + if _, _, err := s.UpsertBatch("media", items); err != nil { + t.Fatalf("UpsertBatch: %v", err) + } + + db := s.DB() + + var generic int + if err := db.QueryRow(`SELECT COUNT(*) FROM resources WHERE resource_type = ?`, "media").Scan(&generic); err != nil { + t.Fatalf("count resources: %v", err) + } + if generic != len(items) { + t.Fatalf("resources count = %d, want %d", generic, len(items)) + } + + var typed int + typedQuery := fmt.Sprintf(`SELECT COUNT(*) FROM "%s"`, "media") + if err := db.QueryRow(typedQuery).Scan(&typed); err != nil { + t.Fatalf("count media: %v", err) + } + if typed != len(items) { + t.Fatalf("media count = %d, want %d (typed table not populated by UpsertBatch)", typed, len(items)) + } +} + +func TestSearchMediaQuotesFTSQuerySyntax(t *testing.T) { + dbPath := filepath.Join(t.TempDir(), "data.db") + s, err := Open(dbPath) + if err != nil { + t.Fatalf("open: %v", err) + } + defer s.Close() + + items := []json.RawMessage{ + json.RawMessage(`{"id": "typed-ip", "message": "10.0.0.1"}`), + json.RawMessage(`{"id": "typed-host", "message": "host.example.com"}`), + json.RawMessage(`{"id": "typed-multi", "message": "error with extra words before timeout"}`), + } + if stored, failed, err := s.UpsertBatch("media", items); err != nil { + t.Fatalf("UpsertBatch: %v", err) + } else if failed != 0 || stored != len(items) { + t.Fatalf("UpsertBatch stored=%d failed=%d, want stored=%d failed=0", stored, failed, len(items)) + } + + for _, query := range []string{ + "10.0.0.1", + "host.example.com", + "error timeout", + } { + results, err := s.SearchMedia(query, 10) + if err != nil { + t.Fatalf("SearchMedia(%q): %v", query, err) + } + if len(results) == 0 { + t.Fatalf("SearchMedia(%q) returned no results", query) + } + } +} + +// TestUpsertBatch_PopulatesMirrorTable verifies that UpsertBatch +// dispatches paginated items into both the generic resources table AND the +// typed mirror table. Regression for issue #268: before the fix, paginated +// syncs only filled the generic resources table, so domain commands that +// query the typed table saw zero rows. +func TestUpsertBatch_PopulatesMirrorTable(t *testing.T) { + dbPath := filepath.Join(t.TempDir(), "data.db") + s, err := Open(dbPath) + if err != nil { + t.Fatalf("open: %v", err) + } + defer s.Close() + + items := []json.RawMessage{ + json.RawMessage(`{"id": "test-001"}`), + json.RawMessage(`{"id": "test-002"}`), + json.RawMessage(`{"id": "test-003"}`), + } + if _, _, err := s.UpsertBatch("mirror", items); err != nil { + t.Fatalf("UpsertBatch: %v", err) + } + + db := s.DB() + + var generic int + if err := db.QueryRow(`SELECT COUNT(*) FROM resources WHERE resource_type = ?`, "mirror").Scan(&generic); err != nil { + t.Fatalf("count resources: %v", err) + } + if generic != len(items) { + t.Fatalf("resources count = %d, want %d", generic, len(items)) + } + + var typed int + typedQuery := fmt.Sprintf(`SELECT COUNT(*) FROM "%s"`, "mirror") + if err := db.QueryRow(typedQuery).Scan(&typed); err != nil { + t.Fatalf("count mirror: %v", err) + } + if typed != len(items) { + t.Fatalf("mirror count = %d, want %d (typed table not populated by UpsertBatch)", typed, len(items)) + } +} + +// TestUpsertBatch_PopulatesRecordsTable verifies that UpsertBatch +// dispatches paginated items into both the generic resources table AND the +// typed records table. Regression for issue #268: before the fix, paginated +// syncs only filled the generic resources table, so domain commands that +// query the typed table saw zero rows. +func TestUpsertBatch_PopulatesRecordsTable(t *testing.T) { + dbPath := filepath.Join(t.TempDir(), "data.db") + s, err := Open(dbPath) + if err != nil { + t.Fatalf("open: %v", err) + } + defer s.Close() + + items := []json.RawMessage{ + json.RawMessage(`{"id": "test-001"}`), + json.RawMessage(`{"id": "test-002"}`), + json.RawMessage(`{"id": "test-003"}`), + } + if _, _, err := s.UpsertBatch("records", items); err != nil { + t.Fatalf("UpsertBatch: %v", err) + } + + db := s.DB() + + var generic int + if err := db.QueryRow(`SELECT COUNT(*) FROM resources WHERE resource_type = ?`, "records").Scan(&generic); err != nil { + t.Fatalf("count resources: %v", err) + } + if generic != len(items) { + t.Fatalf("resources count = %d, want %d", generic, len(items)) + } + + var typed int + typedQuery := fmt.Sprintf(`SELECT COUNT(*) FROM "%s"`, "records") + if err := db.QueryRow(typedQuery).Scan(&typed); err != nil { + t.Fatalf("count records: %v", err) + } + if typed != len(items) { + t.Fatalf("records count = %d, want %d (typed table not populated by UpsertBatch)", typed, len(items)) + } +} + +func TestSearchRecordsQuotesFTSQuerySyntax(t *testing.T) { + dbPath := filepath.Join(t.TempDir(), "data.db") + s, err := Open(dbPath) + if err != nil { + t.Fatalf("open: %v", err) + } + defer s.Close() + + items := []json.RawMessage{ + json.RawMessage(`{"id": "typed-ip", "content": "10.0.0.1"}`), + json.RawMessage(`{"id": "typed-host", "content": "host.example.com"}`), + json.RawMessage(`{"id": "typed-multi", "content": "error with extra words before timeout"}`), + } + if stored, failed, err := s.UpsertBatch("records", items); err != nil { + t.Fatalf("UpsertBatch: %v", err) + } else if failed != 0 || stored != len(items) { + t.Fatalf("UpsertBatch stored=%d failed=%d, want stored=%d failed=0", stored, failed, len(items)) + } + + for _, query := range []string{ + "10.0.0.1", + "host.example.com", + "error timeout", + } { + results, err := s.SearchRecords(query, 10) + if err != nil { + t.Fatalf("SearchRecords(%q): %v", query, err) + } + if len(results) == 0 { + t.Fatalf("SearchRecords(%q) returned no results", query) + } + } +} + +// TestUpsertBatch_PopulatesRuntimeTable verifies that UpsertBatch +// dispatches paginated items into both the generic resources table AND the +// typed runtime table. Regression for issue #268: before the fix, paginated +// syncs only filled the generic resources table, so domain commands that +// query the typed table saw zero rows. +func TestUpsertBatch_PopulatesRuntimeTable(t *testing.T) { + dbPath := filepath.Join(t.TempDir(), "data.db") + s, err := Open(dbPath) + if err != nil { + t.Fatalf("open: %v", err) + } + defer s.Close() + + items := []json.RawMessage{ + json.RawMessage(`{"id": "test-001"}`), + json.RawMessage(`{"id": "test-002"}`), + json.RawMessage(`{"id": "test-003"}`), + } + if _, _, err := s.UpsertBatch("runtime", items); err != nil { + t.Fatalf("UpsertBatch: %v", err) + } + + db := s.DB() + + var generic int + if err := db.QueryRow(`SELECT COUNT(*) FROM resources WHERE resource_type = ?`, "runtime").Scan(&generic); err != nil { + t.Fatalf("count resources: %v", err) + } + if generic != len(items) { + t.Fatalf("resources count = %d, want %d", generic, len(items)) + } + + var typed int + typedQuery := fmt.Sprintf(`SELECT COUNT(*) FROM "%s"`, "runtime") + if err := db.QueryRow(typedQuery).Scan(&typed); err != nil { + t.Fatalf("count runtime: %v", err) + } + if typed != len(items) { + t.Fatalf("runtime count = %d, want %d (typed table not populated by UpsertBatch)", typed, len(items)) + } +} + +// TestUpsertBatch_PopulatesSelectionTable verifies that UpsertBatch +// dispatches paginated items into both the generic resources table AND the +// typed selection table. Regression for issue #268: before the fix, paginated +// syncs only filled the generic resources table, so domain commands that +// query the typed table saw zero rows. +func TestUpsertBatch_PopulatesSelectionTable(t *testing.T) { + dbPath := filepath.Join(t.TempDir(), "data.db") + s, err := Open(dbPath) + if err != nil { + t.Fatalf("open: %v", err) + } + defer s.Close() + + items := []json.RawMessage{ + json.RawMessage(`{"id": "test-001"}`), + json.RawMessage(`{"id": "test-002"}`), + json.RawMessage(`{"id": "test-003"}`), + } + if _, _, err := s.UpsertBatch("selection", items); err != nil { + t.Fatalf("UpsertBatch: %v", err) + } + + db := s.DB() + + var generic int + if err := db.QueryRow(`SELECT COUNT(*) FROM resources WHERE resource_type = ?`, "selection").Scan(&generic); err != nil { + t.Fatalf("count resources: %v", err) + } + if generic != len(items) { + t.Fatalf("resources count = %d, want %d", generic, len(items)) + } + + var typed int + typedQuery := fmt.Sprintf(`SELECT COUNT(*) FROM "%s"`, "selection") + if err := db.QueryRow(typedQuery).Scan(&typed); err != nil { + t.Fatalf("count selection: %v", err) + } + if typed != len(items) { + t.Fatalf("selection count = %d, want %d (typed table not populated by UpsertBatch)", typed, len(items)) + } +} + +func TestSearchSelectionQuotesFTSQuerySyntax(t *testing.T) { + dbPath := filepath.Join(t.TempDir(), "data.db") + s, err := Open(dbPath) + if err != nil { + t.Fatalf("open: %v", err) + } + defer s.Close() + + items := []json.RawMessage{ + json.RawMessage(`{"id": "typed-ip", "name": "10.0.0.1"}`), + json.RawMessage(`{"id": "typed-host", "name": "host.example.com"}`), + json.RawMessage(`{"id": "typed-multi", "name": "error with extra words before timeout"}`), + } + if stored, failed, err := s.UpsertBatch("selection", items); err != nil { + t.Fatalf("UpsertBatch: %v", err) + } else if failed != 0 || stored != len(items) { + t.Fatalf("UpsertBatch stored=%d failed=%d, want stored=%d failed=0", stored, failed, len(items)) + } + + for _, query := range []string{ + "10.0.0.1", + "host.example.com", + "error timeout", + } { + results, err := s.SearchSelection(query, 10) + if err != nil { + t.Fatalf("SearchSelection(%q): %v", query, err) + } + if len(results) == 0 { + t.Fatalf("SearchSelection(%q) returned no results", query) + } + } +} + +// TestUpsertBatch_PopulatesTagsTable verifies that UpsertBatch +// dispatches paginated items into both the generic resources table AND the +// typed tags table. Regression for issue #268: before the fix, paginated +// syncs only filled the generic resources table, so domain commands that +// query the typed table saw zero rows. +func TestUpsertBatch_PopulatesTagsTable(t *testing.T) { + dbPath := filepath.Join(t.TempDir(), "data.db") + s, err := Open(dbPath) + if err != nil { + t.Fatalf("open: %v", err) + } + defer s.Close() + + items := []json.RawMessage{ + json.RawMessage(`{"id": "test-001"}`), + json.RawMessage(`{"id": "test-002"}`), + json.RawMessage(`{"id": "test-003"}`), + } + if _, _, err := s.UpsertBatch("tags", items); err != nil { + t.Fatalf("UpsertBatch: %v", err) + } + + db := s.DB() + + var generic int + if err := db.QueryRow(`SELECT COUNT(*) FROM resources WHERE resource_type = ?`, "tags").Scan(&generic); err != nil { + t.Fatalf("count resources: %v", err) + } + if generic != len(items) { + t.Fatalf("resources count = %d, want %d", generic, len(items)) + } + + var typed int + typedQuery := fmt.Sprintf(`SELECT COUNT(*) FROM "%s"`, "tags") + if err := db.QueryRow(typedQuery).Scan(&typed); err != nil { + t.Fatalf("count tags: %v", err) + } + if typed != len(items) { + t.Fatalf("tags count = %d, want %d (typed table not populated by UpsertBatch)", typed, len(items)) + } +} diff --git a/library/productivity/devonthink/internal/types/types.go b/library/productivity/devonthink/internal/types/types.go new file mode 100644 index 0000000000..b419a4b12c --- /dev/null +++ b/library/productivity/devonthink/internal/types/types.go @@ -0,0 +1,128 @@ +// Copyright 2026 rowdy and contributors. Licensed under Apache-2.0. See LICENSE. +// Generated by CLI Printing Press (https://github.com/mvanhorn/cli-printing-press). DO NOT EDIT. + +package types + +import "encoding/json" + +type BatchPlan struct { + PlanPath string `json:"plan_path"` + ActionCount int `json:"action_count"` + DryRun bool `json:"dry_run"` +} + +type ContextPack struct { + Records int `json:"records"` + Markdown string `json:"markdown"` + Truncated bool `json:"truncated"` +} + +type Database struct { + Uuid string `json:"uuid"` + Name string `json:"name"` + IncomingGroupUuid string `json:"incoming_group_uuid"` + IsOpen bool `json:"is_open"` +} + +type GraphAudit struct { + Issues json.RawMessage `json:"issues"` + Samples json.RawMessage `json:"samples"` +} + +type GraphLinks struct { + Uuid string `json:"uuid"` + Incoming json.RawMessage `json:"incoming"` + Outgoing json.RawMessage `json:"outgoing"` + Unresolved json.RawMessage `json:"unresolved"` +} + +type GroupTree struct { + Uuid string `json:"uuid"` + Name string `json:"name"` + Path string `json:"path"` + ChildCount int `json:"child_count"` +} + +type InventoryExport struct { + SchemaVersion int `json:"schema_version"` + Databases json.RawMessage `json:"databases"` + Documents json.RawMessage `json:"documents"` +} + +type LedgerEntry struct { + Id string `json:"id"` + Time string `json:"time"` + Action string `json:"action"` + Count int `json:"count"` + Status string `json:"status"` +} + +type MCPTool struct { + Name string `json:"name"` + Description string `json:"description"` +} + +type OperationResult struct { + Status string `json:"status"` + DryRun bool `json:"dry_run"` + Message string `json:"message"` + LedgerId string `json:"ledger_id"` +} + +type PrivacyAudit struct { + Records int `json:"records"` + EstimatedChars int `json:"estimated_chars"` + Warnings json.RawMessage `json:"warnings"` +} + +type Record struct { + Uuid string `json:"uuid"` + Name string `json:"name"` + Kind string `json:"kind"` + Path string `json:"path"` + Location string `json:"location"` + Database string `json:"database"` + ItemLink string `json:"item_link"` +} + +type RecordContent struct { + Uuid string `json:"uuid"` + Content string `json:"content"` + ContentFormat string `json:"content_format"` + Truncated bool `json:"truncated"` +} + +type RuntimeStatus struct { + Running bool `json:"running"` + Version string `json:"version"` + ApplescriptOk bool `json:"applescript_ok"` + McpUrl string `json:"mcp_url"` + MirrorPath string `json:"mirror_path"` +} + +type SelectionSnapshot struct { + CreatedAt string `json:"created_at"` + Note string `json:"note"` + Records json.RawMessage `json:"records"` +} + +type SheetResult struct { + Rows json.RawMessage `json:"rows"` +} + +type SyncResult struct { + DatabaseCount int `json:"database_count"` + RecordCount int `json:"record_count"` + MirrorPath string `json:"mirror_path"` +} + +type TagAnalysis struct { + TagCount int `json:"tag_count"` + DuplicateCandidates json.RawMessage `json:"duplicate_candidates"` + ActionTags json.RawMessage `json:"action_tags"` +} + +type TextResult struct { + Text string `json:"text"` + Truncated bool `json:"truncated"` +} diff --git a/library/productivity/devonthink/tools-manifest.json b/library/productivity/devonthink/tools-manifest.json new file mode 100644 index 0000000000..3fe13c96e1 --- /dev/null +++ b/library/productivity/devonthink/tools-manifest.json @@ -0,0 +1,918 @@ +{ + "api_name": "devonthink", + "base_url": "", + "description": "Local-first CLI for DEVONthink automation, search, context packs, and safe batch plans", + "mcp_ready": "full", + "http_transport": "standard", + "auth": { + "type": "none" + }, + "required_headers": [], + "tools": [ + { + "name": "ai_ask", + "description": "Ask DEVONthink AI about selected local records with explicit cloud-use warnings. Required: prompt. Optional: uuid, selection, dry-run. Returns the new TextResult.", + "method": "POST", + "path": "/ai/ask", + "no_auth": true, + "params": [ + { + "name": "prompt", + "type": "string", + "location": "path", + "description": "Prompt to send", + "required": true + }, + { + "name": "uuid", + "type": "string", + "location": "body", + "description": "Record UUID to attach" + }, + { + "name": "selection", + "type": "bool", + "location": "body", + "description": "Attach the current selection" + }, + { + "name": "dry-run", + "wire_name": "dry_run", + "type": "bool", + "location": "body", + "description": "Preview attachments without sending" + } + ] + }, + { + "name": "ai_summarize", + "description": "Summarize records or highlights. Required: uuid. Optional: highlights, dry-run. Returns the new TextResult.", + "method": "POST", + "path": "/ai/summarize", + "no_auth": true, + "params": [ + { + "name": "uuid", + "type": "string", + "location": "path", + "description": "Record UUID or item link", + "required": true + }, + { + "name": "highlights", + "type": "bool", + "location": "body", + "description": "Summarize highlights only" + }, + { + "name": "dry-run", + "wire_name": "dry_run", + "type": "bool", + "location": "body", + "description": "Preview attachments without sending" + } + ] + }, + { + "name": "batch_apply", + "description": "Apply a previously reviewed local JSON plan. Required: plan. Optional: yes. Returns the new OperationResult.", + "method": "POST", + "path": "/batch/apply", + "no_auth": true, + "params": [ + { + "name": "plan", + "type": "string", + "location": "path", + "description": "Plan JSON path", + "required": true + }, + { + "name": "yes", + "type": "bool", + "location": "body", + "description": "Confirm apply without an interactive prompt" + } + ] + }, + { + "name": "batch_plan", + "description": "Stage multi-record changes as a local JSON plan. Optional: query, from, add-tag (plus 3 more). Returns the new BatchPlan.", + "method": "POST", + "path": "/batch/plan", + "no_auth": true, + "params": [ + { + "name": "query", + "type": "string", + "location": "body", + "description": "Search query selecting records" + }, + { + "name": "from", + "type": "string", + "location": "body", + "description": "Source selector such as selection or file" + }, + { + "name": "add-tag", + "wire_name": "add_tag", + "type": "string", + "location": "body", + "description": "Tag to add" + }, + { + "name": "move-to", + "wire_name": "move_to", + "type": "string", + "location": "body", + "description": "Destination group path or UUID" + }, + { + "name": "output", + "type": "string", + "location": "body", + "description": "Plan output path" + }, + { + "name": "dry-run", + "wire_name": "dry_run", + "type": "bool", + "location": "body", + "description": "Preview only" + } + ] + }, + { + "name": "context_pack", + "description": "Build a compact local context pack from records, selection, or search. Optional: query, uuid, selection (plus 1 more). Returns the ContextPack.", + "method": "GET", + "path": "/context/pack", + "no_auth": true, + "params": [ + { + "name": "query", + "type": "string", + "location": "query", + "description": "Search query to seed the pack" + }, + { + "name": "uuid", + "type": "string", + "location": "query", + "description": "Record UUID to seed the pack" + }, + { + "name": "selection", + "type": "bool", + "location": "query", + "description": "Use current DEVONthink selection" + }, + { + "name": "token-budget", + "wire_name": "token_budget", + "type": "int", + "location": "query", + "description": "Approximate token budget" + } + ] + }, + { + "name": "databases_list", + "description": "List open databases. Returns array of Database.", + "method": "GET", + "path": "/databases", + "no_auth": true, + "params": [] + }, + { + "name": "graph_audit", + "description": "Detect orphans, unresolved wiki links, weak hubs, and tag-only clusters. Optional: database, limit (default: 50). Returns the GraphAudit.", + "method": "GET", + "path": "/graph/audit", + "no_auth": true, + "params": [ + { + "name": "database", + "type": "string", + "location": "query", + "description": "Database name or UUID" + }, + { + "name": "limit", + "type": "int", + "location": "query", + "description": "Maximum sample records per issue" + } + ] + }, + { + "name": "graph_links", + "description": "List item links, wiki links, mentions, and unresolved wiki names. Optional: uuid, database, direction (default: both). Returns the GraphLinks.", + "method": "GET", + "path": "/graph/links", + "no_auth": true, + "params": [ + { + "name": "uuid", + "type": "string", + "location": "query", + "description": "Record UUID or item link" + }, + { + "name": "database", + "type": "string", + "location": "query", + "description": "Database name or UUID" + }, + { + "name": "direction", + "type": "string", + "location": "query", + "description": "incoming, outgoing, or both" + } + ] + }, + { + "name": "groups_tree", + "description": "Render a bounded group tree. Optional: database, group, depth (default: 2). Returns the GroupTree.", + "method": "GET", + "path": "/groups/tree", + "no_auth": true, + "params": [ + { + "name": "database", + "type": "string", + "location": "query", + "description": "Database name or UUID" + }, + { + "name": "group", + "type": "string", + "location": "query", + "description": "Root group UUID or path" + }, + { + "name": "depth", + "type": "int", + "location": "query", + "description": "Maximum tree depth" + } + ] + }, + { + "name": "ingest_file", + "description": "Import or index a file or folder. Required: path. Optional: destination, mode (default: import). Returns the new OperationResult.", + "method": "POST", + "path": "/ingest/file", + "no_auth": true, + "params": [ + { + "name": "path", + "type": "string", + "location": "path", + "description": "File or folder path", + "required": true + }, + { + "name": "destination", + "type": "string", + "location": "body", + "description": "Destination group path or UUID" + }, + { + "name": "mode", + "type": "string", + "location": "body", + "description": "import or index" + } + ] + }, + { + "name": "ingest_url", + "description": "Capture a URL as Markdown, HTML, PDF, bookmark, or webarchive. Required: url. Optional: destination, type (default: markdown). Returns the new OperationResult.", + "method": "POST", + "path": "/ingest/url", + "no_auth": true, + "params": [ + { + "name": "url", + "type": "string", + "location": "path", + "description": "URL to capture", + "required": true + }, + { + "name": "destination", + "type": "string", + "location": "body", + "description": "Destination group path or UUID" + }, + { + "name": "type", + "type": "string", + "location": "body", + "description": "Capture format" + } + ] + }, + { + "name": "inventory_export", + "description": "Export databases, groups, tags, and selected document metadata for downstream tools. Optional: database, format (default: maintenance), include-text (plus 1 more). Returns the InventoryExport.", + "method": "GET", + "path": "/inventory/export", + "no_auth": true, + "params": [ + { + "name": "database", + "type": "string", + "location": "query", + "description": "Database name or UUID" + }, + { + "name": "format", + "type": "string", + "location": "query", + "description": "Export format: maintenance, llm-wiki, raw" + }, + { + "name": "include-text", + "wire_name": "include_text", + "type": "bool", + "location": "query", + "description": "Include extracted text where safe" + }, + { + "name": "output", + "type": "string", + "location": "query", + "description": "Output JSON path" + } + ] + }, + { + "name": "ledger_list", + "description": "List recent CLI operation ledger entries. Optional: since, limit (default: 50). Returns array of LedgerEntry.", + "method": "GET", + "path": "/ledger", + "no_auth": true, + "params": [ + { + "name": "since", + "type": "string", + "location": "query", + "description": "Time window such as 7d" + }, + { + "name": "limit", + "type": "int", + "location": "query", + "description": "Maximum entries" + } + ] + }, + { + "name": "ledger_show", + "description": "Show one ledger entry with target proofs and rollback hints. Required: id. Returns the LedgerEntry.", + "method": "GET", + "path": "/ledger/{id}", + "no_auth": true, + "params": [ + { + "name": "id", + "type": "string", + "location": "path", + "description": "Ledger entry ID", + "required": true + } + ] + }, + { + "name": "mcp_call", + "description": "Call a local official DEVONthink MCP tool by name. Required: tool. Optional: args. Returns the new TextResult.", + "method": "POST", + "path": "/mcp/call", + "no_auth": true, + "params": [ + { + "name": "tool", + "type": "string", + "location": "path", + "description": "Official MCP tool name", + "required": true + }, + { + "name": "args", + "type": "string", + "location": "body", + "description": "JSON argument object" + } + ] + }, + { + "name": "mcp_schema", + "description": "Emit cached MCP tool schemas. Returns the TextResult.", + "method": "GET", + "path": "/mcp/schema", + "no_auth": true, + "params": [] + }, + { + "name": "mcp_tools", + "description": "List official DEVONthink MCP tools when local MCP HTTP is enabled. Returns array of MCPTool.", + "method": "GET", + "path": "/mcp/tools", + "no_auth": true, + "params": [] + }, + { + "name": "media_ocr", + "description": "OCR an image or scanned PDF. Required: uuid. Optional: format (default: pdf). Returns the new OperationResult.", + "method": "POST", + "path": "/media/{uuid}/ocr", + "no_auth": true, + "params": [ + { + "name": "uuid", + "type": "string", + "location": "path", + "description": "Record UUID or item link", + "required": true + }, + { + "name": "format", + "type": "string", + "location": "body", + "description": "Output format" + } + ] + }, + { + "name": "media_transcribe", + "description": "Transcribe audio, video, image, or PDF content. Required: uuid. Optional: language, timestamps. Returns the new TextResult.", + "method": "POST", + "path": "/media/{uuid}/transcribe", + "no_auth": true, + "params": [ + { + "name": "uuid", + "type": "string", + "location": "path", + "description": "Record UUID or item link", + "required": true + }, + { + "name": "language", + "type": "string", + "location": "body", + "description": "ISO language code" + }, + { + "name": "timestamps", + "type": "bool", + "location": "body", + "description": "Include timestamps when available" + } + ] + }, + { + "name": "mirror_search", + "description": "Search the local mirror with FTS. Required: query. Optional: limit (default: 20). Returns array of Record.", + "method": "GET", + "path": "/mirror/search", + "no_auth": true, + "params": [ + { + "name": "query", + "type": "string", + "location": "path", + "description": "FTS query", + "required": true + }, + { + "name": "limit", + "type": "int", + "location": "query", + "description": "Maximum rows" + } + ] + }, + { + "name": "mirror_sync", + "description": "Refresh the local SQLite mirror from open DEVONthink databases. Optional: database, full. Returns the SyncResult.", + "method": "GET", + "path": "/mirror/sync", + "no_auth": true, + "params": [ + { + "name": "database", + "type": "string", + "location": "query", + "description": "Database name or UUID" + }, + { + "name": "full", + "type": "bool", + "location": "query", + "description": "Force full sync" + } + ] + }, + { + "name": "privacy_audit", + "description": "Preview database scope, content-size budget, and cloud/MCP exposure before handoff. Optional: query, uuid, limit (default: 20). Returns the PrivacyAudit.", + "method": "GET", + "path": "/privacy/audit", + "no_auth": true, + "params": [ + { + "name": "query", + "type": "string", + "location": "query", + "description": "Search query to audit" + }, + { + "name": "uuid", + "type": "string", + "location": "query", + "description": "Record UUID or item link" + }, + { + "name": "limit", + "type": "int", + "location": "query", + "description": "Maximum records to inspect" + } + ] + }, + { + "name": "records_content", + "description": "Extract text content with length and redaction controls. Required: uuid. Optional: max-chars (default: 8000). Returns the RecordContent.", + "method": "GET", + "path": "/records/{uuid}/content", + "no_auth": true, + "params": [ + { + "name": "uuid", + "type": "string", + "location": "path", + "description": "Record UUID or item link", + "required": true + }, + { + "name": "max-chars", + "wire_name": "max_chars", + "type": "int", + "location": "query", + "description": "Maximum characters to emit" + } + ] + }, + { + "name": "records_create", + "description": "Create a record or group after validating destination. Required: title. Optional: type (default: markdown), destination, content (plus 1 more). Returns the new OperationResult.", + "method": "POST", + "path": "/records/create", + "no_auth": true, + "params": [ + { + "name": "title", + "type": "string", + "location": "body", + "description": "Record title", + "required": true + }, + { + "name": "type", + "type": "string", + "location": "body", + "description": "Record type" + }, + { + "name": "destination", + "type": "string", + "location": "body", + "description": "Destination group path or UUID" + }, + { + "name": "content", + "type": "string", + "location": "body", + "description": "Inline text content" + }, + { + "name": "tags", + "type": "string", + "location": "body", + "description": "Comma-separated tags" + } + ] + }, + { + "name": "records_get", + "description": "Get record metadata. Required: uuid. Returns the Record.", + "method": "GET", + "path": "/records/{uuid}", + "no_auth": true, + "params": [ + { + "name": "uuid", + "type": "string", + "location": "path", + "description": "Record UUID or item link", + "required": true + } + ] + }, + { + "name": "records_highlights", + "description": "Extract highlights and annotations. Required: uuid. Returns the TextResult.", + "method": "GET", + "path": "/records/{uuid}/highlights", + "no_auth": true, + "params": [ + { + "name": "uuid", + "type": "string", + "location": "path", + "description": "Record UUID or item link", + "required": true + } + ] + }, + { + "name": "records_lookup", + "description": "Look up records by exact name, URL, path, filename, location, or comment. Optional: name, url, path (plus 3 more). Returns array of Record.", + "method": "GET", + "path": "/records/lookup", + "no_auth": true, + "params": [ + { + "name": "name", + "type": "string", + "location": "query", + "description": "Exact record name" + }, + { + "name": "url", + "type": "string", + "location": "query", + "description": "Exact record URL" + }, + { + "name": "path", + "type": "string", + "location": "query", + "description": "Exact filesystem path" + }, + { + "name": "location", + "type": "string", + "location": "query", + "description": "Exact DEVONthink location" + }, + { + "name": "filename", + "type": "string", + "location": "query", + "description": "Exact filename" + }, + { + "name": "comment", + "type": "string", + "location": "query", + "description": "Exact comment" + } + ] + }, + { + "name": "records_move", + "description": "Move, duplicate, replicate, or trash a record with dry-run proof. Required: uuid. Optional: destination, mode (default: move), dry-run. Returns the new OperationResult.", + "method": "POST", + "path": "/records/{uuid}/move", + "no_auth": true, + "params": [ + { + "name": "uuid", + "type": "string", + "location": "path", + "description": "Record UUID or item link", + "required": true + }, + { + "name": "destination", + "type": "string", + "location": "body", + "description": "Destination group path or UUID" + }, + { + "name": "mode", + "type": "string", + "location": "body", + "description": "Operation mode: move, duplicate, replicate, trash" + }, + { + "name": "dry-run", + "wire_name": "dry_run", + "type": "bool", + "location": "body", + "description": "Show the planned move without writing" + } + ] + }, + { + "name": "records_related", + "description": "Find related records using DEVONthink similarity. Required: uuid. Optional: limit (default: 20). Returns array of Record.", + "method": "GET", + "path": "/records/{uuid}/related", + "no_auth": true, + "params": [ + { + "name": "uuid", + "type": "string", + "location": "path", + "description": "Record UUID or item link", + "required": true + }, + { + "name": "limit", + "type": "int", + "location": "query", + "description": "Maximum related records" + } + ] + }, + { + "name": "records_search", + "description": "Search records using DEVONthink query syntax or local mirror fallback. Required: query. Optional: database, group, limit (default: 20). Returns array of Record.", + "method": "GET", + "path": "/records/search", + "no_auth": true, + "params": [ + { + "name": "query", + "type": "string", + "location": "path", + "description": "DEVONthink search query", + "required": true + }, + { + "name": "database", + "type": "string", + "location": "query", + "description": "Database name or UUID" + }, + { + "name": "group", + "type": "string", + "location": "query", + "description": "Group UUID or path" + }, + { + "name": "limit", + "type": "int", + "location": "query", + "description": "Maximum records to return" + } + ] + }, + { + "name": "records_update", + "description": "Update record text, properties, tags, comment, URL, aliases, or rating. Required: uuid. Optional: title, tags, add-tag (plus 3 more). Returns the updated OperationResult.", + "method": "PATCH", + "path": "/records/{uuid}", + "no_auth": true, + "params": [ + { + "name": "uuid", + "type": "string", + "location": "path", + "description": "Record UUID or item link", + "required": true + }, + { + "name": "title", + "type": "string", + "location": "body", + "description": "New title" + }, + { + "name": "tags", + "type": "string", + "location": "body", + "description": "Replacement comma-separated tags" + }, + { + "name": "add-tag", + "wire_name": "add_tag", + "type": "string", + "location": "body", + "description": "Tag to add" + }, + { + "name": "comment", + "type": "string", + "location": "body", + "description": "New comment" + }, + { + "name": "append", + "type": "string", + "location": "body", + "description": "Text to append" + }, + { + "name": "dry-run", + "wire_name": "dry_run", + "type": "bool", + "location": "body", + "description": "Show the planned update without writing" + } + ] + }, + { + "name": "records_versions", + "description": "List saved record versions. Required: uuid. Returns array of Record.", + "method": "GET", + "path": "/records/{uuid}/versions", + "no_auth": true, + "params": [ + { + "name": "uuid", + "type": "string", + "location": "path", + "description": "Record UUID or item link", + "required": true + } + ] + }, + { + "name": "runtime_doctor", + "description": "Check DEVONthink app, AppleScript, optional MCP, and local mirror readiness. Returns the RuntimeStatus.", + "method": "GET", + "path": "/runtime/doctor", + "no_auth": true, + "params": [] + }, + { + "name": "selection_get", + "description": "Return currently selected records. Returns array of Record.", + "method": "GET", + "path": "/selection", + "no_auth": true, + "params": [] + }, + { + "name": "selection_snapshot", + "description": "Capture the current selection as a reusable workflow seed. Optional: note, output. Returns the SelectionSnapshot.", + "method": "GET", + "path": "/selection/snapshot", + "no_auth": true, + "params": [ + { + "name": "note", + "type": "string", + "location": "query", + "description": "Operator note to include in the snapshot" + }, + { + "name": "output", + "type": "string", + "location": "query", + "description": "Optional output JSON file" + } + ] + }, + { + "name": "sheets_get", + "description": "Read a sheet as structured rows. Required: uuid. Returns the SheetResult.", + "method": "GET", + "path": "/sheets/{uuid}", + "no_auth": true, + "params": [ + { + "name": "uuid", + "type": "string", + "location": "path", + "description": "Sheet UUID or item link", + "required": true + } + ] + }, + { + "name": "tags_analyze", + "description": "Analyze tags for duplicates, case drift, action tags, and maintenance tags. Optional: database. Returns the TagAnalysis.", + "method": "GET", + "path": "/tags/analyze", + "no_auth": true, + "params": [ + { + "name": "database", + "type": "string", + "location": "query", + "description": "Database name or UUID" + } + ] + } + ] +} From a67c245664b19b27a8a0d5264f2021dc730f2749 Mon Sep 17 00:00:00 2001 From: Johns <19662585+Rowdy@users.noreply.github.com> Date: Mon, 29 Jun 2026 20:19:10 +0200 Subject: [PATCH 2/4] fix(devonthink): address review findings --- .../devonthink-review-stability-fixes.json | 15 +++++++ .../devonthink/internal/cli/mcp_call.go | 2 - .../devonthink/internal/client/client_test.go | 30 ++++++++++++++ .../internal/client/local_devonthink.go | 40 ++++++++++++++----- 4 files changed, 75 insertions(+), 12 deletions(-) create mode 100644 library/productivity/devonthink/.printing-press-patches/devonthink-review-stability-fixes.json diff --git a/library/productivity/devonthink/.printing-press-patches/devonthink-review-stability-fixes.json b/library/productivity/devonthink/.printing-press-patches/devonthink-review-stability-fixes.json new file mode 100644 index 0000000000..9db681bbbb --- /dev/null +++ b/library/productivity/devonthink/.printing-press-patches/devonthink-review-stability-fixes.json @@ -0,0 +1,15 @@ +{ + "schema_version": 2, + "id": "devonthink-review-stability-fixes", + "applied_at": "2026-06-29", + "base_run_id": "20260629T175310Z-b4705e85", + "base_printing_press_version": "4.27.0", + "summary": "Fix review findings around AppleScript list parsing, mirror backend disclosure, and an empty MCP stdin branch.", + "reason": "Publishing review found comma-space AppleScript parsing corruption, a success-shaped mirror noop, and an unfinished empty codegen branch.", + "files": [ + "internal/client/local_devonthink.go", + "internal/client/client_test.go", + "internal/cli/mcp_call.go" + ], + "validated_outcome": "go test ./... passed in the source repo and packaged public-library module." +} diff --git a/library/productivity/devonthink/internal/cli/mcp_call.go b/library/productivity/devonthink/internal/cli/mcp_call.go index d9f4ded366..14b6b83e80 100644 --- a/library/productivity/devonthink/internal/cli/mcp_call.go +++ b/library/productivity/devonthink/internal/cli/mcp_call.go @@ -25,8 +25,6 @@ func newMcpCallCmd(flags *rootFlags) *cobra.Command { if len(args) == 0 { return cmd.Help() } - if !stdinBody { - } c, err := flags.newClient() if err != nil { return err diff --git a/library/productivity/devonthink/internal/client/client_test.go b/library/productivity/devonthink/internal/client/client_test.go index 33313b153e..6618092ecb 100644 --- a/library/productivity/devonthink/internal/client/client_test.go +++ b/library/productivity/devonthink/internal/client/client_test.go @@ -70,3 +70,33 @@ func TestTruncateBody_UTF8RuneAtBoundary(t *testing.T) { t.Fatalf("len = %d, want %d (partial rune should be dropped, not replaced)", len(got), want) } } + +func TestSplitOSAList_RecordSeparatorPreservesCommas(t *testing.T) { + t.Parallel() + + got := splitOSAList("Research, 2026\x1eInbox") + want := []string{"Research, 2026", "Inbox"} + if len(got) != len(want) { + t.Fatalf("len = %d, want %d: %#v", len(got), len(want), got) + } + for i := range want { + if got[i] != want[i] { + t.Fatalf("got[%d] = %q, want %q", i, got[i], want[i]) + } + } +} + +func TestSplitOSAList_CommaFallback(t *testing.T) { + t.Parallel() + + got := splitOSAList("Research, Inbox") + want := []string{"Research", "Inbox"} + if len(got) != len(want) { + t.Fatalf("len = %d, want %d: %#v", len(got), len(want), got) + } + for i := range want { + if got[i] != want[i] { + t.Fatalf("got[%d] = %q, want %q", i, got[i], want[i]) + } + } +} diff --git a/library/productivity/devonthink/internal/client/local_devonthink.go b/library/productivity/devonthink/internal/client/local_devonthink.go index 92cfbcabb3..156a6a5170 100644 --- a/library/productivity/devonthink/internal/client/local_devonthink.go +++ b/library/productivity/devonthink/internal/client/local_devonthink.go @@ -566,11 +566,15 @@ func (c *Client) localPrivacyAudit(ctx context.Context, params map[string]string func (c *Client) localMirrorSync(params map[string]string) map[string]any { return map[string]any{ - "status": "noop", + "status": "not_implemented", "mirror_path": params["path"], "synced": 0, "updated": 0, "deleted": 0, + "warnings": []string{ + "The local SQLite mirror backend is not active in this build; no rows were synced.", + "Use live records search or inventory export for current DEVONthink metadata.", + }, } } @@ -580,17 +584,19 @@ func (c *Client) localMirrorSearch(ctx context.Context, params map[string]string records := c.localRecordSearch(ctx, map[string]string{"query": query, "limit": limit}) if len(records) == 0 { return []map[string]any{{ - "uuid": "mirror-fallback", - "name": "No local mirror rows matched " + query, - "path": "query:" + query, - "query": query, - "source": "live-metadata-fallback", - "note": "Run mirror sync to build the SQLite mirror, or use records search for live MCP metadata.", + "uuid": "mirror-not-implemented", + "name": "Local mirror backend is not active", + "path": "query:" + query, + "query": query, + "source": "live-metadata-fallback", + "status": "not_implemented", + "warning": "The local SQLite mirror backend is not active in this build; use records search for live MCP metadata.", }} } for i := range records { records[i]["mirror_query"] = query records[i]["source"] = "live-metadata-fallback" + records[i]["warning"] = "The local SQLite mirror backend is not active in this build; this result came from live metadata fallback." } return records } @@ -872,7 +878,12 @@ func devonthinkVersion(ctx context.Context) (string, error) { } func devonthinkDatabaseNames(ctx context.Context) ([]string, error) { - out, err := runOSA(ctx, `tell application id "`+devonthinkAppID+`" to get name of databases`) + out, err := runOSA(ctx, `set oldDelimiters to AppleScript's text item delimiters +set AppleScript's text item delimiters to (ASCII character 30) +tell application id "`+devonthinkAppID+`" to set devonthinkNames to name of databases +set joinedNames to devonthinkNames as text +set AppleScript's text item delimiters to oldDelimiters +return joinedNames`) if err != nil { return nil, err } @@ -880,7 +891,12 @@ func devonthinkDatabaseNames(ctx context.Context) ([]string, error) { } func devonthinkSelectedRecordNames(ctx context.Context) ([]map[string]any, error) { - out, err := runOSA(ctx, `tell application id "`+devonthinkAppID+`" to get name of selected records`) + out, err := runOSA(ctx, `set oldDelimiters to AppleScript's text item delimiters +set AppleScript's text item delimiters to (ASCII character 30) +tell application id "`+devonthinkAppID+`" to set devonthinkNames to name of selected records +set joinedNames to devonthinkNames as text +set AppleScript's text item delimiters to oldDelimiters +return joinedNames`) if err != nil { return []map[string]any{}, err } @@ -911,7 +927,11 @@ func splitOSAList(out string) []string { if out == "" || out == "missing value" { return []string{} } - parts := strings.Split(out, ", ") + separator := ", " + if strings.Contains(out, "\x1e") { + separator = "\x1e" + } + parts := strings.Split(out, separator) values := make([]string, 0, len(parts)) for _, part := range parts { part = strings.TrimSpace(part) From 2067d37405a964f9bdefcd38d31c06da7289e1ea Mon Sep 17 00:00:00 2001 From: Johns <19662585+Rowdy@users.noreply.github.com> Date: Mon, 29 Jun 2026 20:22:02 +0200 Subject: [PATCH 3/4] fix(devonthink): tighten review fixes --- .../devonthink/internal/client/client_test.go | 4 +-- .../internal/client/local_devonthink.go | 35 +++++++------------ 2 files changed, 14 insertions(+), 25 deletions(-) diff --git a/library/productivity/devonthink/internal/client/client_test.go b/library/productivity/devonthink/internal/client/client_test.go index 6618092ecb..7ac82b6b9e 100644 --- a/library/productivity/devonthink/internal/client/client_test.go +++ b/library/productivity/devonthink/internal/client/client_test.go @@ -86,11 +86,11 @@ func TestSplitOSAList_RecordSeparatorPreservesCommas(t *testing.T) { } } -func TestSplitOSAList_CommaFallback(t *testing.T) { +func TestSplitOSAList_WithoutRecordSeparatorIsSingleValue(t *testing.T) { t.Parallel() got := splitOSAList("Research, Inbox") - want := []string{"Research", "Inbox"} + want := []string{"Research, Inbox"} if len(got) != len(want) { t.Fatalf("len = %d, want %d: %#v", len(got), len(want), got) } diff --git a/library/productivity/devonthink/internal/client/local_devonthink.go b/library/productivity/devonthink/internal/client/local_devonthink.go index 156a6a5170..69a953bc2a 100644 --- a/library/productivity/devonthink/internal/client/local_devonthink.go +++ b/library/productivity/devonthink/internal/client/local_devonthink.go @@ -578,27 +578,17 @@ func (c *Client) localMirrorSync(params map[string]string) map[string]any { } } -func (c *Client) localMirrorSearch(ctx context.Context, params map[string]string) []map[string]any { +func (c *Client) localMirrorSearch(_ context.Context, params map[string]string) []map[string]any { query := firstNonEmpty(params["query"], params["q"]) - limit := firstNonEmpty(params["limit"], "20") - records := c.localRecordSearch(ctx, map[string]string{"query": query, "limit": limit}) - if len(records) == 0 { - return []map[string]any{{ - "uuid": "mirror-not-implemented", - "name": "Local mirror backend is not active", - "path": "query:" + query, - "query": query, - "source": "live-metadata-fallback", - "status": "not_implemented", - "warning": "The local SQLite mirror backend is not active in this build; use records search for live MCP metadata.", - }} - } - for i := range records { - records[i]["mirror_query"] = query - records[i]["source"] = "live-metadata-fallback" - records[i]["warning"] = "The local SQLite mirror backend is not active in this build; this result came from live metadata fallback." - } - return records + return []map[string]any{{ + "uuid": "mirror-not-implemented", + "name": "Local mirror backend is not active", + "path": "query:" + query, + "query": query, + "source": "local-mirror", + "status": "not_implemented", + "warning": "The local SQLite mirror backend is not active in this build; use records search for live MCP metadata.", + }} } func (c *Client) localContextPack(ctx context.Context, params map[string]string) map[string]any { @@ -927,11 +917,10 @@ func splitOSAList(out string) []string { if out == "" || out == "missing value" { return []string{} } - separator := ", " + parts := []string{out} if strings.Contains(out, "\x1e") { - separator = "\x1e" + parts = strings.Split(out, "\x1e") } - parts := strings.Split(out, separator) values := make([]string, 0, len(parts)) for _, part := range parts { part = strings.TrimSpace(part) From 9f3f8cb715c8bbbba66995c1e5e3cb89412b6e88 Mon Sep 17 00:00:00 2001 From: Johns <19662585+Rowdy@users.noreply.github.com> Date: Mon, 29 Jun 2026 20:32:44 +0200 Subject: [PATCH 4/4] fix(devonthink): correct list fallback reads --- .../devonthink-review-stability-fixes.json | 4 +++- library/productivity/devonthink/internal/cli/graph_audit.go | 2 +- .../productivity/devonthink/internal/cli/records_search.go | 2 +- 3 files changed, 5 insertions(+), 3 deletions(-) diff --git a/library/productivity/devonthink/.printing-press-patches/devonthink-review-stability-fixes.json b/library/productivity/devonthink/.printing-press-patches/devonthink-review-stability-fixes.json index 9db681bbbb..c07af0db06 100644 --- a/library/productivity/devonthink/.printing-press-patches/devonthink-review-stability-fixes.json +++ b/library/productivity/devonthink/.printing-press-patches/devonthink-review-stability-fixes.json @@ -9,7 +9,9 @@ "files": [ "internal/client/local_devonthink.go", "internal/client/client_test.go", - "internal/cli/mcp_call.go" + "internal/cli/mcp_call.go", + "internal/cli/records_search.go", + "internal/cli/graph_audit.go" ], "validated_outcome": "go test ./... passed in the source repo and packaged public-library module." } diff --git a/library/productivity/devonthink/internal/cli/graph_audit.go b/library/productivity/devonthink/internal/cli/graph_audit.go index 85e7e62078..7268325ab4 100644 --- a/library/productivity/devonthink/internal/cli/graph_audit.go +++ b/library/productivity/devonthink/internal/cli/graph_audit.go @@ -34,7 +34,7 @@ func newGraphAuditCmd(flags *rootFlags) *cobra.Command { if flagLimit != 0 { params["limit"] = formatCLIParamValue(flagLimit) } - data, prov, err := resolveReadWithStrategy(cmd.Context(), c, flags, "auto", "graph", false, path, params, nil, cmd.ErrOrStderr()) + data, prov, err := resolveReadWithStrategy(cmd.Context(), c, flags, "auto", "graph", true, path, params, nil, cmd.ErrOrStderr()) if err != nil { return classifyAPIError(err, flags) } diff --git a/library/productivity/devonthink/internal/cli/records_search.go b/library/productivity/devonthink/internal/cli/records_search.go index 9d9a0aaf2e..900fa428bc 100644 --- a/library/productivity/devonthink/internal/cli/records_search.go +++ b/library/productivity/devonthink/internal/cli/records_search.go @@ -58,7 +58,7 @@ func newRecordsSearchCmd(flags *rootFlags) *cobra.Command { if flagLimit != 0 { params["limit"] = formatCLIParamValue(flagLimit) } - data, prov, err := resolveReadWithStrategy(cmd.Context(), c, flags, "auto", "records", false, path, params, nil, cmd.ErrOrStderr()) + data, prov, err := resolveReadWithStrategy(cmd.Context(), c, flags, "auto", "records", true, path, params, nil, cmd.ErrOrStderr()) if err != nil { return classifyAPIError(err, flags) }